diff --git a/Makefile b/Makefile index 16920a1..a2f9e49 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: test test-no-cache test-stores test-stores-no-cache test-integration-all test-integration-all-no-cache test-integration-stores test-integration-stores-no-cache test-integration-postgres test-integration-postgres-no-cache test-integration-mssql test-integration-mssql-no-cache test-integration-msql +.PHONY: test test-no-cache test-stores test-stores-no-cache test-integration-all test-integration-all-no-cache test-integration-stores test-integration-stores-no-cache test-integration-postgres test-integration-postgres-no-cache GO ?= go GOTOOLCHAIN ?= local @@ -37,11 +37,3 @@ test-integration-postgres: test-integration-postgres-no-cache: $(GO_ENV) $(GO) test -count=1 -mod=$(MOD_MODE) -tags=$(INTEGRATION_TAG) -timeout=$(TEST_TIMEOUT) $(TEST_FLAGS) ./stores/postgres - -test-integration-mssql: - $(GO_ENV) $(GO) test -mod=$(MOD_MODE) -tags=$(INTEGRATION_TAG) -timeout=$(TEST_TIMEOUT) $(TEST_FLAGS) ./stores/mssql - -test-integration-mssql-no-cache: - $(GO_ENV) $(GO) test -count=1 -mod=$(MOD_MODE) -tags=$(INTEGRATION_TAG) -timeout=$(TEST_TIMEOUT) $(TEST_FLAGS) ./stores/mssql - -test-integration-msql: test-integration-mssql diff --git a/README.md b/README.md index e2d5289..ebe207f 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,6 @@ A lightweight Go vector-store library inspired by `Microsoft.Extensions.VectorDa MVP scope: - Go 1.24.x - Postgres + `pgvector` -- MSSQL connector - Record-based core API with optional typed codec wrapper This library can be used to build retrieval systems such as: @@ -22,14 +21,12 @@ This library can be used to build retrieval systems such as: - Go 1.24.x - PostgreSQL 16+ with `pgvector` extension -- SQL Server 2022+ (for `stores/mssql`) - Docker (optional, for integration tests and sample compose flows) ## Project layout - `vectordata`: backend-agnostic core interfaces, record model, filters, typed wrapper - `stores/postgres`: Postgres implementation with `pgxpool` -- `stores/mssql`: SQL Server implementation with `database/sql` - `samples`: runnable demos (see `samples/README.md`) - `docs`: architecture and implementation notes @@ -197,25 +194,15 @@ go test ./... ``` Notes: -- Integration tests start Postgres/pgvector and SQL Server automatically via Testcontainers +- Integration tests start Postgres/pgvector automatically via Testcontainers - Docker daemon must be available when running integration tests - Optional override: set `PGVECTOR_TEST_DSN` to use an existing Postgres instance instead of starting a container -- Optional override: set `MSSQL_TEST_DSN` to use an existing SQL Server instance instead of starting a container - -Run MSSQL integration tests against root compose service: - -```bash -docker compose up -d mssql -MSSQL_TEST_DSN="sqlserver://sa:YourStrong%21Passw0rd@localhost:14339?database=master&encrypt=disable" \ - go test -tags=integration ./stores/mssql -``` ## Docker Compose (optional) `docker-compose.yml` at the repository root is kept for manual local runs. - Use root `docker-compose.yml` when you want a persistent local Postgres+pgvector instance outside tests -- Root compose also includes SQL Server (`mssql`) for local MSSQL connector validation - Use Testcontainers (`go test -tags=integration ./...`) for integration tests - Sample apps have their own compose files at `samples/semantic-search/docker-compose.yml` and `samples/ragrimosa/docker-compose.yml` - Sample Dockerfiles use `golang:1.24-alpine` to match the repo Go version (`1.24.x`) @@ -233,8 +220,9 @@ Release options: 2. Tag-driven: push a semver tag and the workflow publishes release notes automatically: ```bash -git tag v0.2.3 -git push origin v0.2.3 +VERSION="$(cat VERSION)" +git tag "v${VERSION}" +git push origin "v${VERSION}" ``` ## Samples @@ -247,6 +235,7 @@ git push origin v0.2.3 - Docs index: [`docs/README.md`](docs/README.md) - Internals and architecture: [`docs/architecture.md`](docs/architecture.md) +- Connector development guide: [`docs/connector-development.md`](docs/connector-development.md) ## License diff --git a/VERSION b/VERSION new file mode 100644 index 0000000..7179039 --- /dev/null +++ b/VERSION @@ -0,0 +1 @@ +0.2.3 diff --git a/docker-compose.yml b/docker-compose.yml index 3d6a6a8..56399e9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -13,13 +13,3 @@ services: interval: 2s timeout: 2s retries: 30 - - mssql: - image: mcr.microsoft.com/mssql/server:2022-latest - container_name: go-vectorstore-mssql - environment: - ACCEPT_EULA: "Y" - MSSQL_PID: "Developer" - MSSQL_SA_PASSWORD: "YourStrong!Passw0rd" - ports: - - "14339:1433" diff --git a/docs/README.md b/docs/README.md index d0de301..efd30cd 100644 --- a/docs/README.md +++ b/docs/README.md @@ -4,3 +4,4 @@ This directory contains internal design and implementation notes for `go-vectors - [`architecture.md`](architecture.md): architecture, request flow, schema behavior, filter system, and index model - [`stores-implementation.md`](stores-implementation.md): detailed store-backend design, component responsibilities, flows, invariants, and backend differences +- [`connector-development.md`](connector-development.md): step-by-step checklist for implementing a new backend connector diff --git a/docs/architecture.md b/docs/architecture.md index fdbfa0b..019e238 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -8,7 +8,6 @@ The project is split into two layers: - `vectordata`: backend-agnostic contracts and primitives - `stores/postgres`: PostgreSQL + pgvector implementation -- `stores/mssql`: SQL Server implementation This keeps the public API stable while allowing additional storage engines later. @@ -58,11 +57,6 @@ All methods require `context.Context`. - `EnsureExtension`: `true` - `StrictByDefault`: `true` -`mssql.StoreOptions` defaults (`mssql.DefaultStoreOptions()`): - -- `Schema`: `dbo` -- `StrictByDefault`: `true` - Collection defaults: - Metric defaults to cosine when omitted @@ -127,23 +121,6 @@ Each record is validated before sending: The pgvector operators are documented in the [pgvector README](https://github.com/pgvector/pgvector#querying). -### MSSQL: Ensure / Search - -`MSSQLVectorStore.EnsureCollection`: - -1. Ensures target schema exists -2. Ensures internal collection metadata table exists -3. Creates or validates the collection table -4. Persists and validates dimension/metric metadata - -`MSSQLCollection.SearchByVector`: - -1. Streams records from SQL Server -2. Evaluates filters against records in-process -3. Computes distance in-process (cosine/l2/inner product) -4. Applies threshold and keeps a bounded in-memory top-k heap -5. Returns top-k sorted by distance - ## 6) Filter System (AST -> SQL) Filters are represented as an AST in `vectordata`: @@ -170,8 +147,6 @@ Behavior details: JSON path extraction behavior comes from PostgreSQL [JSON/JSONB functions and operators](https://www.postgresql.org/docs/current/functions-json.html). -For MSSQL in this MVP, the same AST is evaluated in-process against loaded records. - ## 7) Schema Safety Modes `CollectionSpec.Mode` controls ensure behavior: @@ -264,7 +239,6 @@ Errors are wrapped with context so callers can use `errors.Is(...)` against base Current MVP scope: - Postgres + pgvector backend -- MSSQL backend (vectors stored as JSON payloads) - single-vector column per collection - metadata filtering through a focused AST @@ -273,10 +247,8 @@ Future extension points: - additional backends under `stores/` - richer filter operators - optional reranking strategies -- native MSSQL vector indexing/query pushdown ## 13) References - pgvector project docs: https://github.com/pgvector/pgvector - PostgreSQL docs: https://www.postgresql.org/docs/current/index.html -- SQL Server docs: https://learn.microsoft.com/sql/sql-server/ diff --git a/docs/connector-development.md b/docs/connector-development.md new file mode 100644 index 0000000..08e923a --- /dev/null +++ b/docs/connector-development.md @@ -0,0 +1,159 @@ +# Connector Development Guide + +This guide explains how to add a new backend connector under `stores/`. + +## 1) Scope and Contract + +Every connector must implement the shared interfaces in `vectordata`: + +- `vectordata.VectorStore` +- `vectordata.Collection` + +The connector must preserve shared behavior: + +- validate collection specs (`name`, `dimension`, `metric`, `mode`) +- enforce vector dimension on write and search +- return `vectordata.ErrNotFound` for missing records +- return `vectordata.ErrDimensionMismatch`, `vectordata.ErrSchemaMismatch`, and `vectordata.ErrInvalidFilter` where applicable +- compute score via `vectordata.ScoreFromDistance` + +## 2) Recommended Layout + +Create a new directory: + +- `stores//doc.go` +- `stores//store.go` +- `stores//collection.go` +- `stores//schema.go` +- `stores//helpers.go` +- `stores//_integration_test.go` + +Add extra files only when needed (for example filter compiler/evaluator files). + +## 3) Store Implementation Checklist + +Implement a store type that owns connection/resources and options: + +1. Add `StoreOptions` and `DefaultStoreOptions()`. +2. Add `NewVectorStore(...)` constructor with option normalization. +3. Implement `EnsureCollection(ctx, spec)`: + - normalize and validate spec + - ensure schema/table/metadata structures + - create or validate collection storage +4. Implement `Collection(name, dimension, metric)` as a lightweight handle constructor. + +## 4) Collection Implementation Checklist + +Implement collection operations with strict validation: + +1. `Insert` and `Upsert` + - validate ID and vector dimension + - normalize nil metadata to empty object + - use batching/chunking for bulk writes +2. `Get` and `Delete` + - return `ErrNotFound` from `Get` when missing +3. `Count` + - support nil filter and filter predicates +4. `SearchByVector` + - validate `topK > 0` + - validate query vector dimension + - apply filter (SQL pushdown or in-process evaluation) + - apply optional threshold + - honor projection (`Metadata`, `Content`, `Vector`) + - return results ordered by best match first + - compute `Score` from `Distance` +5. `EnsureIndexes` + - create backend-specific indexes when supported + - return explicit error if unsupported options are requested + +## 5) Filter Strategy + +Choose one strategy: + +- SQL pushdown: compile `vectordata.Filter` to parameterized backend SQL +- In-process: load records and evaluate filter AST in Go + +Requirements: + +- never interpolate raw values into SQL +- return `vectordata.ErrInvalidFilter` for invalid AST/input + +## 6) Schema Safety Modes + +Respect `CollectionSpec.Mode`: + +- `EnsureStrict`: fail on mismatches +- `EnsureAutoMigrate`: add/fix optional schema parts where possible + +When mode is unset, use connector defaults (`StrictByDefault` behavior). + +## 7) Testing Requirements + +Add both unit and integration coverage. + +Unit tests: + +- spec validation +- schema mismatch behavior +- filter behavior (happy path + invalid filters) +- dimension mismatch and error mapping +- projection and threshold behavior in search + +Integration tests: + +- put in `stores//_integration_test.go` +- use `//go:build integration` +- start backend with Testcontainers when DSN env var is absent +- allow DSN override via `_TEST_DSN` + +Run commands: + +```bash +go test ./... +go test -tags=integration ./stores/ +``` + +## 8) Repository Wiring + +When adding a connector, also update: + +1. `README.md`: + - scope + - requirements + - project layout + - integration test notes +2. `docs/architecture.md` and `docs/stores-implementation.md` +3. `docs/README.md` +4. `Makefile` integration targets (if per-backend targets are used) +5. `docker-compose.yml` only if manual local backend compose is required +6. `go.mod` and `go.sum` dependencies + +## 9) Minimal Skeleton + +```go +package mybackend + +import ( + "context" + + "github.com/gabisonia/go-vectorstore/vectordata" +) + +type StoreOptions struct{} + +func DefaultStoreOptions() StoreOptions { return StoreOptions{} } + +type VectorStore struct{} + +func NewVectorStore(opts StoreOptions) (*VectorStore, error) { + return &VectorStore{}, nil +} + +func (s *VectorStore) EnsureCollection(ctx context.Context, spec vectordata.CollectionSpec) (vectordata.Collection, error) { + return nil, nil +} + +func (s *VectorStore) Collection(name string, dimension int, metric vectordata.DistanceMetric) vectordata.Collection { + return nil +} +``` diff --git a/docs/stores-implementation.md b/docs/stores-implementation.md index e3f26bd..48870cc 100644 --- a/docs/stores-implementation.md +++ b/docs/stores-implementation.md @@ -7,7 +7,7 @@ This document explains how `go-vectorstore` store backends are implemented, how - Keep a single backend-agnostic public API in `vectordata` - Isolate backend specifics inside `stores/` - Enforce schema and vector-dimension correctness at runtime -- Keep write/read/search behavior consistent across backends where possible +- Keep write/read/search behavior consistent across implementations - Allow adding future store implementations without changing caller-facing contracts ## 2) Layering and Boundaries @@ -16,7 +16,6 @@ The implementation is split into: - `vectordata`: shared contracts, record/search types, filter AST, filter SQL compiler, typed codec wrapper, error model - `stores/postgres`: PostgreSQL + pgvector implementation -- `stores/mssql`: SQL Server implementation Each backend implements: @@ -121,75 +120,7 @@ content text - Metadata GIN index - optional `jsonb_path_ops` -## 5) MSSQL Store (`stores/mssql`) - -### 5.1 Main Components - -- `MSSQLVectorStore` (`store.go`) - - Owns `*sql.DB` and options (`Schema`, `StrictByDefault`) - - Validates/normalizes collection specs - - Ensures schema and metadata table - - Creates/validates collection table -- `MSSQLCollection` (`collection.go`) - - Implements collection operations - - Uses transactional batched writes - - Executes similarity + filters in process for this MVP -- Schema utilities (`schema.go`) - - Ensures schema - - Ensures internal metadata table `__vector_collections` - - Validates collection table columns/PK - - Stores/validates logical dimension and metric in metadata table -- Filter evaluator (`filter_eval.go`) - - Evaluates filter AST directly against loaded records (column + metadata paths) -- Helpers (`helpers.go`) - - JSON encoding/decoding for vector and metadata - - Distance math (cosine/l2/inner product) - - projection resolver and type helpers - -### 5.2 Data Model and Metadata Strategy - -MSSQL MVP stores: - -- vector as `NVARCHAR(MAX)` JSON (not native vector type) -- metadata as `NVARCHAR(MAX)` JSON -- content as `NVARCHAR(MAX)` nullable - -Because vector dimension and metric are not part of physical type, they are tracked in `__vector_collections`: - -- `name` (PK) -- `dimension` -- `metric` - -`EnsureCollection` validates both table shape and metadata row consistency. - -### 5.3 Write / Read / Search Path - -Write behavior: - -- Validate `ID` and vector dimension -- Serialize vector + metadata to JSON -- Chunk in batches of 500 -- Use transaction per chunk -- Upsert strategy: - - execute single-statement `UPDATE ... WITH (UPDLOCK, SERIALIZABLE)` + conditional `INSERT` - - avoids race conditions under concurrent upserts for the same ID - -Search behavior (MVP): - -1. Validate `topK` and vector dimension -2. Stream records row-by-row from SQL -3. Evaluate filter AST in-process (`matchesFilter`) -4. Compute distance in-process (`distanceBetween`) -5. Apply optional threshold -6. Maintain bounded top-k max-heap in memory (`O(k)` memory) -7. Return results sorted by distance asc (tie-break by `ID`) - -`EnsureIndexes` is explicit in MSSQL backend: - -- returns `nil` when no index options are requested -- returns an error when vector/metadata index options are provided (index management is not supported yet) - -## 6) Filter System and Execution Model +## 5) Filter System and Execution Model Filter AST supports: @@ -200,7 +131,6 @@ Filter AST supports: Execution strategy by backend: - Postgres: compile AST -> parameterized SQL via `CompileFilterSQL` -- MSSQL: evaluate AST in Go against streamed records Important behavior: @@ -208,7 +138,7 @@ Important behavior: - Numeric comparisons are numeric when values are numeric; otherwise textual comparison is used - Missing fields typically evaluate as non-match, except `Exists` which reports presence -## 7) Schema Safety Modes +## 6) Schema Safety Modes `CollectionSpec.Mode` controls schema handling: @@ -220,9 +150,9 @@ Mode default comes from backend options: - `StrictByDefault=true` -> strict mode when spec mode is unset - `StrictByDefault=false` -> auto-migrate mode when spec mode is unset -## 8) Cross-Backend Invariants +## 7) Invariants -These rules are enforced in both backends: +These rules are enforced in all current implementations: - Collection name must be non-empty - Dimension must be greater than zero @@ -232,25 +162,7 @@ These rules are enforced in both backends: - Nil metadata is normalized to empty object - `Get` returns `ErrNotFound` on missing ID -## 9) Key Differences: Postgres vs MSSQL - -- Vector storage: - - Postgres: native `vector(n)` type via pgvector - - MSSQL: JSON string payload (`NVARCHAR(MAX)`) -- Search execution: - - Postgres: SQL-level distance computation and ordering - - MSSQL: in-memory computation after loading rows -- Filter execution: - - Postgres: SQL compilation and pushdown - - MSSQL: in-process AST evaluation -- Indexing: - - Postgres: vector + metadata index creation supported - - MSSQL: explicit unsupported error when index options are requested -- Metric/dimension persistence: - - Postgres: inferred/validated from physical `vector(n)` column - - MSSQL: persisted in `__vector_collections` - -## 10) Extending with a New Backend +## 8) Extending with a New Backend To add a new store backend, follow the same contract shape: @@ -260,3 +172,5 @@ To add a new store backend, follow the same contract shape: 4. Support filter handling (compiled or in-process) 5. Reuse `vectordata` errors and scoring semantics 6. Add integration tests matching existing backend test coverage style + +For a practical checklist and repository wiring steps, see [`connector-development.md`](connector-development.md). diff --git a/go.mod b/go.mod index 11541c9..a594ff5 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,6 @@ toolchain go1.24.9 require ( github.com/jackc/pgx/v5 v5.7.2 - github.com/microsoft/go-mssqldb v1.8.0 github.com/testcontainers/testcontainers-go v0.33.0 ) diff --git a/go.sum b/go.sum index f39156c..7c5c27b 100644 --- a/go.sum +++ b/go.sum @@ -61,8 +61,6 @@ github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= -github.com/microsoft/go-mssqldb v1.8.0 h1:7cyZ/AT7ycDsEoWPIXibd+aVKFtteUNhDGf3aobP+tw= -github.com/microsoft/go-mssqldb v1.8.0/go.mod h1:6znkekS3T2vp0waiMhen4GPU1BiAsrP+iXHcE7a7rFo= github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= github.com/moby/patternmatcher v0.6.0 h1:GmP9lR19aU5GqSSFko+5pRqHi+Ohk1O69aFiKkVGiPk= diff --git a/stores/mssql/collection.go b/stores/mssql/collection.go deleted file mode 100644 index 579fe11..0000000 --- a/stores/mssql/collection.go +++ /dev/null @@ -1,467 +0,0 @@ -package mssql - -import ( - "container/heap" - "context" - "database/sql" - "errors" - "fmt" - "sort" - "strings" - - "github.com/gabisonia/go-vectorstore/vectordata" -) - -type writeMode int - -const ( - writeModeInsert writeMode = iota - writeModeUpsert -) - -// MSSQLCollection is a SQL Server-backed vector collection. -type MSSQLCollection struct { - store *MSSQLVectorStore - name string - dimension int - metric vectordata.DistanceMetric -} - -func (c *MSSQLCollection) Name() string { - return c.name -} - -func (c *MSSQLCollection) Dimension() int { - return c.dimension -} - -func (c *MSSQLCollection) Metric() vectordata.DistanceMetric { - return c.metric -} - -func (c *MSSQLCollection) Insert(ctx context.Context, records []vectordata.Record) error { - return c.writeRecords(ctx, records, writeModeInsert) -} - -func (c *MSSQLCollection) Upsert(ctx context.Context, records []vectordata.Record) error { - return c.writeRecords(ctx, records, writeModeUpsert) -} - -func (c *MSSQLCollection) Get(ctx context.Context, id string) (vectordata.Record, error) { - query := fmt.Sprintf("SELECT %s, %s, %s, %s FROM %s WHERE %s = @p1", - quoteIdent(idColumn), - quoteIdent(vectorColumn), - quoteIdent(metadataColumn), - quoteIdent(contentColumn), - c.tableName(), - quoteIdent(idColumn), - ) - - var out vectordata.Record - var vectorRaw string - var metadataRaw string - var content sql.NullString - - err := c.store.db.QueryRowContext(ctx, query, id).Scan(&out.ID, &vectorRaw, &metadataRaw, &content) - if err != nil { - if errors.Is(err, sql.ErrNoRows) { - return vectordata.Record{}, vectordata.ErrNotFound - } - return vectordata.Record{}, err - } - - vector, err := parseVectorJSON(vectorRaw) - if err != nil { - return vectordata.Record{}, fmt.Errorf("decode vector: %w", err) - } - metadata, err := parseMetadataJSON(metadataRaw) - if err != nil { - return vectordata.Record{}, fmt.Errorf("decode metadata: %w", err) - } - - out.Vector = vector - out.Metadata = metadata - if content.Valid { - value := content.String - out.Content = &value - } - - return out, nil -} - -func (c *MSSQLCollection) Delete(ctx context.Context, ids []string) (int64, error) { - if len(ids) == 0 { - return 0, nil - } - - args := make([]any, 0, len(ids)) - placeholders := make([]string, 0, len(ids)) - for i, id := range ids { - placeholders = append(placeholders, fmt.Sprintf("@p%d", i+1)) - args = append(args, id) - } - - query := fmt.Sprintf("DELETE FROM %s WHERE %s IN (%s)", - c.tableName(), - quoteIdent(idColumn), - strings.Join(placeholders, ", "), - ) - result, err := c.store.db.ExecContext(ctx, query, args...) - if err != nil { - return 0, err - } - rowsAffected, err := result.RowsAffected() - if err != nil { - return 0, err - } - return rowsAffected, nil -} - -func (c *MSSQLCollection) Count(ctx context.Context, filter vectordata.Filter) (int64, error) { - if filter == nil { - query := fmt.Sprintf("SELECT COUNT(*) FROM %s", c.tableName()) - var count int64 - if err := c.store.db.QueryRowContext(ctx, query).Scan(&count); err != nil { - return 0, err - } - return count, nil - } - - filterSQL, filterArgs, _, err := compileMSSQLFilterSQL(filter, 1) - if err == nil { - query := fmt.Sprintf("SELECT COUNT(*) FROM %s WHERE %s", c.tableName(), filterSQL) - var count int64 - if err := c.store.db.QueryRowContext(ctx, query, filterArgs...).Scan(&count); err != nil { - return 0, err - } - return count, nil - } - if !errors.Is(err, errFilterPushdownUnsupported) { - return 0, err - } - - count := int64(0) - if err := c.streamRecords(ctx, false, func(record vectordata.Record) error { - matches, err := matchesFilter(filter, record) - if err != nil { - return err - } - if matches { - count++ - } - return nil - }); err != nil { - return 0, err - } - return count, nil -} - -func (c *MSSQLCollection) SearchByVector(ctx context.Context, vector []float32, topK int, opts vectordata.SearchOptions) ([]vectordata.SearchResult, error) { - if topK <= 0 { - return nil, fmt.Errorf("topK must be > 0") - } - if err := c.validateVectorDimension(vector); err != nil { - return nil, err - } - - plan, err := c.buildSearchSQLPlan(vector, topK, opts) - if err == nil { - return c.executeSearchSQLPlan(ctx, plan) - } - if !errors.Is(err, errFilterPushdownUnsupported) { - return nil, err - } - - return c.searchByVectorStreaming(ctx, vector, topK, opts) -} - -func (c *MSSQLCollection) searchByVectorStreaming(ctx context.Context, vector []float32, topK int, opts vectordata.SearchOptions) ([]vectordata.SearchResult, error) { - projection := resolveProjection(opts.Projection) - metric := defaultMetric(c.metric) - topKHeap := make(searchResultMaxHeap, 0, topK) - heap.Init(&topKHeap) - if err := c.streamRecords(ctx, true, func(record vectordata.Record) error { - if err := c.validateVectorDimension(record.Vector); err != nil { - return fmt.Errorf("invalid stored vector for record %q: %w", record.ID, err) - } - - matches, err := matchesFilter(opts.Filter, record) - if err != nil { - return err - } - if !matches { - return nil - } - - distance, err := distanceBetween(metric, vector, record.Vector) - if err != nil { - return err - } - if opts.Threshold != nil && distance > *opts.Threshold { - return nil - } - - candidate := vectordata.SearchResult{ - Record: projectRecord(record, projection), - Distance: distance, - Score: vectordata.ScoreFromDistance(metric, distance), - } - - if topKHeap.Len() < topK { - heap.Push(&topKHeap, candidate) - return nil - } - worst := topKHeap[0] - if isBetterResult(candidate, worst) { - heap.Pop(&topKHeap) - heap.Push(&topKHeap, candidate) - } - return nil - }); err != nil { - return nil, err - } - - results := make([]vectordata.SearchResult, 0, topKHeap.Len()) - for topKHeap.Len() > 0 { - results = append(results, heap.Pop(&topKHeap).(vectordata.SearchResult)) - } - sort.Slice(results, func(i, j int) bool { return isBetterResult(results[i], results[j]) }) - - return results, nil -} - -func (c *MSSQLCollection) EnsureIndexes(ctx context.Context, opts vectordata.IndexOptions) error { - _ = ctx - if opts.Vector == nil && opts.Metadata == nil { - return nil - } - return fmt.Errorf("%w: index management is not supported by the mssql backend", vectordata.ErrSchemaMismatch) -} - -func (c *MSSQLCollection) writeRecords(ctx context.Context, records []vectordata.Record, mode writeMode) error { - if len(records) == 0 { - return nil - } - - insertQuery := fmt.Sprintf("INSERT INTO %s (%s, %s, %s, %s) VALUES (@p1, @p2, @p3, @p4)", - c.tableName(), - quoteIdent(idColumn), - quoteIdent(vectorColumn), - quoteIdent(metadataColumn), - quoteIdent(contentColumn), - ) - upsertQuery := buildUpsertQuery(c.tableName()) - - for start := 0; start < len(records); start += maxRowsPerStatement { - end := start + maxRowsPerStatement - if end > len(records) { - end = len(records) - } - - tx, err := c.store.db.BeginTx(ctx, nil) - if err != nil { - return err - } - if err := c.writeBatch(ctx, tx, records[start:end], mode, insertQuery, upsertQuery); err != nil { - _ = tx.Rollback() - return err - } - if err := tx.Commit(); err != nil { - _ = tx.Rollback() - return err - } - } - - return nil -} - -func (c *MSSQLCollection) writeBatch( - ctx context.Context, - tx *sql.Tx, - records []vectordata.Record, - mode writeMode, - insertQuery string, - upsertQuery string, -) error { - for _, record := range records { - if strings.TrimSpace(record.ID) == "" { - return fmt.Errorf("record id is empty") - } - if err := c.validateVectorDimension(record.Vector); err != nil { - return err - } - - vectorPayload, err := vectorJSON(record.Vector) - if err != nil { - return fmt.Errorf("encode vector for record %q: %w", record.ID, err) - } - metadataPayload, err := metadataJSON(record.Metadata) - if err != nil { - return fmt.Errorf("encode metadata for record %q: %w", record.ID, err) - } - - var contentArg any - if record.Content != nil { - contentArg = *record.Content - } - - switch mode { - case writeModeInsert: - if _, err := tx.ExecContext(ctx, insertQuery, record.ID, vectorPayload, metadataPayload, contentArg); err != nil { - return err - } - case writeModeUpsert: - if _, err := tx.ExecContext(ctx, upsertQuery, record.ID, vectorPayload, metadataPayload, contentArg); err != nil { - return err - } - default: - return fmt.Errorf("unsupported write mode %d", mode) - } - } - - return nil -} - -func (c *MSSQLCollection) streamRecords(ctx context.Context, includeVector bool, yield func(vectordata.Record) error) error { - selectColumns := []string{quoteIdent(idColumn)} - if includeVector { - selectColumns = append(selectColumns, quoteIdent(vectorColumn)) - } - selectColumns = append(selectColumns, quoteIdent(metadataColumn), quoteIdent(contentColumn)) - - query := fmt.Sprintf("SELECT %s FROM %s", strings.Join(selectColumns, ", "), c.tableName()) - rows, err := c.store.db.QueryContext(ctx, query) - if err != nil { - return err - } - defer rows.Close() - - for rows.Next() { - var record vectordata.Record - var vectorRaw string - var metadataRaw string - var content sql.NullString - - if includeVector { - if err := rows.Scan(&record.ID, &vectorRaw, &metadataRaw, &content); err != nil { - return err - } - parsedVector, err := parseVectorJSON(vectorRaw) - if err != nil { - return fmt.Errorf("decode vector: %w", err) - } - record.Vector = parsedVector - } else { - if err := rows.Scan(&record.ID, &metadataRaw, &content); err != nil { - return err - } - } - - metadata, err := parseMetadataJSON(metadataRaw) - if err != nil { - return fmt.Errorf("decode metadata: %w", err) - } - record.Metadata = metadata - - if content.Valid { - value := content.String - record.Content = &value - } - - if err := yield(record); err != nil { - return err - } - } - if err := rows.Err(); err != nil { - return err - } - return nil -} - -func (c *MSSQLCollection) validateVectorDimension(vector []float32) error { - if len(vector) != c.dimension { - return fmt.Errorf("%w: expected %d, got %d", vectordata.ErrDimensionMismatch, c.dimension, len(vector)) - } - return nil -} - -func (c *MSSQLCollection) tableName() string { - return qualifiedTable(c.store.opts.Schema, c.name) -} - -func projectRecord(record vectordata.Record, projection vectordata.Projection) vectordata.Record { - projected := vectordata.Record{ID: record.ID} - if projection.IncludeVector { - projected.Vector = append([]float32(nil), record.Vector...) - } - if projection.IncludeMetadata { - metadataCopy := make(map[string]any, len(record.Metadata)) - for key, value := range record.Metadata { - metadataCopy[key] = value - } - projected.Metadata = metadataCopy - } - if projection.IncludeContent && record.Content != nil { - contentCopy := *record.Content - projected.Content = &contentCopy - } - return projected -} - -// buildUpsertQuery uses key-range locks so concurrent upserts on the same ID remain atomic. -func buildUpsertQuery(tableName string) string { - return fmt.Sprintf(`UPDATE %s WITH (UPDLOCK, SERIALIZABLE) -SET %s = @p2, %s = @p3, %s = @p4 -WHERE %s = @p1; -IF @@ROWCOUNT = 0 -BEGIN - INSERT INTO %s (%s, %s, %s, %s) VALUES (@p1, @p2, @p3, @p4); -END`, - tableName, - quoteIdent(vectorColumn), - quoteIdent(metadataColumn), - quoteIdent(contentColumn), - quoteIdent(idColumn), - tableName, - quoteIdent(idColumn), - quoteIdent(vectorColumn), - quoteIdent(metadataColumn), - quoteIdent(contentColumn), - ) -} - -type searchResultMaxHeap []vectordata.SearchResult - -func (h *searchResultMaxHeap) Len() int { return len(*h) } - -func (h *searchResultMaxHeap) Less(i, j int) bool { - return isWorseResult((*h)[i], (*h)[j]) -} - -func (h *searchResultMaxHeap) Swap(i, j int) { (*h)[i], (*h)[j] = (*h)[j], (*h)[i] } - -func (h *searchResultMaxHeap) Push(x any) { - *h = append(*h, x.(vectordata.SearchResult)) -} - -func (h *searchResultMaxHeap) Pop() any { - old := *h - n := len(old) - item := old[n-1] - *h = old[:n-1] - return item -} - -func isBetterResult(left, right vectordata.SearchResult) bool { - if left.Distance == right.Distance { - return left.Record.ID < right.Record.ID - } - return left.Distance < right.Distance -} - -func isWorseResult(left, right vectordata.SearchResult) bool { - if left.Distance == right.Distance { - return left.Record.ID > right.Record.ID - } - return left.Distance > right.Distance -} diff --git a/stores/mssql/collection_test.go b/stores/mssql/collection_test.go deleted file mode 100644 index 61ec261..0000000 --- a/stores/mssql/collection_test.go +++ /dev/null @@ -1,136 +0,0 @@ -package mssql - -import ( - "container/heap" - "context" - "errors" - "reflect" - "sort" - "strings" - "testing" - - "github.com/gabisonia/go-vectorstore/vectordata" -) - -func TestBuildUpsertQueryUsesLockingPattern(t *testing.T) { - query := buildUpsertQuery("[dbo].[docs]") - - if !strings.Contains(query, "WITH (UPDLOCK, SERIALIZABLE)") { - t.Fatalf("expected upsert query to use locking hint, got: %s", query) - } - if !strings.Contains(query, "IF @@ROWCOUNT = 0") { - t.Fatalf("expected upsert query to insert on miss, got: %s", query) - } - if !strings.Contains(query, "INSERT INTO [dbo].[docs]") { - t.Fatalf("expected upsert query to target provided table, got: %s", query) - } -} - -func TestSearchResultHeapKeepsBestTopK(t *testing.T) { - const topK = 3 - - h := make(searchResultMaxHeap, 0, topK) - heap.Init(&h) - - pushTopK := func(id string, distance float64) { - candidate := vectordata.SearchResult{ - Record: vectordata.Record{ID: id}, - Distance: distance, - } - if h.Len() < topK { - heap.Push(&h, candidate) - return - } - if isBetterResult(candidate, h[0]) { - heap.Pop(&h) - heap.Push(&h, candidate) - } - } - - pushTopK("x", 0.40) - pushTopK("a", 0.10) - pushTopK("b", 0.20) - pushTopK("c", 0.20) // Same distance as b; tie-break should keep b. - pushTopK("d", 0.05) - - results := make([]vectordata.SearchResult, 0, topK) - for h.Len() > 0 { - results = append(results, heap.Pop(&h).(vectordata.SearchResult)) - } - sort.Slice(results, func(i, j int) bool { return isBetterResult(results[i], results[j]) }) - - if len(results) != topK { - t.Fatalf("expected %d results, got %d", topK, len(results)) - } - - expected := []string{"d", "a", "b"} - for i := range expected { - if results[i].Record.ID != expected[i] { - t.Fatalf("unexpected result order at %d: expected %q got %q", i, expected[i], results[i].Record.ID) - } - } -} - -func TestEnsureIndexesMSSQL(t *testing.T) { - collection := &MSSQLCollection{} - - if err := collection.EnsureIndexes(context.Background(), vectordata.IndexOptions{}); err != nil { - t.Fatalf("expected empty options to succeed, got %v", err) - } - - err := collection.EnsureIndexes(context.Background(), vectordata.IndexOptions{ - Vector: &vectordata.VectorIndexOptions{Method: vectordata.IndexMethodHNSW}, - }) - if err == nil { - t.Fatal("expected error when index options are provided") - } - if !errors.Is(err, vectordata.ErrSchemaMismatch) { - t.Fatalf("expected ErrSchemaMismatch, got %v", err) - } -} - -func TestBuildSearchSQLPlan(t *testing.T) { - threshold := 0.55 - collection := &MSSQLCollection{ - store: &MSSQLVectorStore{opts: StoreOptions{Schema: "dbo"}}, - name: "docs", - dimension: 2, - metric: vectordata.DistanceCosine, - } - - plan, err := collection.buildSearchSQLPlan([]float32{1, 0}, 3, vectordata.SearchOptions{ - Threshold: &threshold, - Projection: &vectordata.Projection{ - IncludeMetadata: true, - IncludeContent: true, - }, - }) - if err != nil { - t.Fatalf("buildSearchSQLPlan: %v", err) - } - if !strings.Contains(plan.query, "FETCH NEXT @p4 ROWS ONLY") { - t.Fatalf("expected limit placeholder in query, got: %s", plan.query) - } - if !reflect.DeepEqual(plan.args, []any{"[1,0]", 2, threshold, 3}) { - t.Fatalf("unexpected args: %#v", plan.args) - } -} - -func TestBuildSearchSQLPlanRejectsUnsupportedFilterPushdown(t *testing.T) { - collection := &MSSQLCollection{ - store: &MSSQLVectorStore{opts: StoreOptions{Schema: "dbo"}}, - name: "docs", - dimension: 2, - metric: vectordata.DistanceCosine, - } - - _, err := collection.buildSearchSQLPlan([]float32{1, 0}, 3, vectordata.SearchOptions{ - Filter: vectordata.Eq(vectordata.Column("id"), 123), - }) - if err == nil { - t.Fatal("expected error") - } - if !errors.Is(err, errFilterPushdownUnsupported) { - t.Fatalf("expected errFilterPushdownUnsupported, got %v", err) - } -} diff --git a/stores/mssql/doc.go b/stores/mssql/doc.go deleted file mode 100644 index f025ff7..0000000 --- a/stores/mssql/doc.go +++ /dev/null @@ -1,2 +0,0 @@ -// Package mssql provides a SQL Server-backed vectordata implementation. -package mssql diff --git a/stores/mssql/filter_eval.go b/stores/mssql/filter_eval.go deleted file mode 100644 index c376d0e..0000000 --- a/stores/mssql/filter_eval.go +++ /dev/null @@ -1,198 +0,0 @@ -package mssql - -import ( - "fmt" - "reflect" - - "github.com/gabisonia/go-vectorstore/vectordata" -) - -func matchesFilter(filter vectordata.Filter, record vectordata.Record) (bool, error) { - if filter == nil { - return true, nil - } - - switch node := filter.(type) { - case vectordata.EqFilter: - left, exists, err := resolveFieldValue(node.Field, record) - if err != nil { - return false, err - } - if !exists { - return false, nil - } - return valuesEqual(left, node.Value), nil - case vectordata.InFilter: - if len(node.Values) == 0 { - return false, fmt.Errorf("%w: IN requires at least one value", vectordata.ErrInvalidFilter) - } - left, exists, err := resolveFieldValue(node.Field, record) - if err != nil { - return false, err - } - if !exists { - return false, nil - } - for _, value := range node.Values { - if valuesEqual(left, value) { - return true, nil - } - } - return false, nil - case vectordata.GtFilter: - left, exists, err := resolveFieldValue(node.Field, record) - if err != nil { - return false, err - } - if !exists { - return false, nil - } - comparison := compareValues(left, node.Value) - return comparison > 0, nil - case vectordata.LtFilter: - left, exists, err := resolveFieldValue(node.Field, record) - if err != nil { - return false, err - } - if !exists { - return false, nil - } - comparison := compareValues(left, node.Value) - return comparison < 0, nil - case vectordata.ExistsFilter: - _, exists, err := resolveFieldValue(node.Field, record) - if err != nil { - return false, err - } - return exists, nil - case vectordata.AndFilter: - if len(node.Children) == 0 { - return false, fmt.Errorf("%w: AND requires at least one child", vectordata.ErrInvalidFilter) - } - for _, child := range node.Children { - if child == nil { - return false, fmt.Errorf("%w: AND contains nil child", vectordata.ErrInvalidFilter) - } - ok, err := matchesFilter(child, record) - if err != nil { - return false, err - } - if !ok { - return false, nil - } - } - return true, nil - case vectordata.OrFilter: - if len(node.Children) == 0 { - return false, fmt.Errorf("%w: OR requires at least one child", vectordata.ErrInvalidFilter) - } - for _, child := range node.Children { - if child == nil { - return false, fmt.Errorf("%w: OR contains nil child", vectordata.ErrInvalidFilter) - } - ok, err := matchesFilter(child, record) - if err != nil { - return false, err - } - if ok { - return true, nil - } - } - return false, nil - case vectordata.NotFilter: - if node.Child == nil { - return false, fmt.Errorf("%w: NOT requires a child", vectordata.ErrInvalidFilter) - } - ok, err := matchesFilter(node.Child, record) - if err != nil { - return false, err - } - return !ok, nil - default: - return false, fmt.Errorf("%w: unsupported node type %T", vectordata.ErrInvalidFilter, filter) - } -} - -func resolveFieldValue(field vectordata.FieldRef, record vectordata.Record) (value any, exists bool, err error) { - normalized, err := vectordata.NormalizeFieldRef(field) - if err != nil { - return nil, false, err - } - - switch normalized.Kind { - case vectordata.FieldColumn: - switch normalized.Name { - case idColumn: - return record.ID, true, nil - case contentColumn: - if record.Content == nil { - return nil, false, nil - } - return *record.Content, true, nil - default: - return nil, false, fmt.Errorf("%w: unknown column %q", vectordata.ErrInvalidFilter, normalized.Name) - } - case vectordata.FieldMetadata: - if record.Metadata == nil { - return nil, false, nil - } - - var current any = record.Metadata - for _, key := range normalized.Path { - asMap, ok := current.(map[string]any) - if !ok { - return nil, false, nil - } - - next, ok := asMap[key] - if !ok { - return nil, false, nil - } - current = next - } - - return current, true, nil - default: - return nil, false, fmt.Errorf("%w: unsupported field kind %q", vectordata.ErrInvalidFilter, normalized.Kind) - } -} - -func valuesEqual(left, right any) bool { - if left == nil || right == nil { - return left == right - } - - leftNumeric, leftIsNumeric := toFloat64(left) - rightNumeric, rightIsNumeric := toFloat64(right) - if leftIsNumeric && rightIsNumeric { - return leftNumeric == rightNumeric - } - - return reflect.DeepEqual(left, right) -} - -func compareValues(left, right any) int { - leftNumeric, leftIsNumeric := toFloat64(left) - rightNumeric, rightIsNumeric := toFloat64(right) - if leftIsNumeric && rightIsNumeric { - switch { - case leftNumeric < rightNumeric: - return -1 - case leftNumeric > rightNumeric: - return 1 - default: - return 0 - } - } - - leftText := fmt.Sprint(left) - rightText := fmt.Sprint(right) - switch { - case leftText < rightText: - return -1 - case leftText > rightText: - return 1 - default: - return 0 - } -} diff --git a/stores/mssql/filter_eval_test.go b/stores/mssql/filter_eval_test.go deleted file mode 100644 index f24e709..0000000 --- a/stores/mssql/filter_eval_test.go +++ /dev/null @@ -1,140 +0,0 @@ -package mssql - -import ( - "testing" - - "github.com/gabisonia/go-vectorstore/vectordata" -) - -func TestMatchesFilterMetadataAndComparison(t *testing.T) { - content := "payload" - record := vectordata.Record{ - ID: "doc-1", - Vector: []float32{1, 0}, - Content: &content, - Metadata: map[string]any{ - "category": "news", - "rank": 2, - "nested": map[string]any{ - "k": "v", - }, - }, - } - - filter := vectordata.And( - vectordata.Eq(vectordata.Metadata("category"), "news"), - vectordata.Gt(vectordata.Metadata("rank"), 1), - vectordata.Eq(vectordata.Metadata("nested", "k"), "v"), - ) - - matched, err := matchesFilter(filter, record) - if err != nil { - t.Fatalf("matchesFilter: %v", err) - } - if !matched { - t.Fatalf("expected record to match filter") - } -} - -func TestMatchesFilterColumnExists(t *testing.T) { - content := "x" - record := vectordata.Record{ID: "doc-1", Content: &content} - - matched, err := matchesFilter(vectordata.Exists(vectordata.Column(contentColumn)), record) - if err != nil { - t.Fatalf("matchesFilter: %v", err) - } - if !matched { - t.Fatalf("expected content exists filter to match") - } -} - -func TestMatchesFilterUnknownColumnReturnsError(t *testing.T) { - record := vectordata.Record{ID: "doc-1"} - _, err := matchesFilter(vectordata.Eq(vectordata.Column("unknown"), "x"), record) - if err == nil { - t.Fatalf("expected error for unknown column") - } -} - -func TestMatchesFilterTrimsFieldReferences(t *testing.T) { - content := "x" - record := vectordata.Record{ - ID: "doc-1", - Content: &content, - Metadata: map[string]any{ - "nested": map[string]any{"key": "value"}, - }, - } - - columnMatch, err := matchesFilter(vectordata.Eq(vectordata.Column(" id "), "doc-1"), record) - if err != nil { - t.Fatalf("column match: %v", err) - } - if !columnMatch { - t.Fatalf("expected trimmed column reference to match") - } - - metadataMatch, err := matchesFilter(vectordata.Eq(vectordata.Metadata(" nested ", " key "), "value"), record) - if err != nil { - t.Fatalf("metadata match: %v", err) - } - if !metadataMatch { - t.Fatalf("expected trimmed metadata path to match") - } -} - -func TestDistanceBetweenMetrics(t *testing.T) { - left := []float32{1, 0} - right := []float32{0.8, 0.2} - - cosineDistance, err := distanceBetween(vectordata.DistanceCosine, left, right) - if err != nil { - t.Fatalf("cosine distance: %v", err) - } - if cosineDistance < 0 { - t.Fatalf("expected non-negative cosine distance, got %f", cosineDistance) - } - - l2Distance, err := distanceBetween(vectordata.DistanceL2, left, right) - if err != nil { - t.Fatalf("l2 distance: %v", err) - } - if l2Distance <= 0 { - t.Fatalf("expected positive l2 distance, got %f", l2Distance) - } - - innerProductDistance, err := distanceBetween(vectordata.DistanceInnerProduct, left, right) - if err != nil { - t.Fatalf("inner product distance: %v", err) - } - if innerProductDistance >= 0 { - t.Fatalf("expected negative inner-product distance, got %f", innerProductDistance) - } -} - -func TestProjectRecordHonorsProjection(t *testing.T) { - content := "hello" - record := vectordata.Record{ - ID: "doc-1", - Vector: []float32{1, 2}, - Content: &content, - Metadata: map[string]any{"category": "news"}, - } - - projected := projectRecord(record, vectordata.Projection{ - IncludeVector: false, - IncludeMetadata: true, - IncludeContent: false, - }) - - if len(projected.Vector) != 0 { - t.Fatalf("expected vector to be omitted") - } - if projected.Metadata["category"] != "news" { - t.Fatalf("expected metadata to be present") - } - if projected.Content != nil { - t.Fatalf("expected content to be omitted") - } -} diff --git a/stores/mssql/filter_pushdown.go b/stores/mssql/filter_pushdown.go deleted file mode 100644 index d4daebf..0000000 --- a/stores/mssql/filter_pushdown.go +++ /dev/null @@ -1,255 +0,0 @@ -package mssql - -import ( - "errors" - "fmt" - "strings" - - "github.com/gabisonia/go-vectorstore/vectordata" -) - -var errFilterPushdownUnsupported = errors.New("mssql filter pushdown unsupported") - -func compileMSSQLFilterSQL(filter vectordata.Filter, startArg int) (sql string, args []any, nextArg int, err error) { - if startArg < 1 { - startArg = 1 - } - if filter == nil { - return "", nil, startArg, nil - } - - c := &mssqlFilterCompiler{ - nextArg: startArg, - } - out, err := c.compile(filter) - if err != nil { - return "", nil, startArg, err - } - return out, c.args, c.nextArg, nil -} - -type mssqlFilterCompiler struct { - args []any - nextArg int -} - -func (c *mssqlFilterCompiler) compile(filter vectordata.Filter) (string, error) { - switch node := filter.(type) { - case vectordata.EqFilter: - return c.compileEq(node) - case vectordata.InFilter: - return c.compileIn(node) - case vectordata.GtFilter: - return c.compileGt(node) - case vectordata.LtFilter: - return c.compileLt(node) - case vectordata.ExistsFilter: - return c.compileExists(node) - case vectordata.AndFilter: - return c.compileLogical("AND", node.Children) - case vectordata.OrFilter: - return c.compileLogical("OR", node.Children) - case vectordata.NotFilter: - if node.Child == nil { - return "", fmt.Errorf("%w: NOT requires a child", vectordata.ErrInvalidFilter) - } - childSQL, err := c.compile(node.Child) - if err != nil { - return "", err - } - return fmt.Sprintf("(NOT %s)", childSQL), nil - default: - return "", fmt.Errorf("%w: unsupported node type %T", vectordata.ErrInvalidFilter, filter) - } -} - -func (c *mssqlFilterCompiler) compileEq(node vectordata.EqFilter) (string, error) { - columnExpr, metadataPath, isMetadata, err := c.resolveField(node.Field) - if err != nil { - return "", err - } - if !isMetadata { - value, ok := node.Value.(string) - if !ok { - return "", unsupportedPushdown("column equality only supports string values") - } - return fmt.Sprintf("(%s = %s)", columnExpr, c.bind(value)), nil - } - return c.compileMetadataEq(metadataPath, node.Value) -} - -func (c *mssqlFilterCompiler) compileIn(node vectordata.InFilter) (string, error) { - if len(node.Values) == 0 { - return "", fmt.Errorf("%w: IN requires at least one value", vectordata.ErrInvalidFilter) - } - - columnExpr, metadataPath, isMetadata, err := c.resolveField(node.Field) - if err != nil { - return "", err - } - - if !isMetadata { - parts := make([]string, 0, len(node.Values)) - for _, value := range node.Values { - text, ok := value.(string) - if !ok { - return "", unsupportedPushdown("column IN only supports string values") - } - parts = append(parts, c.bind(text)) - } - return fmt.Sprintf("(%s IN (%s))", columnExpr, strings.Join(parts, ", ")), nil - } - - predicates := make([]string, 0, len(node.Values)) - for _, value := range node.Values { - predicate, err := c.compileMetadataEq(metadataPath, value) - if err != nil { - return "", err - } - predicates = append(predicates, predicate) - } - return fmt.Sprintf("(%s)", strings.Join(predicates, " OR ")), nil -} - -func (c *mssqlFilterCompiler) compileGt(node vectordata.GtFilter) (string, error) { - columnExpr, metadataPath, isMetadata, err := c.resolveField(node.Field) - if err != nil { - return "", err - } - if !isMetadata { - value, ok := node.Value.(string) - if !ok { - return "", unsupportedPushdown("column greater-than only supports string values") - } - return fmt.Sprintf("(%s > %s)", columnExpr, c.bind(value)), nil - } - - if number, ok := toFloat64(node.Value); ok { - pathPlaceholder := c.bind(metadataPath) - valueExpr := fmt.Sprintf("JSON_VALUE(%s, %s)", quoteIdent(metadataColumn), pathPlaceholder) - return fmt.Sprintf("(TRY_CONVERT(float, %s) > %s)", valueExpr, c.bind(number)), nil - } - return "", unsupportedPushdown("metadata greater-than only supports numeric values") -} - -func (c *mssqlFilterCompiler) compileLt(node vectordata.LtFilter) (string, error) { - columnExpr, metadataPath, isMetadata, err := c.resolveField(node.Field) - if err != nil { - return "", err - } - if !isMetadata { - value, ok := node.Value.(string) - if !ok { - return "", unsupportedPushdown("column less-than only supports string values") - } - return fmt.Sprintf("(%s < %s)", columnExpr, c.bind(value)), nil - } - - if number, ok := toFloat64(node.Value); ok { - pathPlaceholder := c.bind(metadataPath) - valueExpr := fmt.Sprintf("JSON_VALUE(%s, %s)", quoteIdent(metadataColumn), pathPlaceholder) - return fmt.Sprintf("(TRY_CONVERT(float, %s) < %s)", valueExpr, c.bind(number)), nil - } - return "", unsupportedPushdown("metadata less-than only supports numeric values") -} - -func (c *mssqlFilterCompiler) compileExists(node vectordata.ExistsFilter) (string, error) { - columnExpr, metadataPath, isMetadata, err := c.resolveField(node.Field) - if err != nil { - return "", err - } - if !isMetadata { - return fmt.Sprintf("(%s IS NOT NULL)", columnExpr), nil - } - pathPlaceholder := c.bind(metadataPath) - return fmt.Sprintf("(JSON_PATH_EXISTS(%s, %s) = 1)", quoteIdent(metadataColumn), pathPlaceholder), nil -} - -func (c *mssqlFilterCompiler) compileLogical(op string, children []vectordata.Filter) (string, error) { - if len(children) == 0 { - return "", fmt.Errorf("%w: %s requires at least one child", vectordata.ErrInvalidFilter, op) - } - - parts := make([]string, 0, len(children)) - for _, child := range children { - if child == nil { - return "", fmt.Errorf("%w: %s contains nil child", vectordata.ErrInvalidFilter, op) - } - childSQL, err := c.compile(child) - if err != nil { - return "", err - } - parts = append(parts, childSQL) - } - return fmt.Sprintf("(%s)", strings.Join(parts, fmt.Sprintf(" %s ", op))), nil -} - -func (c *mssqlFilterCompiler) resolveField(ref vectordata.FieldRef) (columnExpr string, metadataPath string, isMetadata bool, err error) { - normalized, err := vectordata.NormalizeFieldRef(ref) - if err != nil { - return "", "", false, err - } - - switch normalized.Kind { - case vectordata.FieldColumn: - switch normalized.Name { - case idColumn: - return quoteIdent(idColumn), "", false, nil - case contentColumn: - return quoteIdent(contentColumn), "", false, nil - default: - return "", "", false, fmt.Errorf("%w: unknown column %q", vectordata.ErrInvalidFilter, normalized.Name) - } - case vectordata.FieldMetadata: - return "", metadataPathLiteral(normalized.Path), true, nil - default: - return "", "", false, fmt.Errorf("%w: unsupported field kind %q", vectordata.ErrInvalidFilter, normalized.Kind) - } -} - -func (c *mssqlFilterCompiler) compileMetadataEq(metadataPath string, value any) (string, error) { - pathPlaceholder := c.bind(metadataPath) - valueExpr := fmt.Sprintf("JSON_VALUE(%s, %s)", quoteIdent(metadataColumn), pathPlaceholder) - - switch typed := value.(type) { - case nil: - return "", unsupportedPushdown("metadata equality with nil is not supported by SQL pushdown") - case string: - return fmt.Sprintf("(%s = %s)", valueExpr, c.bind(typed)), nil - case bool: - boolValue := "false" - if typed { - boolValue = "true" - } - return fmt.Sprintf("(LOWER(%s) = %s)", valueExpr, c.bind(boolValue)), nil - default: - if number, ok := toFloat64(value); ok { - return fmt.Sprintf("(TRY_CONVERT(float, %s) = %s)", valueExpr, c.bind(number)), nil - } - return "", unsupportedPushdown("metadata scalar comparison does not support value type %T", value) - } -} - -func (c *mssqlFilterCompiler) bind(value any) string { - placeholder := fmt.Sprintf("@p%d", c.nextArg) - c.nextArg++ - c.args = append(c.args, value) - return placeholder -} - -func metadataPathLiteral(path []string) string { - var b strings.Builder - b.WriteString("$") - for _, segment := range path { - escaped := strings.ReplaceAll(segment, `\`, `\\`) - escaped = strings.ReplaceAll(escaped, `"`, `\"`) - b.WriteString(`."`) - b.WriteString(escaped) - b.WriteString(`"`) - } - return b.String() -} - -func unsupportedPushdown(format string, args ...any) error { - return fmt.Errorf("%w: %s", errFilterPushdownUnsupported, fmt.Sprintf(format, args...)) -} diff --git a/stores/mssql/filter_pushdown_test.go b/stores/mssql/filter_pushdown_test.go deleted file mode 100644 index 140e62a..0000000 --- a/stores/mssql/filter_pushdown_test.go +++ /dev/null @@ -1,73 +0,0 @@ -package mssql - -import ( - "errors" - "reflect" - "testing" - - "github.com/gabisonia/go-vectorstore/vectordata" -) - -func TestCompileMSSQLFilterSQL_TrimsColumnReference(t *testing.T) { - sql, args, next, err := compileMSSQLFilterSQL(vectordata.Eq(vectordata.Column(" id "), "doc-1"), 1) - if err != nil { - t.Fatalf("compileMSSQLFilterSQL: %v", err) - } - if sql != `([id] = @p1)` { - t.Fatalf("unexpected SQL: %s", sql) - } - if !reflect.DeepEqual(args, []any{"doc-1"}) { - t.Fatalf("unexpected args: %#v", args) - } - if next != 2 { - t.Fatalf("unexpected next arg index: %d", next) - } -} - -func TestCompileMSSQLFilterSQL_MetadataExists(t *testing.T) { - sql, args, _, err := compileMSSQLFilterSQL(vectordata.Exists(vectordata.Metadata(" nested ", "value")), 3) - if err != nil { - t.Fatalf("compileMSSQLFilterSQL: %v", err) - } - expectedSQL := `(JSON_PATH_EXISTS([metadata], @p3) = 1)` - if sql != expectedSQL { - t.Fatalf("unexpected SQL:\nwant: %s\n got: %s", expectedSQL, sql) - } - if !reflect.DeepEqual(args, []any{`$."nested"."value"`}) { - t.Fatalf("unexpected args: %#v", args) - } -} - -func TestCompileMSSQLFilterSQL_MetadataEqAndGt(t *testing.T) { - filter := vectordata.And( - vectordata.Eq(vectordata.Metadata("category"), "news"), - vectordata.Gt(vectordata.Metadata("rank"), 1), - ) - - sql, args, next, err := compileMSSQLFilterSQL(filter, 2) - if err != nil { - t.Fatalf("compileMSSQLFilterSQL: %v", err) - } - - expectedSQL := `((JSON_VALUE([metadata], @p2) = @p3) AND (TRY_CONVERT(float, JSON_VALUE([metadata], @p4)) > @p5))` - if sql != expectedSQL { - t.Fatalf("unexpected SQL:\nwant: %s\n got: %s", expectedSQL, sql) - } - expectedArgs := []any{`$."category"`, "news", `$."rank"`, float64(1)} - if !reflect.DeepEqual(args, expectedArgs) { - t.Fatalf("unexpected args:\nwant: %#v\n got: %#v", expectedArgs, args) - } - if next != 6 { - t.Fatalf("unexpected next arg index: %d", next) - } -} - -func TestCompileMSSQLFilterSQL_UnsupportedColumnValueType(t *testing.T) { - _, _, _, err := compileMSSQLFilterSQL(vectordata.Eq(vectordata.Column("id"), 123), 1) - if err == nil { - t.Fatal("expected error") - } - if !errors.Is(err, errFilterPushdownUnsupported) { - t.Fatalf("expected errFilterPushdownUnsupported, got %v", err) - } -} diff --git a/stores/mssql/helpers.go b/stores/mssql/helpers.go deleted file mode 100644 index deb5bdb..0000000 --- a/stores/mssql/helpers.go +++ /dev/null @@ -1,216 +0,0 @@ -package mssql - -import ( - "encoding/json" - "fmt" - "math" - "strconv" - "strings" - - "github.com/gabisonia/go-vectorstore/vectordata" -) - -const ( - idColumn = "id" - vectorColumn = "vector" - metadataColumn = "metadata" - contentColumn = "content" - collectionMetaTable = "__vector_collections" - maxRowsPerStatement = 500 -) - -func quoteIdent(ident string) string { - return "[" + strings.ReplaceAll(ident, "]", "]]") + "]" -} - -func qualifiedTable(schema, table string) string { - return quoteIdent(schema) + "." + quoteIdent(table) -} - -func objectIDName(schema, table string) string { - return quoteIdent(schema) + "." + quoteIdent(table) -} - -func escapeSQLString(value string) string { - return strings.ReplaceAll(value, "'", "''") -} - -func defaultMetric(metric vectordata.DistanceMetric) vectordata.DistanceMetric { - if metric == "" { - return vectordata.DistanceCosine - } - return metric -} - -func defaultMode(mode vectordata.EnsureMode, strictByDefault bool) vectordata.EnsureMode { - if mode != "" { - return mode - } - if strictByDefault { - return vectordata.EnsureStrict - } - return vectordata.EnsureAutoMigrate -} - -func vectorJSON(vector []float32) (string, error) { - payload, err := json.Marshal(vector) - if err != nil { - return "", err - } - return string(payload), nil -} - -func parseVectorJSON(raw string) ([]float32, error) { - raw = strings.TrimSpace(raw) - if raw == "" { - return nil, nil - } - - var values []float64 - if err := json.Unmarshal([]byte(raw), &values); err != nil { - return nil, err - } - - out := make([]float32, 0, len(values)) - for _, value := range values { - out = append(out, float32(value)) - } - return out, nil -} - -func normalizeMetadata(metadata map[string]any) map[string]any { - if metadata == nil { - return map[string]any{} - } - return metadata -} - -func metadataJSON(metadata map[string]any) (string, error) { - payload, err := json.Marshal(normalizeMetadata(metadata)) - if err != nil { - return "", err - } - return string(payload), nil -} - -func parseMetadataJSON(raw string) (map[string]any, error) { - raw = strings.TrimSpace(raw) - if raw == "" { - return map[string]any{}, nil - } - - var metadata map[string]any - if err := json.Unmarshal([]byte(raw), &metadata); err != nil { - return nil, err - } - if metadata == nil { - return map[string]any{}, nil - } - return metadata, nil -} - -func resolveProjection(projection *vectordata.Projection) vectordata.Projection { - if projection == nil { - return vectordata.DefaultProjection() - } - return *projection -} - -func distanceBetween(metric vectordata.DistanceMetric, query, candidate []float32) (float64, error) { - if len(query) != len(candidate) { - return 0, fmt.Errorf("%w: expected %d, got %d", vectordata.ErrDimensionMismatch, len(query), len(candidate)) - } - - switch metric { - case vectordata.DistanceCosine: - return cosineDistance(query, candidate), nil - case vectordata.DistanceL2: - return l2Distance(query, candidate), nil - case vectordata.DistanceInnerProduct: - return -dot(query, candidate), nil - default: - return 0, fmt.Errorf("%w: unsupported distance metric %q", vectordata.ErrSchemaMismatch, metric) - } -} - -func cosineDistance(left, right []float32) float64 { - leftNorm := norm(left) - rightNorm := norm(right) - if leftNorm == 0 || rightNorm == 0 { - return 1 - } - similarity := dot(left, right) / (leftNorm * rightNorm) - return 1 - similarity -} - -func l2Distance(left, right []float32) float64 { - sum := 0.0 - for i := range left { - delta := float64(left[i] - right[i]) - sum += delta * delta - } - return math.Sqrt(sum) -} - -func dot(left, right []float32) float64 { - sum := 0.0 - for i := range left { - sum += float64(left[i] * right[i]) - } - return sum -} - -func norm(vector []float32) float64 { - sum := 0.0 - for _, value := range vector { - f := float64(value) - sum += f * f - } - return math.Sqrt(sum) -} - -func isStringType(dataType string) bool { - switch strings.ToLower(strings.TrimSpace(dataType)) { - case "varchar", "nvarchar", "char", "nchar", "text", "ntext": - return true - default: - return false - } -} - -func toFloat64(value any) (float64, bool) { - switch n := value.(type) { - case int: - return float64(n), true - case int8: - return float64(n), true - case int16: - return float64(n), true - case int32: - return float64(n), true - case int64: - return float64(n), true - case uint: - return float64(n), true - case uint8: - return float64(n), true - case uint16: - return float64(n), true - case uint32: - return float64(n), true - case uint64: - return float64(n), true - case float32: - return float64(n), true - case float64: - return n, true - case json.Number: - parsed, err := strconv.ParseFloat(string(n), 64) - if err != nil { - return 0, false - } - return parsed, true - default: - return 0, false - } -} diff --git a/stores/mssql/mssql_integration_test.go b/stores/mssql/mssql_integration_test.go deleted file mode 100644 index bc4ae7c..0000000 --- a/stores/mssql/mssql_integration_test.go +++ /dev/null @@ -1,462 +0,0 @@ -//go:build integration - -package mssql - -import ( - "context" - "database/sql" - "fmt" - "net" - "net/url" - "os" - "strings" - "sync/atomic" - "testing" - "time" - - "github.com/gabisonia/go-vectorstore/vectordata" - _ "github.com/microsoft/go-mssqldb" - testcontainers "github.com/testcontainers/testcontainers-go" - "github.com/testcontainers/testcontainers-go/wait" -) - -const ( - integrationMSSQLUser = "sa" - integrationMSSQLPassword = "YourStrong!Passw0rd" - integrationMSSQLDatabase = "vectorstore_test" -) - -var ( - schemaSeq atomic.Uint64 - integrationDSN string - integrationContainer testcontainers.Container -) - -func TestMain(m *testing.M) { - ctx, cancel := context.WithTimeout(context.Background(), 12*time.Minute) - defer cancel() - - dsn := strings.TrimSpace(os.Getenv("MSSQL_TEST_DSN")) - if dsn == "" { - container, generatedDSN, err := startMSSQLContainer(ctx) - if err != nil { - fmt.Fprintf(os.Stderr, "failed to start integration container: %v\n", err) - os.Exit(1) - } - integrationContainer = container - integrationDSN = generatedDSN - } else { - integrationDSN = dsn - } - - exitCode := m.Run() - - if integrationContainer != nil { - shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 30*time.Second) - defer shutdownCancel() - if err := integrationContainer.Terminate(shutdownCtx); err != nil { - fmt.Fprintf(os.Stderr, "failed to terminate integration container: %v\n", err) - if exitCode == 0 { - exitCode = 1 - } - } - } - - os.Exit(exitCode) -} - -func startMSSQLContainer(ctx context.Context) (testcontainers.Container, string, error) { - request := testcontainers.ContainerRequest{ - Image: "mcr.microsoft.com/mssql/server:2022-latest", - ImagePlatform: "linux/amd64", - AlwaysPullImage: true, - ExposedPorts: []string{"1433/tcp"}, - Env: map[string]string{ - "ACCEPT_EULA": "Y", - "MSSQL_SA_PASSWORD": integrationMSSQLPassword, - "MSSQL_PID": "Developer", - }, - WaitingFor: wait.ForLog("SQL Server is now ready for client connections"). - WithStartupTimeout(10 * time.Minute), - } - - container, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ - ContainerRequest: request, - Started: true, - }) - if err != nil { - return nil, "", fmt.Errorf("start sql server container: %w", err) - } - - host, err := container.Host(ctx) - if err != nil { - _ = container.Terminate(context.Background()) - return nil, "", fmt.Errorf("resolve container host: %w", err) - } - - mappedPort, err := container.MappedPort(ctx, "1433/tcp") - if err != nil { - _ = container.Terminate(context.Background()) - return nil, "", fmt.Errorf("resolve container port: %w", err) - } - - masterDSN := buildMSSQLDSN(host, mappedPort.Port(), "master") - if err := waitForDatabase(ctx, masterDSN); err != nil { - _ = container.Terminate(context.Background()) - return nil, "", err - } - if err := ensureDatabase(ctx, masterDSN, integrationMSSQLDatabase); err != nil { - _ = container.Terminate(context.Background()) - return nil, "", err - } - - dsn := buildMSSQLDSN(host, mappedPort.Port(), integrationMSSQLDatabase) - if err := waitForDatabase(ctx, dsn); err != nil { - _ = container.Terminate(context.Background()) - return nil, "", err - } - - return container, dsn, nil -} - -func buildMSSQLDSN(host string, port string, database string) string { - u := &url.URL{ - Scheme: "sqlserver", - User: url.UserPassword(integrationMSSQLUser, integrationMSSQLPassword), - Host: net.JoinHostPort(host, port), - } - - query := u.Query() - query.Set("database", database) - query.Set("encrypt", "disable") - u.RawQuery = query.Encode() - - return u.String() -} - -func waitForDatabase(parent context.Context, dsn string) error { - ctx, cancel := context.WithTimeout(parent, 2*time.Minute) - defer cancel() - - for { - db, err := sql.Open("sqlserver", dsn) - if err == nil { - pingCtx, pingCancel := context.WithTimeout(ctx, 4*time.Second) - pingErr := db.PingContext(pingCtx) - pingCancel() - _ = db.Close() - if pingErr == nil { - return nil - } - } - - select { - case <-ctx.Done(): - if err != nil { - return fmt.Errorf("connect integration database: %w", err) - } - return fmt.Errorf("wait for integration database: %w", ctx.Err()) - case <-time.After(500 * time.Millisecond): - } - } -} - -func ensureDatabase(ctx context.Context, dsn string, database string) error { - db, err := sql.Open("sqlserver", dsn) - if err != nil { - return fmt.Errorf("connect master database: %w", err) - } - defer db.Close() - - query := fmt.Sprintf("IF DB_ID(N'%s') IS NULL CREATE DATABASE %s", escapeSQLString(database), quoteIdent(database)) - if _, err := db.ExecContext(ctx, query); err != nil { - return fmt.Errorf("ensure integration database: %w", err) - } - - return nil -} - -func integrationDB(t *testing.T) *sql.DB { - t.Helper() - - dsn := strings.TrimSpace(integrationDSN) - if dsn == "" { - t.Fatal("integration DSN is not initialized") - } - - db, err := sql.Open("sqlserver", dsn) - if err != nil { - t.Fatalf("open db: %v", err) - } - - ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) - defer cancel() - if err := db.PingContext(ctx); err != nil { - _ = db.Close() - t.Fatalf("ping db: %v", err) - } - - t.Cleanup(func() { - _ = db.Close() - }) - - return db -} - -func newTestStore(t *testing.T, db *sql.DB) *MSSQLVectorStore { - t.Helper() - - seq := schemaSeq.Add(1) - schema := fmt.Sprintf("it_%d_%d", time.Now().UnixNano(), seq) - schema = strings.ReplaceAll(schema, "-", "_") - - store, err := NewVectorStore(db, StoreOptions{ - Schema: schema, - StrictByDefault: true, - }) - if err != nil { - t.Fatalf("NewVectorStore: %v", err) - } - - t.Cleanup(func() { - ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) - defer cancel() - - cleanupQuery := fmt.Sprintf(` - DECLARE @schema SYSNAME = N'%s'; - DECLARE @dropSql NVARCHAR(MAX) = N''; - SELECT @dropSql = @dropSql + N'DROP TABLE ' + QUOTENAME(SCHEMA_NAME(schema_id)) + N'.' + QUOTENAME(name) + N';' - FROM sys.tables - WHERE schema_id = SCHEMA_ID(@schema); - - IF LEN(@dropSql) > 0 - BEGIN - EXEC sp_executesql @dropSql; - END - - IF SCHEMA_ID(@schema) IS NOT NULL - BEGIN - EXEC(N'DROP SCHEMA ' + QUOTENAME(@schema)); - END - `, escapeSQLString(schema)) - _, _ = db.ExecContext(ctx, cleanupQuery) - }) - - return store -} - -func TestIntegrationEnsureCollection(t *testing.T) { - db := integrationDB(t) - store := newTestStore(t, db) - - ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) - defer cancel() - - _, err := store.EnsureCollection(ctx, vectordata.CollectionSpec{ - Name: "docs", - Dimension: 3, - Metric: vectordata.DistanceCosine, - Mode: vectordata.EnsureStrict, - }) - if err != nil { - t.Fatalf("EnsureCollection: %v", err) - } - - var count int - err = db.QueryRowContext(ctx, ` - SELECT COUNT(1) - FROM INFORMATION_SCHEMA.TABLES - WHERE TABLE_SCHEMA = @p1 AND TABLE_NAME = @p2 - `, store.opts.Schema, "docs").Scan(&count) - if err != nil { - t.Fatalf("query table exists: %v", err) - } - if count == 0 { - t.Fatalf("expected collection table to exist") - } - - dimension, metric, found, err := store.readCollectionMetadata(ctx, "docs") - if err != nil { - t.Fatalf("readCollectionMetadata: %v", err) - } - if !found { - t.Fatalf("expected collection metadata row to exist") - } - if dimension != 3 { - t.Fatalf("expected dimension 3, got %d", dimension) - } - if metric != vectordata.DistanceCosine { - t.Fatalf("expected metric %q, got %q", vectordata.DistanceCosine, metric) - } -} - -func TestIntegrationUpsertAndGet(t *testing.T) { - db := integrationDB(t) - store := newTestStore(t, db) - - ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) - defer cancel() - - collection, err := store.EnsureCollection(ctx, vectordata.CollectionSpec{ - Name: "docs", - Dimension: 2, - Metric: vectordata.DistanceCosine, - Mode: vectordata.EnsureStrict, - }) - if err != nil { - t.Fatalf("EnsureCollection: %v", err) - } - - content := "first" - if err := collection.Upsert(ctx, []vectordata.Record{{ - ID: "r1", - Vector: []float32{1, 0}, - Content: &content, - Metadata: map[string]any{ - "category": "news", - "rank": 1, - }, - }}); err != nil { - t.Fatalf("Upsert: %v", err) - } - - record, err := collection.Get(ctx, "r1") - if err != nil { - t.Fatalf("Get: %v", err) - } - if record.ID != "r1" { - t.Fatalf("expected ID r1, got %q", record.ID) - } - if len(record.Vector) != 2 || record.Vector[0] != 1 || record.Vector[1] != 0 { - t.Fatalf("unexpected vector: %#v", record.Vector) - } - if record.Content == nil || *record.Content != "first" { - t.Fatalf("unexpected content: %#v", record.Content) - } - if record.Metadata["category"] != "news" { - t.Fatalf("unexpected metadata category: %#v", record.Metadata["category"]) - } - - updated := "updated" - if err := collection.Upsert(ctx, []vectordata.Record{{ - ID: "r1", - Vector: []float32{0.5, 0.5}, - Content: &updated, - Metadata: map[string]any{ - "category": "blog", - "rank": 3, - }, - }}); err != nil { - t.Fatalf("Upsert second call: %v", err) - } - - updatedRecord, err := collection.Get(ctx, "r1") - if err != nil { - t.Fatalf("Get after upsert: %v", err) - } - if updatedRecord.Content == nil || *updatedRecord.Content != "updated" { - t.Fatalf("content not updated: %#v", updatedRecord.Content) - } - if updatedRecord.Metadata["category"] != "blog" { - t.Fatalf("metadata not updated: %#v", updatedRecord.Metadata) - } -} - -func TestIntegrationSearchByMetric(t *testing.T) { - metrics := []vectordata.DistanceMetric{ - vectordata.DistanceCosine, - vectordata.DistanceL2, - vectordata.DistanceInnerProduct, - } - - for _, metric := range metrics { - t.Run(string(metric), func(t *testing.T) { - db := integrationDB(t) - store := newTestStore(t, db) - - ctx, cancel := context.WithTimeout(context.Background(), 25*time.Second) - defer cancel() - - collection, err := store.EnsureCollection(ctx, vectordata.CollectionSpec{ - Name: "search_docs", - Dimension: 2, - Metric: metric, - Mode: vectordata.EnsureStrict, - }) - if err != nil { - t.Fatalf("EnsureCollection: %v", err) - } - - err = collection.Upsert(ctx, []vectordata.Record{ - {ID: "a", Vector: []float32{1, 0}, Metadata: map[string]any{"kind": "a"}}, - {ID: "b", Vector: []float32{0.8, 0.2}, Metadata: map[string]any{"kind": "b"}}, - {ID: "c", Vector: []float32{0, 1}, Metadata: map[string]any{"kind": "c"}}, - }) - if err != nil { - t.Fatalf("Upsert: %v", err) - } - - results, err := collection.SearchByVector(ctx, []float32{1, 0}, 2, vectordata.SearchOptions{}) - if err != nil { - t.Fatalf("SearchByVector: %v", err) - } - if len(results) != 2 { - t.Fatalf("expected 2 results, got %d", len(results)) - } - if results[0].Record.ID != "a" || results[1].Record.ID != "b" { - t.Fatalf("unexpected ordering: [%s, %s]", results[0].Record.ID, results[1].Record.ID) - } - }) - } -} - -func TestIntegrationMetadataFilter(t *testing.T) { - db := integrationDB(t) - store := newTestStore(t, db) - - ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) - defer cancel() - - collection, err := store.EnsureCollection(ctx, vectordata.CollectionSpec{ - Name: "docs", - Dimension: 2, - Metric: vectordata.DistanceCosine, - Mode: vectordata.EnsureStrict, - }) - if err != nil { - t.Fatalf("EnsureCollection: %v", err) - } - - err = collection.Upsert(ctx, []vectordata.Record{ - {ID: "a", Vector: []float32{1, 0}, Metadata: map[string]any{"category": "news", "rank": 1}}, - {ID: "b", Vector: []float32{0.9, 0.1}, Metadata: map[string]any{"category": "news", "rank": 2}}, - {ID: "c", Vector: []float32{0, 1}, Metadata: map[string]any{"category": "other", "rank": 3}}, - }) - if err != nil { - t.Fatalf("Upsert: %v", err) - } - - filter := vectordata.And( - vectordata.Eq(vectordata.Metadata("category"), "news"), - vectordata.Gt(vectordata.Metadata("rank"), 1), - ) - - results, searchErr := collection.SearchByVector(ctx, []float32{1, 0}, 10, vectordata.SearchOptions{Filter: filter}) - count, countErr := collection.Count(ctx, filter) - - if searchErr != nil { - t.Fatalf("SearchByVector with filter: %v", searchErr) - } - if len(results) != 1 { - t.Fatalf("expected 1 result, got %d", len(results)) - } - if results[0].Record.ID != "b" { - t.Fatalf("expected result b, got %s", results[0].Record.ID) - } - if countErr != nil { - t.Fatalf("Count with filter: %v", countErr) - } - if count != 1 { - t.Fatalf("expected count 1, got %d", count) - } -} diff --git a/stores/mssql/schema.go b/stores/mssql/schema.go deleted file mode 100644 index 207f63e..0000000 --- a/stores/mssql/schema.go +++ /dev/null @@ -1,298 +0,0 @@ -package mssql - -import ( - "context" - "database/sql" - "fmt" - - "github.com/gabisonia/go-vectorstore/vectordata" -) - -func (s *MSSQLVectorStore) ensureBaseSchema(ctx context.Context) error { - schemaLiteral := escapeSQLString(s.opts.Schema) - query := fmt.Sprintf("IF SCHEMA_ID(N'%s') IS NULL EXEC(N'CREATE SCHEMA %s')", schemaLiteral, quoteIdent(s.opts.Schema)) - if _, err := s.db.ExecContext(ctx, query); err != nil { - return fmt.Errorf("ensure schema %q: %w", s.opts.Schema, err) - } - - if err := s.ensureCollectionsMetadataTable(ctx); err != nil { - return err - } - - return nil -} - -func (s *MSSQLVectorStore) ensureCollectionsMetadataTable(ctx context.Context) error { - query := fmt.Sprintf(` - IF OBJECT_ID(N'%s', N'U') IS NULL - BEGIN - CREATE TABLE %s ( - %s NVARCHAR(255) NOT NULL PRIMARY KEY, - %s INT NOT NULL, - %s NVARCHAR(64) NOT NULL - ) - END - `, - escapeSQLString(objectIDName(s.opts.Schema, collectionMetaTable)), - qualifiedTable(s.opts.Schema, collectionMetaTable), - quoteIdent("name"), - quoteIdent("dimension"), - quoteIdent("metric"), - ) - - if _, err := s.db.ExecContext(ctx, query); err != nil { - return fmt.Errorf("ensure collection metadata table: %w", err) - } - return nil -} - -func (s *MSSQLVectorStore) ensureTableWithValidation(ctx context.Context, spec vectordata.CollectionSpec, mode vectordata.EnsureMode) error { - exists, err := s.tableExists(ctx, spec.Name) - if err != nil { - return err - } - - if !exists { - if err := s.createCollectionTable(ctx, spec.Name); err != nil { - return err - } - if err := s.upsertCollectionMetadata(ctx, spec.Name, spec.Dimension, spec.Metric); err != nil { - return err - } - return nil - } - - if err := s.validateCollectionSchema(ctx, spec.Name, mode); err != nil { - return err - } - - dimension, metric, found, err := s.readCollectionMetadata(ctx, spec.Name) - if err != nil { - return err - } - if !found { - if mode == vectordata.EnsureStrict { - return fmt.Errorf("%w: missing collection metadata for %q", vectordata.ErrSchemaMismatch, spec.Name) - } - return s.upsertCollectionMetadata(ctx, spec.Name, spec.Dimension, spec.Metric) - } - - if dimension != spec.Dimension { - return fmt.Errorf("%w: expected vector dimension %d, got %d", vectordata.ErrSchemaMismatch, spec.Dimension, dimension) - } - - if defaultMetric(metric) != defaultMetric(spec.Metric) { - return fmt.Errorf("%w: expected metric %q, got %q", vectordata.ErrSchemaMismatch, spec.Metric, metric) - } - - return nil -} - -func (s *MSSQLVectorStore) tableExists(ctx context.Context, table string) (bool, error) { - var count int - err := s.db.QueryRowContext(ctx, ` - SELECT COUNT(1) - FROM INFORMATION_SCHEMA.TABLES - WHERE TABLE_SCHEMA = @p1 AND TABLE_NAME = @p2 - `, s.opts.Schema, table).Scan(&count) - if err != nil { - return false, fmt.Errorf("check table exists: %w", err) - } - - return count > 0, nil -} - -func (s *MSSQLVectorStore) createCollectionTable(ctx context.Context, table string) error { - query := fmt.Sprintf(` - IF OBJECT_ID(N'%s', N'U') IS NULL - BEGIN - CREATE TABLE %s ( - %s NVARCHAR(255) NOT NULL PRIMARY KEY, - %s NVARCHAR(MAX) NOT NULL, - %s NVARCHAR(MAX) NOT NULL DEFAULT N'{}', - %s NVARCHAR(MAX) NULL - ) - END - `, - escapeSQLString(objectIDName(s.opts.Schema, table)), - qualifiedTable(s.opts.Schema, table), - quoteIdent(idColumn), - quoteIdent(vectorColumn), - quoteIdent(metadataColumn), - quoteIdent(contentColumn), - ) - - if _, err := s.db.ExecContext(ctx, query); err != nil { - return fmt.Errorf("create collection table %q: %w", table, err) - } - return nil -} - -func (s *MSSQLVectorStore) validateCollectionSchema(ctx context.Context, table string, mode vectordata.EnsureMode) error { - rows, err := s.db.QueryContext(ctx, ` - SELECT COLUMN_NAME, DATA_TYPE - FROM INFORMATION_SCHEMA.COLUMNS - WHERE TABLE_SCHEMA = @p1 AND TABLE_NAME = @p2 - `, s.opts.Schema, table) - if err != nil { - return fmt.Errorf("read schema columns: %w", err) - } - defer rows.Close() - - columns := make(map[string]string) - for rows.Next() { - var columnName string - var dataType string - if err := rows.Scan(&columnName, &dataType); err != nil { - return fmt.Errorf("scan schema columns: %w", err) - } - columns[columnName] = dataType - } - if err := rows.Err(); err != nil { - return fmt.Errorf("iterate schema columns: %w", err) - } - - idType, ok := columns[idColumn] - if !ok { - return fmt.Errorf("%w: missing column %q", vectordata.ErrSchemaMismatch, idColumn) - } - if !isStringType(idType) { - return fmt.Errorf("%w: expected %q to be string-compatible type, got %q", vectordata.ErrSchemaMismatch, idColumn, idType) - } - - vectorType, ok := columns[vectorColumn] - if !ok { - return fmt.Errorf("%w: missing column %q", vectordata.ErrSchemaMismatch, vectorColumn) - } - if !isStringType(vectorType) { - return fmt.Errorf("%w: expected %q to be string-compatible type, got %q", vectordata.ErrSchemaMismatch, vectorColumn, vectorType) - } - - if err := s.ensurePrimaryKeyOnID(ctx, table); err != nil { - return err - } - - metadataType, hasMetadata := columns[metadataColumn] - if !hasMetadata { - if mode == vectordata.EnsureStrict { - return fmt.Errorf("%w: missing column %q", vectordata.ErrSchemaMismatch, metadataColumn) - } - if err := s.addMetadataColumn(ctx, table); err != nil { - return err - } - } else if !isStringType(metadataType) { - return fmt.Errorf("%w: expected %q to be string-compatible type, got %q", vectordata.ErrSchemaMismatch, metadataColumn, metadataType) - } - - contentType, hasContent := columns[contentColumn] - if !hasContent { - if mode == vectordata.EnsureStrict { - return fmt.Errorf("%w: missing column %q", vectordata.ErrSchemaMismatch, contentColumn) - } - if err := s.addContentColumn(ctx, table); err != nil { - return err - } - } else if !isStringType(contentType) { - return fmt.Errorf("%w: expected %q to be string-compatible type, got %q", vectordata.ErrSchemaMismatch, contentColumn, contentType) - } - - return nil -} - -func (s *MSSQLVectorStore) ensurePrimaryKeyOnID(ctx context.Context, table string) error { - var count int - err := s.db.QueryRowContext(ctx, ` - SELECT COUNT(1) - FROM INFORMATION_SCHEMA.TABLE_CONSTRAINTS tc - INNER JOIN INFORMATION_SCHEMA.KEY_COLUMN_USAGE kcu - ON tc.CONSTRAINT_NAME = kcu.CONSTRAINT_NAME - AND tc.TABLE_SCHEMA = kcu.TABLE_SCHEMA - AND tc.TABLE_NAME = kcu.TABLE_NAME - WHERE tc.TABLE_SCHEMA = @p1 - AND tc.TABLE_NAME = @p2 - AND tc.CONSTRAINT_TYPE = 'PRIMARY KEY' - AND kcu.COLUMN_NAME = @p3 - `, s.opts.Schema, table, idColumn).Scan(&count) - if err != nil { - return fmt.Errorf("check primary key: %w", err) - } - if count == 0 { - return fmt.Errorf("%w: primary key on %q is required", vectordata.ErrSchemaMismatch, idColumn) - } - return nil -} - -func (s *MSSQLVectorStore) addMetadataColumn(ctx context.Context, table string) error { - query := fmt.Sprintf("ALTER TABLE %s ADD %s NVARCHAR(MAX) NOT NULL DEFAULT N'{}'", qualifiedTable(s.opts.Schema, table), quoteIdent(metadataColumn)) - if _, err := s.db.ExecContext(ctx, query); err != nil { - return fmt.Errorf("auto-migrate metadata column: %w", err) - } - return nil -} - -func (s *MSSQLVectorStore) addContentColumn(ctx context.Context, table string) error { - query := fmt.Sprintf("ALTER TABLE %s ADD %s NVARCHAR(MAX) NULL", qualifiedTable(s.opts.Schema, table), quoteIdent(contentColumn)) - if _, err := s.db.ExecContext(ctx, query); err != nil { - return fmt.Errorf("auto-migrate content column: %w", err) - } - return nil -} - -func (s *MSSQLVectorStore) readCollectionMetadata(ctx context.Context, collectionName string) (dimension int, metric vectordata.DistanceMetric, found bool, err error) { - query := fmt.Sprintf("SELECT %s, %s FROM %s WHERE %s = @p1", - quoteIdent("dimension"), - quoteIdent("metric"), - qualifiedTable(s.opts.Schema, collectionMetaTable), - quoteIdent("name"), - ) - - var metricRaw string - err = s.db.QueryRowContext(ctx, query, collectionName).Scan(&dimension, &metricRaw) - if err != nil { - if err == sql.ErrNoRows { - return 0, "", false, nil - } - return 0, "", false, fmt.Errorf("read collection metadata: %w", err) - } - - return dimension, vectordata.DistanceMetric(metricRaw), true, nil -} - -func (s *MSSQLVectorStore) upsertCollectionMetadata(ctx context.Context, collectionName string, dimension int, metric vectordata.DistanceMetric) error { - query := fmt.Sprintf(` - MERGE %s AS target - USING ( - SELECT @p1 AS %s, @p2 AS %s, @p3 AS %s - ) AS src - ON target.%s = src.%s - WHEN MATCHED THEN - UPDATE SET - target.%s = src.%s, - target.%s = src.%s - WHEN NOT MATCHED THEN - INSERT (%s, %s, %s) - VALUES (src.%s, src.%s, src.%s); - `, - qualifiedTable(s.opts.Schema, collectionMetaTable), - quoteIdent("name"), - quoteIdent("dimension"), - quoteIdent("metric"), - quoteIdent("name"), - quoteIdent("name"), - quoteIdent("dimension"), - quoteIdent("dimension"), - quoteIdent("metric"), - quoteIdent("metric"), - quoteIdent("name"), - quoteIdent("dimension"), - quoteIdent("metric"), - quoteIdent("name"), - quoteIdent("dimension"), - quoteIdent("metric"), - ) - - if _, err := s.db.ExecContext(ctx, query, collectionName, dimension, string(metric)); err != nil { - return fmt.Errorf("upsert collection metadata: %w", err) - } - return nil -} diff --git a/stores/mssql/search_sql.go b/stores/mssql/search_sql.go deleted file mode 100644 index e023414..0000000 --- a/stores/mssql/search_sql.go +++ /dev/null @@ -1,212 +0,0 @@ -package mssql - -import ( - "context" - "database/sql" - "fmt" - "strings" - - "github.com/gabisonia/go-vectorstore/vectordata" -) - -type searchSQLPlan struct { - query string - args []any - projection vectordata.Projection -} - -func (c *MSSQLCollection) buildSearchSQLPlan(vector []float32, topK int, opts vectordata.SearchOptions) (searchSQLPlan, error) { - projection := resolveProjection(opts.Projection) - vectorPayload, err := vectorJSON(vector) - if err != nil { - return searchSQLPlan{}, fmt.Errorf("encode query vector: %w", err) - } - - distanceExpr, err := searchDistanceExpr(defaultMetric(c.metric)) - if err != nil { - return searchSQLPlan{}, err - } - - args := make([]any, 0, 8) - nextArg := 1 - - queryVectorPlaceholder := fmt.Sprintf("@p%d", nextArg) - args = append(args, vectorPayload) - nextArg++ - - expectedDimPlaceholder := fmt.Sprintf("@p%d", nextArg) - args = append(args, c.dimension) - nextArg++ - - whereParts := []string{ - fmt.Sprintf("candidate_stats.candidate_dim = %s", expectedDimPlaceholder), - fmt.Sprintf("vec_stats.matched_dim = %s", expectedDimPlaceholder), - } - - if opts.Filter != nil { - filterSQL, filterArgs, next, err := compileMSSQLFilterSQL(opts.Filter, nextArg) - if err != nil { - return searchSQLPlan{}, err - } - if filterSQL != "" { - whereParts = append(whereParts, filterSQL) - } - args = append(args, filterArgs...) - nextArg = next - } - - outerWhere := "" - if opts.Threshold != nil { - thresholdPlaceholder := fmt.Sprintf("@p%d", nextArg) - args = append(args, *opts.Threshold) - nextArg++ - outerWhere = fmt.Sprintf("WHERE ranked.%s <= %s", quoteIdent("distance"), thresholdPlaceholder) - } - - limitPlaceholder := fmt.Sprintf("@p%d", nextArg) - args = append(args, topK) - - innerSelectCols := []string{ - fmt.Sprintf("t.%s AS %s", quoteIdent(idColumn), quoteIdent(idColumn)), - } - outerSelectCols := []string{ - fmt.Sprintf("ranked.%s", quoteIdent(idColumn)), - } - if projection.IncludeVector { - innerSelectCols = append(innerSelectCols, fmt.Sprintf("t.%s AS %s", quoteIdent(vectorColumn), quoteIdent(vectorColumn))) - outerSelectCols = append(outerSelectCols, fmt.Sprintf("ranked.%s", quoteIdent(vectorColumn))) - } - if projection.IncludeMetadata { - innerSelectCols = append(innerSelectCols, fmt.Sprintf("t.%s AS %s", quoteIdent(metadataColumn), quoteIdent(metadataColumn))) - outerSelectCols = append(outerSelectCols, fmt.Sprintf("ranked.%s", quoteIdent(metadataColumn))) - } - if projection.IncludeContent { - innerSelectCols = append(innerSelectCols, fmt.Sprintf("t.%s AS %s", quoteIdent(contentColumn), quoteIdent(contentColumn))) - outerSelectCols = append(outerSelectCols, fmt.Sprintf("ranked.%s", quoteIdent(contentColumn))) - } - innerSelectCols = append(innerSelectCols, fmt.Sprintf("%s AS %s", distanceExpr, quoteIdent("distance"))) - outerSelectCols = append(outerSelectCols, fmt.Sprintf("ranked.%s", quoteIdent("distance"))) - - query := fmt.Sprintf(` -SELECT %s -FROM ( - SELECT %s - FROM %s AS t - CROSS APPLY ( - SELECT - COUNT(*) AS candidate_dim, - SUM(CONVERT(float, tv.[value]) * CONVERT(float, tv.[value])) AS candidate_norm_sq - FROM OPENJSON(t.%s) AS tv - ) AS candidate_stats - CROSS APPLY ( - SELECT - COUNT(*) AS matched_dim, - SUM(POWER(CONVERT(float, tv.[value]) - CONVERT(float, qv.[value]), 2.0)) AS sum_sq_diff, - SUM(CONVERT(float, tv.[value]) * CONVERT(float, qv.[value])) AS dot_product, - SUM(CONVERT(float, qv.[value]) * CONVERT(float, qv.[value])) AS query_norm_sq - FROM OPENJSON(t.%s) AS tv - INNER JOIN OPENJSON(%s) AS qv ON qv.[key] = tv.[key] - ) AS vec_stats - WHERE %s -) AS ranked -%s -ORDER BY ranked.%s ASC, ranked.%s ASC -OFFSET 0 ROWS FETCH NEXT %s ROWS ONLY`, - strings.Join(outerSelectCols, ", "), - strings.Join(innerSelectCols, ", "), - c.tableName(), - quoteIdent(vectorColumn), - quoteIdent(vectorColumn), - queryVectorPlaceholder, - strings.Join(whereParts, " AND "), - outerWhere, - quoteIdent("distance"), - quoteIdent(idColumn), - limitPlaceholder, - ) - - return searchSQLPlan{ - query: query, - args: args, - projection: projection, - }, nil -} - -func (c *MSSQLCollection) executeSearchSQLPlan(ctx context.Context, plan searchSQLPlan) ([]vectordata.SearchResult, error) { - rows, err := c.store.db.QueryContext(ctx, plan.query, plan.args...) - if err != nil { - return nil, err - } - defer rows.Close() - - results := make([]vectordata.SearchResult, 0) - for rows.Next() { - var rec vectordata.Record - var vectorRaw string - var metadataRaw string - var content sql.NullString - var distance float64 - - scanTargets := []any{&rec.ID} - if plan.projection.IncludeVector { - scanTargets = append(scanTargets, &vectorRaw) - } - if plan.projection.IncludeMetadata { - scanTargets = append(scanTargets, &metadataRaw) - } - if plan.projection.IncludeContent { - scanTargets = append(scanTargets, &content) - } - scanTargets = append(scanTargets, &distance) - - if err := rows.Scan(scanTargets...); err != nil { - return nil, err - } - - if plan.projection.IncludeVector { - parsedVector, err := parseVectorJSON(vectorRaw) - if err != nil { - return nil, fmt.Errorf("decode vector: %w", err) - } - rec.Vector = parsedVector - } - if plan.projection.IncludeMetadata { - parsedMetadata, err := parseMetadataJSON(metadataRaw) - if err != nil { - return nil, fmt.Errorf("decode metadata: %w", err) - } - rec.Metadata = parsedMetadata - } - if plan.projection.IncludeContent && content.Valid { - value := content.String - rec.Content = &value - } - - results = append(results, vectordata.SearchResult{ - Record: rec, - Distance: distance, - Score: vectordata.ScoreFromDistance(defaultMetric(c.metric), distance), - }) - } - if err := rows.Err(); err != nil { - return nil, err - } - - return results, nil -} - -func searchDistanceExpr(metric vectordata.DistanceMetric) (string, error) { - switch metric { - case vectordata.DistanceCosine: - return `CASE - WHEN vec_stats.query_norm_sq = 0 OR candidate_stats.candidate_norm_sq = 0 THEN 1.0 - ELSE 1.0 - (vec_stats.dot_product / (SQRT(vec_stats.query_norm_sq) * SQRT(candidate_stats.candidate_norm_sq))) - END`, nil - case vectordata.DistanceL2: - return "SQRT(vec_stats.sum_sq_diff)", nil - case vectordata.DistanceInnerProduct: - return "(-vec_stats.dot_product)", nil - default: - return "", fmt.Errorf("%w: unsupported distance metric %q", vectordata.ErrSchemaMismatch, metric) - } -} diff --git a/stores/mssql/store.go b/stores/mssql/store.go deleted file mode 100644 index 88526bc..0000000 --- a/stores/mssql/store.go +++ /dev/null @@ -1,111 +0,0 @@ -package mssql - -import ( - "context" - "database/sql" - "fmt" - "strings" - - "github.com/gabisonia/go-vectorstore/vectordata" -) - -// StoreOptions configures MSSQLVectorStore behavior. -type StoreOptions struct { - Schema string - StrictByDefault bool -} - -// DefaultStoreOptions returns production-safe defaults. -func DefaultStoreOptions() StoreOptions { - return StoreOptions{ - Schema: "dbo", - StrictByDefault: true, - } -} - -// MSSQLVectorStore implements vectordata.VectorStore using database/sql. -type MSSQLVectorStore struct { - db *sql.DB - opts StoreOptions -} - -// NewVectorStore creates a SQL Server-backed vector store. -func NewVectorStore(db *sql.DB, opts StoreOptions) (*MSSQLVectorStore, error) { - if db == nil { - return nil, fmt.Errorf("nil sql db") - } - - normalized := opts.withDefaults() - if err := normalized.validate(); err != nil { - return nil, err - } - - return &MSSQLVectorStore{db: db, opts: normalized}, nil -} - -// Collection returns a handle to a collection without schema checks. -func (s *MSSQLVectorStore) Collection(name string, dimension int, metric vectordata.DistanceMetric) vectordata.Collection { - return s.newCollectionHandle(name, dimension, metric) -} - -// EnsureCollection creates or validates a collection schema and returns its handle. -func (s *MSSQLVectorStore) EnsureCollection(ctx context.Context, spec vectordata.CollectionSpec) (vectordata.Collection, error) { - normalizedSpec, mode, err := s.normalizeCollectionSpec(spec) - if err != nil { - return nil, err - } - - if err := s.ensureBaseSchema(ctx); err != nil { - return nil, err - } - if err := s.ensureTableWithValidation(ctx, normalizedSpec, mode); err != nil { - return nil, err - } - - return s.newCollectionHandle(normalizedSpec.Name, normalizedSpec.Dimension, normalizedSpec.Metric), nil -} - -func (s *MSSQLVectorStore) normalizeCollectionSpec(spec vectordata.CollectionSpec) (vectordata.CollectionSpec, vectordata.EnsureMode, error) { - spec.Name = strings.TrimSpace(spec.Name) - if spec.Name == "" { - return vectordata.CollectionSpec{}, "", fmt.Errorf("%w: collection name is empty", vectordata.ErrSchemaMismatch) - } - if spec.Dimension <= 0 { - return vectordata.CollectionSpec{}, "", fmt.Errorf("%w: dimension must be > 0", vectordata.ErrSchemaMismatch) - } - - spec.Metric = defaultMetric(spec.Metric) - if err := spec.Metric.Validate(); err != nil { - return vectordata.CollectionSpec{}, "", err - } - - mode := defaultMode(spec.Mode, s.opts.StrictByDefault) - if mode != vectordata.EnsureStrict && mode != vectordata.EnsureAutoMigrate { - return vectordata.CollectionSpec{}, "", fmt.Errorf("%w: unsupported ensure mode %q", vectordata.ErrSchemaMismatch, mode) - } - - return spec, mode, nil -} - -func (s *MSSQLVectorStore) newCollectionHandle(name string, dimension int, metric vectordata.DistanceMetric) vectordata.Collection { - return &MSSQLCollection{ - store: s, - name: strings.TrimSpace(name), - dimension: dimension, - metric: defaultMetric(metric), - } -} - -func (s StoreOptions) withDefaults() StoreOptions { - if strings.TrimSpace(s.Schema) == "" { - s.Schema = "dbo" - } - return s -} - -func (s StoreOptions) validate() error { - if strings.TrimSpace(s.Schema) == "" { - return fmt.Errorf("%w: schema is empty", vectordata.ErrSchemaMismatch) - } - return nil -}