From dc941d7e1aeb893188a9fc36863e797bfe71fe5a Mon Sep 17 00:00:00 2001 From: gabisonia Date: Mon, 16 Feb 2026 17:13:27 +0400 Subject: [PATCH] Prepare v0.2.3 release --- .github/workflows/release.yml | 2 +- README.md | 6 +- docs/stores-implementation.md | 7 +- stores/mssql/collection.go | 32 +++- stores/mssql/collection_test.go | 67 +++++++ stores/mssql/filter_eval.go | 29 ++- stores/mssql/filter_eval_test.go | 27 +++ stores/mssql/filter_pushdown.go | 255 +++++++++++++++++++++++++++ stores/mssql/filter_pushdown_test.go | 73 ++++++++ stores/mssql/search_sql.go | 212 ++++++++++++++++++++++ vectordata/filter_normalize.go | 34 ++++ vectordata/filter_normalize_test.go | 20 +++ vectordata/filter_sql.go | 23 ++- vectordata/filter_sql_test.go | 35 ++++ 14 files changed, 782 insertions(+), 40 deletions(-) create mode 100644 stores/mssql/filter_pushdown.go create mode 100644 stores/mssql/filter_pushdown_test.go create mode 100644 stores/mssql/search_sql.go create mode 100644 vectordata/filter_normalize.go create mode 100644 vectordata/filter_normalize_test.go diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7799f9d..dae0d78 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -7,7 +7,7 @@ on: workflow_dispatch: inputs: version: - description: "Semver version (e.g. 0.2.2 or v0.2.2)" + description: "Semver version (e.g. 0.2.3 or v0.2.3)" required: true type: string diff --git a/README.md b/README.md index c508557..e2d5289 100644 --- a/README.md +++ b/README.md @@ -229,12 +229,12 @@ GitHub Actions workflows are configured for: Release options: -1. Manual (recommended): run `Release` workflow via GitHub UI with `version` input (`0.2.2` or `v0.2.2`). +1. Manual (recommended): run `Release` workflow via GitHub UI with `version` input (`0.2.3` or `v0.2.3`). 2. Tag-driven: push a semver tag and the workflow publishes release notes automatically: ```bash -git tag v0.2.2 -git push origin v0.2.2 +git tag v0.2.3 +git push origin v0.2.3 ``` ## Samples diff --git a/docs/stores-implementation.md b/docs/stores-implementation.md index 912014b..e3f26bd 100644 --- a/docs/stores-implementation.md +++ b/docs/stores-implementation.md @@ -184,7 +184,10 @@ Search behavior (MVP): 6. Maintain bounded top-k max-heap in memory (`O(k)` memory) 7. Return results sorted by distance asc (tie-break by `ID`) -`EnsureIndexes` is intentionally a no-op in MSSQL backend for this MVP. +`EnsureIndexes` is explicit in MSSQL backend: + +- returns `nil` when no index options are requested +- returns an error when vector/metadata index options are provided (index management is not supported yet) ## 6) Filter System and Execution Model @@ -242,7 +245,7 @@ These rules are enforced in both backends: - MSSQL: in-process AST evaluation - Indexing: - Postgres: vector + metadata index creation supported - - MSSQL: no-op in MVP + - MSSQL: explicit unsupported error when index options are requested - Metric/dimension persistence: - Postgres: inferred/validated from physical `vector(n)` column - MSSQL: persisted in `__vector_collections` diff --git a/stores/mssql/collection.go b/stores/mssql/collection.go index f67988e..579fe11 100644 --- a/stores/mssql/collection.go +++ b/stores/mssql/collection.go @@ -127,6 +127,19 @@ func (c *MSSQLCollection) Count(ctx context.Context, filter vectordata.Filter) ( return count, nil } + filterSQL, filterArgs, _, err := compileMSSQLFilterSQL(filter, 1) + if err == nil { + query := fmt.Sprintf("SELECT COUNT(*) FROM %s WHERE %s", c.tableName(), filterSQL) + var count int64 + if err := c.store.db.QueryRowContext(ctx, query, filterArgs...).Scan(&count); err != nil { + return 0, err + } + return count, nil + } + if !errors.Is(err, errFilterPushdownUnsupported) { + return 0, err + } + count := int64(0) if err := c.streamRecords(ctx, false, func(record vectordata.Record) error { matches, err := matchesFilter(filter, record) @@ -151,6 +164,18 @@ func (c *MSSQLCollection) SearchByVector(ctx context.Context, vector []float32, return nil, err } + plan, err := c.buildSearchSQLPlan(vector, topK, opts) + if err == nil { + return c.executeSearchSQLPlan(ctx, plan) + } + if !errors.Is(err, errFilterPushdownUnsupported) { + return nil, err + } + + return c.searchByVectorStreaming(ctx, vector, topK, opts) +} + +func (c *MSSQLCollection) searchByVectorStreaming(ctx context.Context, vector []float32, topK int, opts vectordata.SearchOptions) ([]vectordata.SearchResult, error) { projection := resolveProjection(opts.Projection) metric := defaultMetric(c.metric) topKHeap := make(searchResultMaxHeap, 0, topK) @@ -206,10 +231,11 @@ func (c *MSSQLCollection) SearchByVector(ctx context.Context, vector []float32, } func (c *MSSQLCollection) EnsureIndexes(ctx context.Context, opts vectordata.IndexOptions) error { - // SQL Server backend stores vectors as JSON payloads in this MVP, so index options are no-op. _ = ctx - _ = opts - return nil + if opts.Vector == nil && opts.Metadata == nil { + return nil + } + return fmt.Errorf("%w: index management is not supported by the mssql backend", vectordata.ErrSchemaMismatch) } func (c *MSSQLCollection) writeRecords(ctx context.Context, records []vectordata.Record, mode writeMode) error { diff --git a/stores/mssql/collection_test.go b/stores/mssql/collection_test.go index bf8d1dd..61ec261 100644 --- a/stores/mssql/collection_test.go +++ b/stores/mssql/collection_test.go @@ -2,6 +2,9 @@ package mssql import ( "container/heap" + "context" + "errors" + "reflect" "sort" "strings" "testing" @@ -67,3 +70,67 @@ func TestSearchResultHeapKeepsBestTopK(t *testing.T) { } } } + +func TestEnsureIndexesMSSQL(t *testing.T) { + collection := &MSSQLCollection{} + + if err := collection.EnsureIndexes(context.Background(), vectordata.IndexOptions{}); err != nil { + t.Fatalf("expected empty options to succeed, got %v", err) + } + + err := collection.EnsureIndexes(context.Background(), vectordata.IndexOptions{ + Vector: &vectordata.VectorIndexOptions{Method: vectordata.IndexMethodHNSW}, + }) + if err == nil { + t.Fatal("expected error when index options are provided") + } + if !errors.Is(err, vectordata.ErrSchemaMismatch) { + t.Fatalf("expected ErrSchemaMismatch, got %v", err) + } +} + +func TestBuildSearchSQLPlan(t *testing.T) { + threshold := 0.55 + collection := &MSSQLCollection{ + store: &MSSQLVectorStore{opts: StoreOptions{Schema: "dbo"}}, + name: "docs", + dimension: 2, + metric: vectordata.DistanceCosine, + } + + plan, err := collection.buildSearchSQLPlan([]float32{1, 0}, 3, vectordata.SearchOptions{ + Threshold: &threshold, + Projection: &vectordata.Projection{ + IncludeMetadata: true, + IncludeContent: true, + }, + }) + if err != nil { + t.Fatalf("buildSearchSQLPlan: %v", err) + } + if !strings.Contains(plan.query, "FETCH NEXT @p4 ROWS ONLY") { + t.Fatalf("expected limit placeholder in query, got: %s", plan.query) + } + if !reflect.DeepEqual(plan.args, []any{"[1,0]", 2, threshold, 3}) { + t.Fatalf("unexpected args: %#v", plan.args) + } +} + +func TestBuildSearchSQLPlanRejectsUnsupportedFilterPushdown(t *testing.T) { + collection := &MSSQLCollection{ + store: &MSSQLVectorStore{opts: StoreOptions{Schema: "dbo"}}, + name: "docs", + dimension: 2, + metric: vectordata.DistanceCosine, + } + + _, err := collection.buildSearchSQLPlan([]float32{1, 0}, 3, vectordata.SearchOptions{ + Filter: vectordata.Eq(vectordata.Column("id"), 123), + }) + if err == nil { + t.Fatal("expected error") + } + if !errors.Is(err, errFilterPushdownUnsupported) { + t.Fatalf("expected errFilterPushdownUnsupported, got %v", err) + } +} diff --git a/stores/mssql/filter_eval.go b/stores/mssql/filter_eval.go index 4a84fb8..c376d0e 100644 --- a/stores/mssql/filter_eval.go +++ b/stores/mssql/filter_eval.go @@ -3,7 +3,6 @@ package mssql import ( "fmt" "reflect" - "strings" "github.com/gabisonia/go-vectorstore/vectordata" ) @@ -115,14 +114,14 @@ func matchesFilter(filter vectordata.Filter, record vectordata.Record) (bool, er } func resolveFieldValue(field vectordata.FieldRef, record vectordata.Record) (value any, exists bool, err error) { - switch field.Kind { - case vectordata.FieldColumn: - name := strings.TrimSpace(field.Name) - if name == "" { - return nil, false, fmt.Errorf("%w: column field name is empty", vectordata.ErrInvalidFilter) - } + normalized, err := vectordata.NormalizeFieldRef(field) + if err != nil { + return nil, false, err + } - switch name { + switch normalized.Kind { + case vectordata.FieldColumn: + switch normalized.Name { case idColumn: return record.ID, true, nil case contentColumn: @@ -131,23 +130,15 @@ func resolveFieldValue(field vectordata.FieldRef, record vectordata.Record) (val } return *record.Content, true, nil default: - return nil, false, fmt.Errorf("%w: unknown column %q", vectordata.ErrInvalidFilter, name) + return nil, false, fmt.Errorf("%w: unknown column %q", vectordata.ErrInvalidFilter, normalized.Name) } case vectordata.FieldMetadata: - if len(field.Path) == 0 { - return nil, false, fmt.Errorf("%w: metadata path is empty", vectordata.ErrInvalidFilter) - } if record.Metadata == nil { return nil, false, nil } var current any = record.Metadata - for _, segment := range field.Path { - key := strings.TrimSpace(segment) - if key == "" { - return nil, false, fmt.Errorf("%w: metadata path segment is empty", vectordata.ErrInvalidFilter) - } - + for _, key := range normalized.Path { asMap, ok := current.(map[string]any) if !ok { return nil, false, nil @@ -162,7 +153,7 @@ func resolveFieldValue(field vectordata.FieldRef, record vectordata.Record) (val return current, true, nil default: - return nil, false, fmt.Errorf("%w: unsupported field kind %q", vectordata.ErrInvalidFilter, field.Kind) + return nil, false, fmt.Errorf("%w: unsupported field kind %q", vectordata.ErrInvalidFilter, normalized.Kind) } } diff --git a/stores/mssql/filter_eval_test.go b/stores/mssql/filter_eval_test.go index 89a6ae2..f24e709 100644 --- a/stores/mssql/filter_eval_test.go +++ b/stores/mssql/filter_eval_test.go @@ -57,6 +57,33 @@ func TestMatchesFilterUnknownColumnReturnsError(t *testing.T) { } } +func TestMatchesFilterTrimsFieldReferences(t *testing.T) { + content := "x" + record := vectordata.Record{ + ID: "doc-1", + Content: &content, + Metadata: map[string]any{ + "nested": map[string]any{"key": "value"}, + }, + } + + columnMatch, err := matchesFilter(vectordata.Eq(vectordata.Column(" id "), "doc-1"), record) + if err != nil { + t.Fatalf("column match: %v", err) + } + if !columnMatch { + t.Fatalf("expected trimmed column reference to match") + } + + metadataMatch, err := matchesFilter(vectordata.Eq(vectordata.Metadata(" nested ", " key "), "value"), record) + if err != nil { + t.Fatalf("metadata match: %v", err) + } + if !metadataMatch { + t.Fatalf("expected trimmed metadata path to match") + } +} + func TestDistanceBetweenMetrics(t *testing.T) { left := []float32{1, 0} right := []float32{0.8, 0.2} diff --git a/stores/mssql/filter_pushdown.go b/stores/mssql/filter_pushdown.go new file mode 100644 index 0000000..d4daebf --- /dev/null +++ b/stores/mssql/filter_pushdown.go @@ -0,0 +1,255 @@ +package mssql + +import ( + "errors" + "fmt" + "strings" + + "github.com/gabisonia/go-vectorstore/vectordata" +) + +var errFilterPushdownUnsupported = errors.New("mssql filter pushdown unsupported") + +func compileMSSQLFilterSQL(filter vectordata.Filter, startArg int) (sql string, args []any, nextArg int, err error) { + if startArg < 1 { + startArg = 1 + } + if filter == nil { + return "", nil, startArg, nil + } + + c := &mssqlFilterCompiler{ + nextArg: startArg, + } + out, err := c.compile(filter) + if err != nil { + return "", nil, startArg, err + } + return out, c.args, c.nextArg, nil +} + +type mssqlFilterCompiler struct { + args []any + nextArg int +} + +func (c *mssqlFilterCompiler) compile(filter vectordata.Filter) (string, error) { + switch node := filter.(type) { + case vectordata.EqFilter: + return c.compileEq(node) + case vectordata.InFilter: + return c.compileIn(node) + case vectordata.GtFilter: + return c.compileGt(node) + case vectordata.LtFilter: + return c.compileLt(node) + case vectordata.ExistsFilter: + return c.compileExists(node) + case vectordata.AndFilter: + return c.compileLogical("AND", node.Children) + case vectordata.OrFilter: + return c.compileLogical("OR", node.Children) + case vectordata.NotFilter: + if node.Child == nil { + return "", fmt.Errorf("%w: NOT requires a child", vectordata.ErrInvalidFilter) + } + childSQL, err := c.compile(node.Child) + if err != nil { + return "", err + } + return fmt.Sprintf("(NOT %s)", childSQL), nil + default: + return "", fmt.Errorf("%w: unsupported node type %T", vectordata.ErrInvalidFilter, filter) + } +} + +func (c *mssqlFilterCompiler) compileEq(node vectordata.EqFilter) (string, error) { + columnExpr, metadataPath, isMetadata, err := c.resolveField(node.Field) + if err != nil { + return "", err + } + if !isMetadata { + value, ok := node.Value.(string) + if !ok { + return "", unsupportedPushdown("column equality only supports string values") + } + return fmt.Sprintf("(%s = %s)", columnExpr, c.bind(value)), nil + } + return c.compileMetadataEq(metadataPath, node.Value) +} + +func (c *mssqlFilterCompiler) compileIn(node vectordata.InFilter) (string, error) { + if len(node.Values) == 0 { + return "", fmt.Errorf("%w: IN requires at least one value", vectordata.ErrInvalidFilter) + } + + columnExpr, metadataPath, isMetadata, err := c.resolveField(node.Field) + if err != nil { + return "", err + } + + if !isMetadata { + parts := make([]string, 0, len(node.Values)) + for _, value := range node.Values { + text, ok := value.(string) + if !ok { + return "", unsupportedPushdown("column IN only supports string values") + } + parts = append(parts, c.bind(text)) + } + return fmt.Sprintf("(%s IN (%s))", columnExpr, strings.Join(parts, ", ")), nil + } + + predicates := make([]string, 0, len(node.Values)) + for _, value := range node.Values { + predicate, err := c.compileMetadataEq(metadataPath, value) + if err != nil { + return "", err + } + predicates = append(predicates, predicate) + } + return fmt.Sprintf("(%s)", strings.Join(predicates, " OR ")), nil +} + +func (c *mssqlFilterCompiler) compileGt(node vectordata.GtFilter) (string, error) { + columnExpr, metadataPath, isMetadata, err := c.resolveField(node.Field) + if err != nil { + return "", err + } + if !isMetadata { + value, ok := node.Value.(string) + if !ok { + return "", unsupportedPushdown("column greater-than only supports string values") + } + return fmt.Sprintf("(%s > %s)", columnExpr, c.bind(value)), nil + } + + if number, ok := toFloat64(node.Value); ok { + pathPlaceholder := c.bind(metadataPath) + valueExpr := fmt.Sprintf("JSON_VALUE(%s, %s)", quoteIdent(metadataColumn), pathPlaceholder) + return fmt.Sprintf("(TRY_CONVERT(float, %s) > %s)", valueExpr, c.bind(number)), nil + } + return "", unsupportedPushdown("metadata greater-than only supports numeric values") +} + +func (c *mssqlFilterCompiler) compileLt(node vectordata.LtFilter) (string, error) { + columnExpr, metadataPath, isMetadata, err := c.resolveField(node.Field) + if err != nil { + return "", err + } + if !isMetadata { + value, ok := node.Value.(string) + if !ok { + return "", unsupportedPushdown("column less-than only supports string values") + } + return fmt.Sprintf("(%s < %s)", columnExpr, c.bind(value)), nil + } + + if number, ok := toFloat64(node.Value); ok { + pathPlaceholder := c.bind(metadataPath) + valueExpr := fmt.Sprintf("JSON_VALUE(%s, %s)", quoteIdent(metadataColumn), pathPlaceholder) + return fmt.Sprintf("(TRY_CONVERT(float, %s) < %s)", valueExpr, c.bind(number)), nil + } + return "", unsupportedPushdown("metadata less-than only supports numeric values") +} + +func (c *mssqlFilterCompiler) compileExists(node vectordata.ExistsFilter) (string, error) { + columnExpr, metadataPath, isMetadata, err := c.resolveField(node.Field) + if err != nil { + return "", err + } + if !isMetadata { + return fmt.Sprintf("(%s IS NOT NULL)", columnExpr), nil + } + pathPlaceholder := c.bind(metadataPath) + return fmt.Sprintf("(JSON_PATH_EXISTS(%s, %s) = 1)", quoteIdent(metadataColumn), pathPlaceholder), nil +} + +func (c *mssqlFilterCompiler) compileLogical(op string, children []vectordata.Filter) (string, error) { + if len(children) == 0 { + return "", fmt.Errorf("%w: %s requires at least one child", vectordata.ErrInvalidFilter, op) + } + + parts := make([]string, 0, len(children)) + for _, child := range children { + if child == nil { + return "", fmt.Errorf("%w: %s contains nil child", vectordata.ErrInvalidFilter, op) + } + childSQL, err := c.compile(child) + if err != nil { + return "", err + } + parts = append(parts, childSQL) + } + return fmt.Sprintf("(%s)", strings.Join(parts, fmt.Sprintf(" %s ", op))), nil +} + +func (c *mssqlFilterCompiler) resolveField(ref vectordata.FieldRef) (columnExpr string, metadataPath string, isMetadata bool, err error) { + normalized, err := vectordata.NormalizeFieldRef(ref) + if err != nil { + return "", "", false, err + } + + switch normalized.Kind { + case vectordata.FieldColumn: + switch normalized.Name { + case idColumn: + return quoteIdent(idColumn), "", false, nil + case contentColumn: + return quoteIdent(contentColumn), "", false, nil + default: + return "", "", false, fmt.Errorf("%w: unknown column %q", vectordata.ErrInvalidFilter, normalized.Name) + } + case vectordata.FieldMetadata: + return "", metadataPathLiteral(normalized.Path), true, nil + default: + return "", "", false, fmt.Errorf("%w: unsupported field kind %q", vectordata.ErrInvalidFilter, normalized.Kind) + } +} + +func (c *mssqlFilterCompiler) compileMetadataEq(metadataPath string, value any) (string, error) { + pathPlaceholder := c.bind(metadataPath) + valueExpr := fmt.Sprintf("JSON_VALUE(%s, %s)", quoteIdent(metadataColumn), pathPlaceholder) + + switch typed := value.(type) { + case nil: + return "", unsupportedPushdown("metadata equality with nil is not supported by SQL pushdown") + case string: + return fmt.Sprintf("(%s = %s)", valueExpr, c.bind(typed)), nil + case bool: + boolValue := "false" + if typed { + boolValue = "true" + } + return fmt.Sprintf("(LOWER(%s) = %s)", valueExpr, c.bind(boolValue)), nil + default: + if number, ok := toFloat64(value); ok { + return fmt.Sprintf("(TRY_CONVERT(float, %s) = %s)", valueExpr, c.bind(number)), nil + } + return "", unsupportedPushdown("metadata scalar comparison does not support value type %T", value) + } +} + +func (c *mssqlFilterCompiler) bind(value any) string { + placeholder := fmt.Sprintf("@p%d", c.nextArg) + c.nextArg++ + c.args = append(c.args, value) + return placeholder +} + +func metadataPathLiteral(path []string) string { + var b strings.Builder + b.WriteString("$") + for _, segment := range path { + escaped := strings.ReplaceAll(segment, `\`, `\\`) + escaped = strings.ReplaceAll(escaped, `"`, `\"`) + b.WriteString(`."`) + b.WriteString(escaped) + b.WriteString(`"`) + } + return b.String() +} + +func unsupportedPushdown(format string, args ...any) error { + return fmt.Errorf("%w: %s", errFilterPushdownUnsupported, fmt.Sprintf(format, args...)) +} diff --git a/stores/mssql/filter_pushdown_test.go b/stores/mssql/filter_pushdown_test.go new file mode 100644 index 0000000..140e62a --- /dev/null +++ b/stores/mssql/filter_pushdown_test.go @@ -0,0 +1,73 @@ +package mssql + +import ( + "errors" + "reflect" + "testing" + + "github.com/gabisonia/go-vectorstore/vectordata" +) + +func TestCompileMSSQLFilterSQL_TrimsColumnReference(t *testing.T) { + sql, args, next, err := compileMSSQLFilterSQL(vectordata.Eq(vectordata.Column(" id "), "doc-1"), 1) + if err != nil { + t.Fatalf("compileMSSQLFilterSQL: %v", err) + } + if sql != `([id] = @p1)` { + t.Fatalf("unexpected SQL: %s", sql) + } + if !reflect.DeepEqual(args, []any{"doc-1"}) { + t.Fatalf("unexpected args: %#v", args) + } + if next != 2 { + t.Fatalf("unexpected next arg index: %d", next) + } +} + +func TestCompileMSSQLFilterSQL_MetadataExists(t *testing.T) { + sql, args, _, err := compileMSSQLFilterSQL(vectordata.Exists(vectordata.Metadata(" nested ", "value")), 3) + if err != nil { + t.Fatalf("compileMSSQLFilterSQL: %v", err) + } + expectedSQL := `(JSON_PATH_EXISTS([metadata], @p3) = 1)` + if sql != expectedSQL { + t.Fatalf("unexpected SQL:\nwant: %s\n got: %s", expectedSQL, sql) + } + if !reflect.DeepEqual(args, []any{`$."nested"."value"`}) { + t.Fatalf("unexpected args: %#v", args) + } +} + +func TestCompileMSSQLFilterSQL_MetadataEqAndGt(t *testing.T) { + filter := vectordata.And( + vectordata.Eq(vectordata.Metadata("category"), "news"), + vectordata.Gt(vectordata.Metadata("rank"), 1), + ) + + sql, args, next, err := compileMSSQLFilterSQL(filter, 2) + if err != nil { + t.Fatalf("compileMSSQLFilterSQL: %v", err) + } + + expectedSQL := `((JSON_VALUE([metadata], @p2) = @p3) AND (TRY_CONVERT(float, JSON_VALUE([metadata], @p4)) > @p5))` + if sql != expectedSQL { + t.Fatalf("unexpected SQL:\nwant: %s\n got: %s", expectedSQL, sql) + } + expectedArgs := []any{`$."category"`, "news", `$."rank"`, float64(1)} + if !reflect.DeepEqual(args, expectedArgs) { + t.Fatalf("unexpected args:\nwant: %#v\n got: %#v", expectedArgs, args) + } + if next != 6 { + t.Fatalf("unexpected next arg index: %d", next) + } +} + +func TestCompileMSSQLFilterSQL_UnsupportedColumnValueType(t *testing.T) { + _, _, _, err := compileMSSQLFilterSQL(vectordata.Eq(vectordata.Column("id"), 123), 1) + if err == nil { + t.Fatal("expected error") + } + if !errors.Is(err, errFilterPushdownUnsupported) { + t.Fatalf("expected errFilterPushdownUnsupported, got %v", err) + } +} diff --git a/stores/mssql/search_sql.go b/stores/mssql/search_sql.go new file mode 100644 index 0000000..e023414 --- /dev/null +++ b/stores/mssql/search_sql.go @@ -0,0 +1,212 @@ +package mssql + +import ( + "context" + "database/sql" + "fmt" + "strings" + + "github.com/gabisonia/go-vectorstore/vectordata" +) + +type searchSQLPlan struct { + query string + args []any + projection vectordata.Projection +} + +func (c *MSSQLCollection) buildSearchSQLPlan(vector []float32, topK int, opts vectordata.SearchOptions) (searchSQLPlan, error) { + projection := resolveProjection(opts.Projection) + vectorPayload, err := vectorJSON(vector) + if err != nil { + return searchSQLPlan{}, fmt.Errorf("encode query vector: %w", err) + } + + distanceExpr, err := searchDistanceExpr(defaultMetric(c.metric)) + if err != nil { + return searchSQLPlan{}, err + } + + args := make([]any, 0, 8) + nextArg := 1 + + queryVectorPlaceholder := fmt.Sprintf("@p%d", nextArg) + args = append(args, vectorPayload) + nextArg++ + + expectedDimPlaceholder := fmt.Sprintf("@p%d", nextArg) + args = append(args, c.dimension) + nextArg++ + + whereParts := []string{ + fmt.Sprintf("candidate_stats.candidate_dim = %s", expectedDimPlaceholder), + fmt.Sprintf("vec_stats.matched_dim = %s", expectedDimPlaceholder), + } + + if opts.Filter != nil { + filterSQL, filterArgs, next, err := compileMSSQLFilterSQL(opts.Filter, nextArg) + if err != nil { + return searchSQLPlan{}, err + } + if filterSQL != "" { + whereParts = append(whereParts, filterSQL) + } + args = append(args, filterArgs...) + nextArg = next + } + + outerWhere := "" + if opts.Threshold != nil { + thresholdPlaceholder := fmt.Sprintf("@p%d", nextArg) + args = append(args, *opts.Threshold) + nextArg++ + outerWhere = fmt.Sprintf("WHERE ranked.%s <= %s", quoteIdent("distance"), thresholdPlaceholder) + } + + limitPlaceholder := fmt.Sprintf("@p%d", nextArg) + args = append(args, topK) + + innerSelectCols := []string{ + fmt.Sprintf("t.%s AS %s", quoteIdent(idColumn), quoteIdent(idColumn)), + } + outerSelectCols := []string{ + fmt.Sprintf("ranked.%s", quoteIdent(idColumn)), + } + if projection.IncludeVector { + innerSelectCols = append(innerSelectCols, fmt.Sprintf("t.%s AS %s", quoteIdent(vectorColumn), quoteIdent(vectorColumn))) + outerSelectCols = append(outerSelectCols, fmt.Sprintf("ranked.%s", quoteIdent(vectorColumn))) + } + if projection.IncludeMetadata { + innerSelectCols = append(innerSelectCols, fmt.Sprintf("t.%s AS %s", quoteIdent(metadataColumn), quoteIdent(metadataColumn))) + outerSelectCols = append(outerSelectCols, fmt.Sprintf("ranked.%s", quoteIdent(metadataColumn))) + } + if projection.IncludeContent { + innerSelectCols = append(innerSelectCols, fmt.Sprintf("t.%s AS %s", quoteIdent(contentColumn), quoteIdent(contentColumn))) + outerSelectCols = append(outerSelectCols, fmt.Sprintf("ranked.%s", quoteIdent(contentColumn))) + } + innerSelectCols = append(innerSelectCols, fmt.Sprintf("%s AS %s", distanceExpr, quoteIdent("distance"))) + outerSelectCols = append(outerSelectCols, fmt.Sprintf("ranked.%s", quoteIdent("distance"))) + + query := fmt.Sprintf(` +SELECT %s +FROM ( + SELECT %s + FROM %s AS t + CROSS APPLY ( + SELECT + COUNT(*) AS candidate_dim, + SUM(CONVERT(float, tv.[value]) * CONVERT(float, tv.[value])) AS candidate_norm_sq + FROM OPENJSON(t.%s) AS tv + ) AS candidate_stats + CROSS APPLY ( + SELECT + COUNT(*) AS matched_dim, + SUM(POWER(CONVERT(float, tv.[value]) - CONVERT(float, qv.[value]), 2.0)) AS sum_sq_diff, + SUM(CONVERT(float, tv.[value]) * CONVERT(float, qv.[value])) AS dot_product, + SUM(CONVERT(float, qv.[value]) * CONVERT(float, qv.[value])) AS query_norm_sq + FROM OPENJSON(t.%s) AS tv + INNER JOIN OPENJSON(%s) AS qv ON qv.[key] = tv.[key] + ) AS vec_stats + WHERE %s +) AS ranked +%s +ORDER BY ranked.%s ASC, ranked.%s ASC +OFFSET 0 ROWS FETCH NEXT %s ROWS ONLY`, + strings.Join(outerSelectCols, ", "), + strings.Join(innerSelectCols, ", "), + c.tableName(), + quoteIdent(vectorColumn), + quoteIdent(vectorColumn), + queryVectorPlaceholder, + strings.Join(whereParts, " AND "), + outerWhere, + quoteIdent("distance"), + quoteIdent(idColumn), + limitPlaceholder, + ) + + return searchSQLPlan{ + query: query, + args: args, + projection: projection, + }, nil +} + +func (c *MSSQLCollection) executeSearchSQLPlan(ctx context.Context, plan searchSQLPlan) ([]vectordata.SearchResult, error) { + rows, err := c.store.db.QueryContext(ctx, plan.query, plan.args...) + if err != nil { + return nil, err + } + defer rows.Close() + + results := make([]vectordata.SearchResult, 0) + for rows.Next() { + var rec vectordata.Record + var vectorRaw string + var metadataRaw string + var content sql.NullString + var distance float64 + + scanTargets := []any{&rec.ID} + if plan.projection.IncludeVector { + scanTargets = append(scanTargets, &vectorRaw) + } + if plan.projection.IncludeMetadata { + scanTargets = append(scanTargets, &metadataRaw) + } + if plan.projection.IncludeContent { + scanTargets = append(scanTargets, &content) + } + scanTargets = append(scanTargets, &distance) + + if err := rows.Scan(scanTargets...); err != nil { + return nil, err + } + + if plan.projection.IncludeVector { + parsedVector, err := parseVectorJSON(vectorRaw) + if err != nil { + return nil, fmt.Errorf("decode vector: %w", err) + } + rec.Vector = parsedVector + } + if plan.projection.IncludeMetadata { + parsedMetadata, err := parseMetadataJSON(metadataRaw) + if err != nil { + return nil, fmt.Errorf("decode metadata: %w", err) + } + rec.Metadata = parsedMetadata + } + if plan.projection.IncludeContent && content.Valid { + value := content.String + rec.Content = &value + } + + results = append(results, vectordata.SearchResult{ + Record: rec, + Distance: distance, + Score: vectordata.ScoreFromDistance(defaultMetric(c.metric), distance), + }) + } + if err := rows.Err(); err != nil { + return nil, err + } + + return results, nil +} + +func searchDistanceExpr(metric vectordata.DistanceMetric) (string, error) { + switch metric { + case vectordata.DistanceCosine: + return `CASE + WHEN vec_stats.query_norm_sq = 0 OR candidate_stats.candidate_norm_sq = 0 THEN 1.0 + ELSE 1.0 - (vec_stats.dot_product / (SQRT(vec_stats.query_norm_sq) * SQRT(candidate_stats.candidate_norm_sq))) + END`, nil + case vectordata.DistanceL2: + return "SQRT(vec_stats.sum_sq_diff)", nil + case vectordata.DistanceInnerProduct: + return "(-vec_stats.dot_product)", nil + default: + return "", fmt.Errorf("%w: unsupported distance metric %q", vectordata.ErrSchemaMismatch, metric) + } +} diff --git a/vectordata/filter_normalize.go b/vectordata/filter_normalize.go new file mode 100644 index 0000000..77bcb22 --- /dev/null +++ b/vectordata/filter_normalize.go @@ -0,0 +1,34 @@ +package vectordata + +import ( + "fmt" + "strings" +) + +// NormalizeFieldRef applies shared validation and canonicalization rules +// so every backend resolves fields the same way. +func NormalizeFieldRef(ref FieldRef) (FieldRef, error) { + switch ref.Kind { + case FieldColumn: + name := strings.TrimSpace(ref.Name) + if name == "" { + return FieldRef{}, fmt.Errorf("%w: column field name is empty", ErrInvalidFilter) + } + return FieldRef{Kind: FieldColumn, Name: name}, nil + case FieldMetadata: + if len(ref.Path) == 0 { + return FieldRef{}, fmt.Errorf("%w: metadata path is empty", ErrInvalidFilter) + } + path := make([]string, len(ref.Path)) + for i, segment := range ref.Path { + trimmed := strings.TrimSpace(segment) + if trimmed == "" { + return FieldRef{}, fmt.Errorf("%w: metadata path segment is empty", ErrInvalidFilter) + } + path[i] = trimmed + } + return FieldRef{Kind: FieldMetadata, Path: path}, nil + default: + return FieldRef{}, fmt.Errorf("%w: unsupported field kind %q", ErrInvalidFilter, ref.Kind) + } +} diff --git a/vectordata/filter_normalize_test.go b/vectordata/filter_normalize_test.go new file mode 100644 index 0000000..2fba44f --- /dev/null +++ b/vectordata/filter_normalize_test.go @@ -0,0 +1,20 @@ +package vectordata + +import "testing" + +func TestNormalizeFieldRef_MetadataTrimmed(t *testing.T) { + normalized, err := NormalizeFieldRef(Metadata(" flags ", " pinned ")) + if err != nil { + t.Fatalf("NormalizeFieldRef: %v", err) + } + if len(normalized.Path) != 2 || normalized.Path[0] != "flags" || normalized.Path[1] != "pinned" { + t.Fatalf("unexpected normalized path: %#v", normalized.Path) + } +} + +func TestNormalizeFieldRef_ColumnRejectsEmpty(t *testing.T) { + _, err := NormalizeFieldRef(Column(" ")) + if err == nil { + t.Fatal("expected error for empty column") + } +} diff --git a/vectordata/filter_sql.go b/vectordata/filter_sql.go index 815d22c..6dfa255 100644 --- a/vectordata/filter_sql.go +++ b/vectordata/filter_sql.go @@ -186,31 +186,30 @@ func (c *filterCompiler) compileLogical(op string, children []Filter) (string, e } func (c *filterCompiler) resolveField(ref FieldRef) (expr string, isMetadata bool, path []string, err error) { - switch ref.Kind { + normalized, err := NormalizeFieldRef(ref) + if err != nil { + return "", false, nil, err + } + + switch normalized.Kind { case FieldColumn: - if ref.Name == "" { - return "", false, nil, fmt.Errorf("%w: column field name is empty", ErrInvalidFilter) - } if c.cfg.ColumnExpr == nil { return "", false, nil, fmt.Errorf("%w: no column mapping configured", ErrInvalidFilter) } - expr, ok := c.cfg.ColumnExpr[ref.Name] + expr, ok := c.cfg.ColumnExpr[normalized.Name] if !ok || expr == "" { - return "", false, nil, fmt.Errorf("%w: unknown column %q", ErrInvalidFilter, ref.Name) + return "", false, nil, fmt.Errorf("%w: unknown column %q", ErrInvalidFilter, normalized.Name) } return expr, false, nil, nil case FieldMetadata: if c.cfg.MetadataExpr == "" { return "", false, nil, fmt.Errorf("%w: metadata expression not configured", ErrInvalidFilter) } - if len(ref.Path) == 0 { - return "", false, nil, fmt.Errorf("%w: metadata path is empty", ErrInvalidFilter) - } - cp := make([]string, len(ref.Path)) - copy(cp, ref.Path) + cp := make([]string, len(normalized.Path)) + copy(cp, normalized.Path) return c.cfg.MetadataExpr, true, cp, nil default: - return "", false, nil, fmt.Errorf("%w: unsupported field kind %q", ErrInvalidFilter, ref.Kind) + return "", false, nil, fmt.Errorf("%w: unsupported field kind %q", ErrInvalidFilter, normalized.Kind) } } diff --git a/vectordata/filter_sql_test.go b/vectordata/filter_sql_test.go index 8809363..d9c987b 100644 --- a/vectordata/filter_sql_test.go +++ b/vectordata/filter_sql_test.go @@ -154,3 +154,38 @@ func TestCompileFilterSQL_MetadataEqFilter(t *testing.T) { t.Fatalf("unexpected next arg index: %d", next) } } + +func TestCompileFilterSQL_TrimsColumnFieldName(t *testing.T) { + // Arrange + filter := Eq(Column(" id "), "r1") + + // Act + sql, args, _, err := CompileFilterSQL(filter, testFilterConfig(), 1) + + // Assert + if err != nil { + t.Fatalf("CompileFilterSQL error: %v", err) + } + if sql != `("id" = $1)` { + t.Fatalf("unexpected SQL: %s", sql) + } + if !reflect.DeepEqual(args, []any{"r1"}) { + t.Fatalf("unexpected args: %#v", args) + } +} + +func TestCompileFilterSQL_MetadataPathRejectsWhitespaceSegment(t *testing.T) { + // Arrange + filter := Eq(Metadata("rank", " "), 1) + + // Act + _, _, _, err := CompileFilterSQL(filter, testFilterConfig(), 1) + + // Assert + if err == nil { + t.Fatal("expected error") + } + if !errors.Is(err, ErrInvalidFilter) { + t.Fatalf("expected ErrInvalidFilter, got %v", err) + } +}