diff --git a/database/query/ddl/00017_events_lookup_gist_trgm.sql b/database/query/ddl/00017_events_lookup_gist_trgm.sql new file mode 100644 index 00000000..c95688dd --- /dev/null +++ b/database/query/ddl/00017_events_lookup_gist_trgm.sql @@ -0,0 +1,4 @@ +-- SPDX-License-Identifier: ice License 1.0 + +DROP INDEX IF EXISTS idx_events_lookup_trgm; +CREATE INDEX IF NOT EXISTS idx_events_lookup_gist_trgm ON events USING gist (lookup gist_trgm_ops); diff --git a/database/query/fuzzer/data_gen.go b/database/query/fuzzer/data_gen.go index 5feab965..fb92a204 100644 --- a/database/query/fuzzer/data_gen.go +++ b/database/query/fuzzer/data_gen.go @@ -121,6 +121,37 @@ func createPosts(ctx context.Context, keys []string, n int) []*model.Event { return posts } +func createFollowLists(ctx context.Context, keys []string) { + bar := progressbar.Default(int64(len(keys)), "creating follows lists") + defer bar.Finish() + + for _, key := range keys { + bar.Add(1) + numFollowers := rand.IntN(20) + 5 + var ev model.Event + ev.Kind = nostr.KindFollowList + ev.CreatedAt = model.Timestamp(time.Now().UnixNano()) + ev.Content = "" + selectedKeys := make(map[int]bool) + for range numFollowers { + idx := rand.IntN(len(keys)) + if selectedKeys[idx] { + continue + } + selectedKeys[idx] = true + + followerKey := keys[idx] + pubkey, err := model.GetPublicKey(followerKey) + if err != nil { + continue + } + ev.Tags = append(ev.Tags, model.Tag{"p", pubkey}) + } + panicOnErr(ev.SignWithAlg(key, model.SignAlgEDDSA, model.KeyAlgCurve25519)) + panicOnErr(query.AcceptEvents(ctx, &ev)) + } +} + func createPostsReactions(ctx context.Context, keys []string, posts []*model.Event, n int) { const ( reactionLike = iota diff --git a/database/query/fuzzer/main.go b/database/query/fuzzer/main.go index 1803215d..377bf90d 100644 --- a/database/query/fuzzer/main.go +++ b/database/query/fuzzer/main.go @@ -100,7 +100,13 @@ var ( "kind0>kind6400+kind3+group+p", "kind0>kind6400+kind30175+expiration", } - allSearchExtensions = []string{ + + kind0SearchExtensions = []string{ + "keyword_lookup_strategy:prefix", "keyword_lookup_strategy:infix", + "followed_by:{{.Pubkey}}", "follower_of:{{.Pubkey}}", + } + + otherKindsSearchExtensions = []string{ "media:true", "media:false", "quotes:true", "quotes:false", "references:true", "references:false", @@ -108,7 +114,7 @@ var ( "images:true", "images:false", "expiration:true", "expiration:false", } - // allSearchExtensions + allSearchDependencies = 95,550,759 combinations. + // allSearchExtensions + allSearchDependencies = = 96,075,043 combinations. ) func newContext() context.Context { @@ -256,6 +262,7 @@ func main() { Int("reactions", *reactionsCount). Msg("creating dummy data ...") keys := createUsers(ctx, *usersCount) + createFollowLists(ctx, keys) posts := createPosts(ctx, keys, *postsCount) createPostsReactions(ctx, keys, posts, *reactionsCount) log.Info().Msg("dummy data created") @@ -287,19 +294,31 @@ func main() { } log.Info().Msg("preparing combinations ...") + templateData := map[string]string{ + "Pubkey": currentPublicKey, + } + for i := range allSearchDependencies { var buf bytes.Buffer buf.WriteString(`include:dependencies:`) tpl := template.Must(template.New(strconv.Itoa(i)).Parse(allSearchDependencies[i])) - tpl.Execute(&buf, map[string]string{ - "Pubkey": currentPublicKey, - }) + tpl.Execute(&buf, templateData) allSearchDependencies[i] = buf.String() log.Info().Msgf("search dependency %d: %s", i, allSearchDependencies[i]) } - iterator, err := NewSearchCombinationsIterator(allSearchDependencies, allSearchExtensions, *positionToUse) + allExts := append(append([]string{}, otherKindsSearchExtensions...), kind0SearchExtensions...) + for i := range allExts { + if strings.Contains(allExts[i], "{{") { + var buf bytes.Buffer + tpl := template.Must(template.New("ext_" + strconv.Itoa(i)).Parse(allExts[i])) + tpl.Execute(&buf, templateData) + allExts[i] = buf.String() + } + } + + iterator, err := NewSearchCombinationsIterator(allSearchDependencies, allExts, *positionToUse) if err != nil { log.Fatal().Err(err).Msg("failed to create combinations iterator") } @@ -345,12 +364,30 @@ func main() { i := iterator.Position() - 1 + hasTextSearch := false + for _, ext := range combination { + if strings.Contains(ext, "keyword_lookup_strategy") || + strings.Contains(ext, "followed_by") || + strings.Contains(ext, "follower_of") { + hasTextSearch = true + + break + } + } + + var kinds []int + if hasTextSearch { + kinds = []int{nostr.KindProfileMetadata} + } else { + kinds = []int{model.CustomIONKindEditableTextNote, nostr.KindArticle, nostr.KindGenericRepost} + } + for ctx.Err() == nil { err = pool.Submit(func() { defer bar.Add(1) filter := model.Filter{ - Kinds: []int{model.CustomIONKindEditableTextNote, nostr.KindArticle, nostr.KindGenericRepost}, + Kinds: kinds, Search: strings.Join(combination, " "), } diff --git a/database/query/query_text_search_test.go b/database/query/query_text_search_test.go index 00c8cc30..8c9214de 100644 --- a/database/query/query_text_search_test.go +++ b/database/query/query_text_search_test.go @@ -13,6 +13,7 @@ import ( "github.com/nbd-wtf/go-nostr" "github.com/stretchr/testify/require" + "github.com/ice-blockchain/subzero/database/query/internal/connector" "github.com/ice-blockchain/subzero/model" ) @@ -1185,3 +1186,359 @@ func TestPrepareSearchContentRemoveEmojis(t *testing.T) { }) } } + +func TestSearchExtensions_StartsWithAndContains(t *testing.T) { + t.Parallel() + db := helperNewDatabase(t) + defer db.Close() + profiles := []struct { + name string + displayName string + masterKey string + privateKey string + }{ + {"alice", "Alice Smith", "", ""}, + {"alison", "Alison Cooper", "", ""}, + {"alexander", "Alexander Great", "", ""}, + {"lexandra", "Lexandra Wilson", "", ""}, + {"lexander", "Lexander Brown", "", ""}, + {"allison", "Allison Taylor", "", ""}, + {"bob", "Bob Jones", "", ""}, + {"bobby", "Bobby Wilson", "", ""}, + {"charlie", "Charlie Brown", "", ""}, + {"madison", "Madison Lee", "", ""}, + {"ellison", "Ellison Ford", "", ""}, + } + + for i := range profiles { + profiles[i].privateKey, profiles[i].masterKey = model.GenerateKeyPair() + + content := `{"name":"` + profiles[i].name + `","display_name":"` + profiles[i].displayName + `"}` + profileEvent := &model.Event{ + Event: nostr.Event{ + Kind: nostr.KindProfileMetadata, + CreatedAt: nostr.Now(), + Content: content, + }, + } + require.NoError(t, profileEvent.SignWithAlg(profiles[i].privateKey, model.SignAlgEDDSA, model.KeyAlgCurve25519)) + require.NoError(t, db.AcceptEvents(t.Context(), profileEvent)) + } + + t.Run("StartsWith search - find alice, alison, alexander", func(t *testing.T) { + stored := helperSelectEvents(t, db, model.Filter{ + Kinds: []int{nostr.KindProfileMetadata}, + Search: `"ali" keyword_lookup_strategy:prefix`, + }) + require.Equal(t, len(stored), 2) + names := make(map[string]bool) + for _, ev := range stored { + var content struct { + Name string `json:"name"` + } + require.NoError(t, json.Unmarshal([]byte(ev.Content), &content)) + names[content.Name] = true + } + require.True(t, names["alice"] || names["alison"], "Should find alice/alison") + }) + t.Run("StartsWith search - find bob and bobby", func(t *testing.T) { + stored := helperSelectEvents(t, db, model.Filter{ + Kinds: []int{nostr.KindProfileMetadata}, + Search: `"bob" keyword_lookup_strategy:prefix`, + }) + require.Equal(t, 2, len(stored), "Should find 2 results starting with 'bob'") + foundNames := make(map[string]bool) + for _, ev := range stored { + var content struct { + Name string `json:"name"` + } + require.NoError(t, json.Unmarshal([]byte(ev.Content), &content)) + foundNames[content.Name] = true + } + require.True(t, foundNames["bob"], "Should find bob") + require.True(t, foundNames["bobby"], "Should find bobby") + }) + + t.Run("Contains search - find 'lexand' inside multiple names", func(t *testing.T) { + stored := helperSelectEvents(t, db, model.Filter{ + Kinds: []int{nostr.KindProfileMetadata}, + Search: `"lexand" keyword_lookup_strategy:infix`, + }) + require.Equal(t, 3, len(stored), "Should find exactly 3 results with 'lexand' inside") + foundNames := make(map[string]bool) + for _, ev := range stored { + var content struct { + Name string `json:"name"` + } + json.Unmarshal([]byte(ev.Content), &content) + foundNames[content.Name] = true + } + require.True(t, foundNames["alexander"], "Should find alexander (contains 'lexand')") + require.True(t, foundNames["lexandra"], "Should find lexandra (contains 'lexand')") + require.True(t, foundNames["lexander"], "Should find lexander (contains 'lexand')") + }) + + t.Run("Contains search - find 'lison' inside multiple names", func(t *testing.T) { + stored := helperSelectEvents(t, db, model.Filter{ + Kinds: []int{nostr.KindProfileMetadata}, + Search: `"lison" keyword_lookup_strategy:infix`, + }) + require.Equal(t, 3, len(stored), "Should find exactly 3 results with 'lison'") + foundNames := make(map[string]bool) + for _, ev := range stored { + var content struct { + Name string `json:"name"` + } + json.Unmarshal([]byte(ev.Content), &content) + foundNames[content.Name] = true + } + require.True(t, foundNames["alison"], "Should find alison (contains 'lison')") + require.True(t, foundNames["allison"], "Should find allison (contains 'lison')") + require.True(t, foundNames["ellison"], "Should find ellison (contains 'lison')") + }) +} + +func TestSearchExtensions_FollowedByAndFollowerOf(t *testing.T) { + t.Parallel() + + db := helperNewDatabase(t) + defer db.Close() + + users := []struct { + name string + masterKey string + privateKey string + verified bool + }{ + {"alice", "", "", false}, + {"bob", "", "", true}, + {"charlie", "", "", false}, + {"david", "", "", false}, + {"eve", "", "", true}, + {"frank", "", "", false}, + {"bobby", "", "", false}, + {"robert", "", "", true}, + {"alicia", "", "", false}, + {"alison", "", "", false}, + {"alexander", "", "", true}, + {"carol", "", "", false}, + {"catherine", "", "", false}, + {"chris", "", "", true}, + {"dan", "", "", false}, + {"diana", "", "", false}, + {"emily", "", "", true}, + {"ethan", "", "", false}, + {"fiona", "", "", false}, + {"fred", "", "", false}, + } + + for i := range users { + users[i].privateKey, users[i].masterKey = model.GenerateKeyPair() + content := `{"name":"` + users[i].name + `","display_name":"` + users[i].name + `"}` + profileEvent := &model.Event{ + Event: nostr.Event{ + Kind: nostr.KindProfileMetadata, + CreatedAt: nostr.Now(), + Content: content, + }, + } + if users[i].verified { + profileEvent.Event.Tags = model.Tags{{"verified", "true"}} + } + require.NoError(t, profileEvent.SignWithAlg(users[i].privateKey, model.SignAlgEDDSA, model.KeyAlgCurve25519)) + require.NoError(t, db.AcceptEvents(t.Context(), profileEvent)) + + if users[i].verified { + _, err := connector.Exec(t.Context(), db.db, `UPDATE events SET verified = true WHERE master_pubkey = $1 AND kind = 0`, users[i].masterKey) + require.NoError(t, err) + } + } + + // Alice follows: bob, charlie, bobby, robert, alicia, alison, alexander, carol, chris, dan, emily + aliceFollowList := &model.Event{ + Event: nostr.Event{ + Kind: nostr.KindFollowList, + CreatedAt: nostr.Now(), + Tags: model.Tags{ + {"p", users[1].masterKey}, // bob (verified) + {"p", users[2].masterKey}, // charlie + {"p", users[6].masterKey}, // bobby + {"p", users[7].masterKey}, // robert (verified) + {"p", users[8].masterKey}, // alicia + {"p", users[9].masterKey}, // alison + {"p", users[10].masterKey}, // alexander (verified) + {"p", users[11].masterKey}, // carol + {"p", users[13].masterKey}, // chris (verified) + {"p", users[14].masterKey}, // dan + {"p", users[16].masterKey}, // emily (verified) + }, + }, + } + require.NoError(t, aliceFollowList.SignWithAlg(users[0].privateKey, model.SignAlgEDDSA, model.KeyAlgCurve25519)) + require.NoError(t, db.AcceptEvents(t.Context(), aliceFollowList)) + + // Bob (verified) follows: alice, david, eve, frank, alicia, catherine, diana, ethan, fiona + bobFollowList := &model.Event{ + Event: nostr.Event{ + Kind: nostr.KindFollowList, + CreatedAt: nostr.Now(), + Tags: model.Tags{ + {"p", users[0].masterKey}, // alice + {"p", users[3].masterKey}, // david + {"p", users[4].masterKey}, // eve (verified) + {"p", users[5].masterKey}, // frank + {"p", users[8].masterKey}, // alicia + {"p", users[12].masterKey}, // catherine + {"p", users[15].masterKey}, // diana + {"p", users[17].masterKey}, // ethan + {"p", users[18].masterKey}, // fiona + }, + }, + } + require.NoError(t, bobFollowList.SignWithAlg(users[1].privateKey, model.SignAlgEDDSA, model.KeyAlgCurve25519)) + require.NoError(t, db.AcceptEvents(t.Context(), bobFollowList)) + + // Charlie follows: alice, bob, bobby, robert, alicia, alison, chris, carol + charlieFollowList := &model.Event{ + Event: nostr.Event{ + Kind: nostr.KindFollowList, + CreatedAt: nostr.Now(), + Tags: model.Tags{ + {"p", users[0].masterKey}, // alice + {"p", users[1].masterKey}, // bob (verified) + {"p", users[6].masterKey}, // bobby + {"p", users[7].masterKey}, // robert (verified) + {"p", users[8].masterKey}, // alicia + {"p", users[9].masterKey}, // alison + {"p", users[11].masterKey}, // carol + {"p", users[13].masterKey}, // chris (verified) + }, + }, + } + require.NoError(t, charlieFollowList.SignWithAlg(users[2].privateKey, model.SignAlgEDDSA, model.KeyAlgCurve25519)) + require.NoError(t, db.AcceptEvents(t.Context(), charlieFollowList)) + + // David follows: charlie, catherine, carol, chris, dan, diana + davidFollowList := &model.Event{ + Event: nostr.Event{ + Kind: nostr.KindFollowList, + CreatedAt: nostr.Now(), + Tags: model.Tags{ + {"p", users[2].masterKey}, // charlie + {"p", users[11].masterKey}, // carol + {"p", users[12].masterKey}, // catherine + {"p", users[13].masterKey}, // chris (verified) + {"p", users[14].masterKey}, // dan + {"p", users[15].masterKey}, // diana + }, + }, + } + require.NoError(t, davidFollowList.SignWithAlg(users[3].privateKey, model.SignAlgEDDSA, model.KeyAlgCurve25519)) + require.NoError(t, db.AcceptEvents(t.Context(), davidFollowList)) + + // Eve (verified) follows: emily, ethan, eve (self-follow for testing), alexander, alicia + eveFollowList := &model.Event{ + Event: nostr.Event{ + Kind: nostr.KindFollowList, + CreatedAt: nostr.Now(), + Tags: model.Tags{ + {"p", users[8].masterKey}, // alicia + {"p", users[10].masterKey}, // alexander (verified) + {"p", users[16].masterKey}, // emily (verified) + {"p", users[17].masterKey}, // ethan + }, + }, + } + require.NoError(t, eveFollowList.SignWithAlg(users[4].privateKey, model.SignAlgEDDSA, model.KeyAlgCurve25519)) + require.NoError(t, db.AcceptEvents(t.Context(), eveFollowList)) + + t.Run("FollowedBy - search 'bob' - multiple results (bob, bobby)", func(t *testing.T) { + // Alice follows: bob, bobby, robert. Expected: 2 results - bob (verified), bobby (non-verified) + stored := helperSelectEvents(t, db, model.Filter{ + Kinds: []int{nostr.KindProfileMetadata}, + Search: `"bob" followed_by:` + users[0].masterKey, + }) + require.Equal(t, 2, len(stored), "Should find bob and bobby") + require.Equal(t, users[1].masterKey, stored[0].GetMasterPublicKey(), "Position 0 should be bob (verified)") + require.Equal(t, users[6].masterKey, stored[1].GetMasterPublicKey(), "Position 1 should be bobby (non-verified)") + }) + + t.Run("FollowedBy - multiple authors [alice, charlie] search 'bob' - UNION", func(t *testing.T) { + // Alice follows: bob, bobby, robert, charlie + // Charlie follows: bob, bobby, robert, alice, alicia, alison, carol, chris + stored := helperSelectEvents(t, db, model.Filter{ + Kinds: []int{nostr.KindProfileMetadata}, + Search: `"bob" followed_by:` + users[0].masterKey + `,` + users[2].masterKey, + }) + require.Equal(t, 4, len(stored), "Should find 4 results matching 'bob'") + require.Equal(t, users[1].masterKey, stored[0].GetMasterPublicKey(), "Position 0 should be bob (verified, exact match)") + found := make(map[string]bool) + for _, ev := range stored { + found[ev.GetMasterPublicKey()] = true + } + require.True(t, found[users[6].masterKey], "Should find bobby") + }) + + t.Run("FollowedBy - search 'ali' contains - multiple results", func(t *testing.T) { + // Alice follows: alicia, alison + // Search for profiles containing "ali" among alice's followings + stored := helperSelectEvents(t, db, model.Filter{ + Kinds: []int{nostr.KindProfileMetadata}, + Search: `"ali" keyword_lookup_strategy:infix followed_by:` + users[0].masterKey, + }) + require.Equal(t, 2, len(stored), "Should find alicia and alison") + found := make(map[string]bool) + for _, ev := range stored { + found[ev.GetMasterPublicKey()] = true + } + require.True(t, found[users[8].masterKey], "Should find alicia") + require.True(t, found[users[9].masterKey], "Should find alison") + }) + + t.Run("FollowedBy - multiple authors [alice, bob, charlie] search 'ali' - UNION", func(t *testing.T) { + // Alice follows: alicia, alison, alexander + // Bob follows: alicia, alice, david, eve, frank, catherine, diana, ethan, fiona + // Charlie follows: alicia, alison, bob, bobby, robert, carol, chris, alice + stored := helperSelectEvents(t, db, model.Filter{ + Kinds: []int{nostr.KindProfileMetadata}, + Search: `"ali" keyword_lookup_strategy:infix followed_by:` + users[0].masterKey + `,` + users[1].masterKey + `,` + users[2].masterKey, + }) + require.Equal(t, 7, len(stored), "Should find 7 results matching 'ali'") + found := make(map[string]bool) + for _, ev := range stored { + found[ev.GetMasterPublicKey()] = true + } + require.True(t, found[users[0].masterKey], "Should find alice") + require.True(t, found[users[8].masterKey], "Should find alicia") + require.True(t, found[users[9].masterKey], "Should find alison") + }) + + t.Run("FollowerOf - search 'd' with startsWith - david follows charlie", func(t *testing.T) { + // Charlie is followed by: nobody with name starting with "d" + // David follows: charlie. + stored := helperSelectEvents(t, db, model.Filter{ + Kinds: []int{nostr.KindProfileMetadata}, + Search: `"d" keyword_lookup_strategy:prefix follower_of:` + users[2].masterKey, + }) + for _, ev := range stored { + require.NotEqual(t, users[3].masterKey, ev.GetMasterPublicKey(), + "Should NOT find david (david doesn't follow charlie)") + } + }) + + t.Run("FollowedBy - multiple authors [alice, eve] search 'em'", func(t *testing.T) { + // Alice follows: emily + // Eve follows: emily, ethan, alicia, alexander + stored := helperSelectEvents(t, db, model.Filter{ + Kinds: []int{nostr.KindProfileMetadata}, + Search: `"em" keyword_lookup_strategy:infix followed_by:` + users[0].masterKey + `,` + users[4].masterKey, + }) + + require.Equal(t, 2, len(stored), "Should find 2 results with 'em'") + found := make(map[string]bool) + for _, ev := range stored { + found[ev.GetMasterPublicKey()] = true + } + require.True(t, found[users[16].masterKey], "Should find emily (verified)") + }) +} diff --git a/database/query/query_where_builder.go b/database/query/query_where_builder.go index 765c58ca..1e5b41de 100644 --- a/database/query/query_where_builder.go +++ b/database/query/query_where_builder.go @@ -69,15 +69,19 @@ type ( Leafs map[string]*filterDependency // Origin/dependency name -> dependency. } databaseFilterSearch struct { - Expiration *bool - Videos *bool - Images *bool - Media *bool - Quotes *bool - References *bool - CurrentUserPubkey *string - ID string - SearchText string + Expiration *bool + Videos *bool + Images *bool + Media *bool + Quotes *bool + References *bool + CurrentUserPubkey *string + ID string + SearchText string + SearchType string + FollowedBy bool + FollowerOf bool + SocialFilterPubkeys []string model.Filter TagMarkers []databaseFilterMarker Dependencies []*filterDependency @@ -624,7 +628,15 @@ func replaceSpecialChars(input string) string { }, input) } +func isKindProfileMetadataOnly(filter *databaseFilterSearch) bool { + return len(filter.Kinds) == 1 && filter.Kinds[0] == nostr.KindProfileMetadata +} + func (b *queryBuilder) ApplyTextSearch(filter *databaseFilterSearch) { + if isKindProfileMetadataOnly(filter) { + return + } + if filter.SearchText == "" { return } @@ -1397,7 +1409,132 @@ func shouldApplyVerifiedFirst(filter *databaseFilterSearch) bool { return filter.Limit == 1 && len(filter.Kinds) > 0 && len(filter.Authors) == 0 && len(filter.IDs) == 0 } +func (b *queryBuilder) buildSimilaritySQL(sourceAlias string, filter *databaseFilterSearch, text string) string { + var sb strings.Builder + sb.WriteString(`GREATEST(similarity(`) + sb.WriteString(sourceAlias) + sb.WriteString(`.lookup, :`) + sb.WriteString(b.PushValue(filter.ID, "search_text", text)) + sb.WriteString(`), word_similarity(:`) + sb.WriteString(b.PushValue(filter.ID, "search_text_word", text)) + sb.WriteString(`, `) + sb.WriteString(sourceAlias) + sb.WriteString(`.lookup)) + (CASE WHEN `) + sb.WriteString(sourceAlias) + sb.WriteString(`.lookup LIKE :`) + sb.WriteString(b.PushValue(filter.ID, "search_prefix_word", text+" %")) + sb.WriteString(` THEN 1.0 WHEN `) + sb.WriteString(sourceAlias) + sb.WriteString(`.lookup LIKE :`) + sb.WriteString(b.PushValue(filter.ID, "search_prefix", text+"%")) + sb.WriteString(` THEN 0.5 ELSE 0.0 END)`) + + return sb.String() +} + +func (b *queryBuilder) buildFollowedByFollowerOfJoins(filter *databaseFilterSearch) string { + if !filter.FollowedBy && !filter.FollowerOf { + return "" + } + + tempBuilder := queryBuilder{Params: b.Params} + if filter.FollowedBy { + tempBuilder.WriteString(` + INNER JOIN event_tags et ON et.event_tag_key = 'p' AND et.event_tag_value1 = e.master_pubkey + INNER JOIN events e_follow ON e_follow.id = et.event_id + AND e_follow.kind = 3 + AND e_follow.hidden = false + AND `) + buildFromSlice(&tempBuilder, sqlOpCodeNONE, filter.ID, filter.SocialFilterPubkeys, "e_follow.master_pubkey", "followed_by") + } else if filter.FollowerOf { + tempBuilder.WriteString(` + INNER JOIN events e_follow ON e_follow.master_pubkey = e.master_pubkey + AND e_follow.kind = 3 + AND e_follow.hidden = false + INNER JOIN event_tags et ON et.event_id = e_follow.id + AND et.event_tag_key = 'p' + AND `) + buildFromSlice(&tempBuilder, sqlOpCodeNONE, filter.ID, filter.SocialFilterPubkeys, "et.event_tag_value1", "follower_of") + } + return tempBuilder.String() +} + +func (b *queryBuilder) BuildCTEWithGiSTKNN(filter *databaseFilterSearch) (*databaseCTE, error) { + if filter.Limit == 0 { + filter.Limit = whereBuilderDefaultLimit + } + + whereBuffer := queryBuilder{Params: b.Params} + where, _, err := whereBuffer.BuildWhere(filter) + if err != nil { + return nil, err + } + + text := strings.ToLower(replaceSpecialChars(filter.SearchText)) + name := filter.ID + "events_cte" + fields := b.fieldsNames("c", name) + fields = append(fields, "c.sim", "c.verified") + + orderBy := `verified desc, sim desc, ` + whereBuilderDefaultOrderBy + + var sb strings.Builder + + sb.WriteString(`(WITH candidates AS ( + SELECT e.*, `) + sb.WriteString(b.buildSimilaritySQL("e", filter, text)) + sb.WriteString(` AS sim + FROM events e`) + + sb.WriteString(b.buildFollowedByFollowerOfJoins(filter)) + + sb.WriteString(` + WHERE `) + sb.WriteString(where) + sb.WriteString(` AND e.lookup LIKE :`) + + if filter.SearchType == keywordLookupStrategyPrefix { + sb.WriteString(b.PushValue(filter.ID, "search_lookup_like", text+"%")) + } else { + sb.WriteString(b.PushValue(filter.ID, "search_lookup_like", "%"+text+"%")) + } + + sb.WriteString(` AND similarity(e.lookup, :`) + sb.WriteString(b.PushValue(filter.ID, "search_similarity_check", text)) + sb.WriteString(`) >= 0.2 + ORDER BY e.lookup <-> :`) + sb.WriteString(b.PushValue(filter.ID, "search_knn", text)) + sb.WriteString(` + LIMIT 250 + ) + SELECT `) + + for i, f := range fields { + if i > 0 { + sb.WriteString(", ") + } + sb.WriteString(f) + } + sb.WriteString(` FROM candidates c + ORDER BY `) + sb.WriteString(orderBy) + sb.WriteString(` + LIMIT :`) + sb.WriteString(b.PushValue(filter.ID, "limit", filter.Limit)) + sb.WriteString(`)`) + + return &databaseCTE{ + Name: name, + Body: sb.String(), + OrderBy: orderBy, + Filter: filter, + }, nil +} + func (b *queryBuilder) BuildCTE(filter *databaseFilterSearch) (cte *databaseCTE, err error) { + if isKindProfileMetadataOnly(filter) && (filter.SearchText != "" || filter.FollowedBy || filter.FollowerOf) { + return b.BuildCTEWithGiSTKNN(filter) + } + whereBuffer := queryBuilder{Params: b.Params} where, _, err := whereBuffer.BuildWhere(filter) if err != nil { diff --git a/database/query/query_where_builder_filter.go b/database/query/query_where_builder_filter.go index 469f6b6f..83f53958 100644 --- a/database/query/query_where_builder_filter.go +++ b/database/query/query_where_builder_filter.go @@ -9,6 +9,13 @@ import ( "github.com/ice-blockchain/subzero/model" ) +const ( + keywordLookupStrategyPrefix = "prefix" + keywordLookupStrategyInfix = "infix" + searchExtensionFollowedBy = "followed_by" + searchExtensionFollowerOf = "follower_of" +) + func parseNostrFilterFlags(f *databaseFilterSearch) *databaseFilterSearch { flags := []struct { Name string @@ -118,6 +125,51 @@ func parseNostrFilterDependencies(f *databaseFilterSearch) (*databaseFilterSearc } func parseNostrFilterText(f *databaseFilterSearch) *databaseFilterSearch { + if idx := strings.Index(f.Search, "keyword_lookup_strategy:"); idx != -1 { + start := idx + len("keyword_lookup_strategy:") + end := strings.Index(f.Search[start:], " ") + if end == -1 { + end = len(f.Search) - start + } + strategy := strings.TrimSpace(f.Search[start : start+end]) + if strategy == keywordLookupStrategyPrefix || strategy == keywordLookupStrategyInfix { + f.SearchType = strategy + } + f.Search = strings.TrimSpace(f.Search[:idx] + f.Search[start+end:]) + } + + if idx := strings.Index(f.Search, searchExtensionFollowedBy+":"); idx != -1 { + f.FollowedBy = true + start := idx + len(searchExtensionFollowedBy) + 1 + end := strings.Index(f.Search[start:], " ") + if end == -1 { + end = len(f.Search) - start + } + pubkeys := strings.TrimSpace(f.Search[start : start+end]) + for _, pk := range strings.Split(pubkeys, ",") { + if pk = strings.TrimSpace(pk); pk != "" { + f.SocialFilterPubkeys = append(f.SocialFilterPubkeys, pk) + } + } + f.Search = strings.TrimSpace(f.Search[:idx] + f.Search[start+end:]) + } + + if idx := strings.Index(f.Search, searchExtensionFollowerOf+":"); idx != -1 { + f.FollowerOf = true + start := idx + len(searchExtensionFollowerOf) + 1 + end := strings.Index(f.Search[start:], " ") + if end == -1 { + end = len(f.Search) - start + } + pubkeys := strings.TrimSpace(f.Search[start : start+end]) + for _, pk := range strings.Split(pubkeys, ",") { + if pk = strings.TrimSpace(pk); pk != "" { + f.SocialFilterPubkeys = append(f.SocialFilterPubkeys, pk) + } + } + f.Search = strings.TrimSpace(f.Search[:idx] + f.Search[start+end:]) + } + quoteStart := strings.Index(f.Search, "\"") quoteEnd := strings.LastIndex(f.Search, "\"") diff --git a/go.mod b/go.mod index 78871b84..a8a4c74c 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/ice-blockchain/subzero -go 1.25.3 +go 1.25.4 replace ( github.com/dgraph-io/ristretto => github.com/dgraph-io/ristretto v0.1.0 @@ -65,8 +65,8 @@ require ( github.com/spf13/viper v1.21.0 github.com/stretchr/testify v1.11.1 github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d - github.com/testcontainers/testcontainers-go v0.39.0 - github.com/testcontainers/testcontainers-go/modules/postgres v0.39.0 + github.com/testcontainers/testcontainers-go v0.40.0 + github.com/testcontainers/testcontainers-go/modules/postgres v0.40.0 github.com/tidwall/gjson v1.18.0 github.com/tidwall/wal v1.2.1 github.com/tus/tusd/v2 v2.8.0 @@ -199,7 +199,7 @@ require ( github.com/google/renameio/v2 v2.0.0 // indirect github.com/google/s2a-go v0.1.9 // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect - github.com/googleapis/enterprise-certificate-proxy v0.3.6 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.7 // indirect github.com/googleapis/gax-go/v2 v2.15.0 // indirect github.com/gookit/color v1.6.0 // indirect github.com/gorilla/css v1.0.1 // indirect diff --git a/go.sum b/go.sum index 1c1cdf37..0dda0e6d 100644 --- a/go.sum +++ b/go.sum @@ -430,8 +430,8 @@ github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/enterprise-certificate-proxy v0.3.6 h1:GW/XbdyBFQ8Qe+YAmFU9uHLo7OnF5tL52HFAgMmyrf4= -github.com/googleapis/enterprise-certificate-proxy v0.3.6/go.mod h1:MkHOF77EYAE7qfSuSS9PU6g4Nt4e11cnsDUowfwewLA= +github.com/googleapis/enterprise-certificate-proxy v0.3.7 h1:zrn2Ee/nWmHulBx5sAVrGgAa0f2/R35S4DJwfFaUPFQ= +github.com/googleapis/enterprise-certificate-proxy v0.3.7/go.mod h1:MkHOF77EYAE7qfSuSS9PU6g4Nt4e11cnsDUowfwewLA= github.com/googleapis/gax-go/v2 v2.15.0 h1:SyjDc1mGgZU5LncH8gimWo9lW1DtIfPibOG81vgd/bo= github.com/googleapis/gax-go/v2 v2.15.0/go.mod h1:zVVkkxAQHa1RQpg9z2AUCMnKhi0Qld9rcmyfL1OZhoc= github.com/gookit/assert v0.1.1 h1:lh3GcawXe/p+cU7ESTZ5Ui3Sm/x8JWpIis4/1aF0mY0= @@ -803,10 +803,10 @@ github.com/supranational/blst v0.3.16/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3 github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d h1:vfofYNRScrDdvS342BElfbETmL1Aiz3i2t0zfRj16Hs= github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d/go.mod h1:RRCYJbIwD5jmqPI9XoAFR0OcDxqUctll6zUj/+B4S48= -github.com/testcontainers/testcontainers-go v0.39.0 h1:uCUJ5tA+fcxbFAB0uP3pIK3EJ2IjjDUHFSZ1H1UxAts= -github.com/testcontainers/testcontainers-go v0.39.0/go.mod h1:qmHpkG7H5uPf/EvOORKvS6EuDkBUPE3zpVGaH9NL7f8= -github.com/testcontainers/testcontainers-go/modules/postgres v0.39.0 h1:REJz+XwNpGC/dCgTfYvM4SKqobNqDBfvhq74s2oHTUM= -github.com/testcontainers/testcontainers-go/modules/postgres v0.39.0/go.mod h1:4K2OhtHEeT+JSIFX4V8DkGKsyLa96Y2vLdd3xsxD5HE= +github.com/testcontainers/testcontainers-go v0.40.0 h1:pSdJYLOVgLE8YdUY2FHQ1Fxu+aMnb6JfVz1mxk7OeMU= +github.com/testcontainers/testcontainers-go v0.40.0/go.mod h1:FSXV5KQtX2HAMlm7U3APNyLkkap35zNLxukw9oBi/MY= +github.com/testcontainers/testcontainers-go/modules/postgres v0.40.0 h1:s2bIayFXlbDFexo96y+htn7FzuhpXLYJNnIuglNKqOk= +github.com/testcontainers/testcontainers-go/modules/postgres v0.40.0/go.mod h1:h+u/2KoREGTnTl9UwrQ/g+XhasAT8E6dClclAADeXoQ= github.com/tidwall/gjson v1.10.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY=