Search enhancement

database/query/ddl/00017_events_lookup_gist_trgm.sql

@@ -0,0 +1,4 @@
+-- SPDX-License-Identifier: ice License 1.0
+
+DROP INDEX IF EXISTS idx_events_lookup_trgm;
+CREATE INDEX IF NOT EXISTS idx_events_lookup_gist_trgm ON events USING gist (lookup gist_trgm_ops);

database/query/fuzzer/data_gen.go

@@ -121,6 +121,37 @@ func createPosts(ctx context.Context, keys []string, n int) []*model.Event {
 	return posts
 }
 
+func createFollowLists(ctx context.Context, keys []string) {
+	bar := progressbar.Default(int64(len(keys)), "creating follows lists")
+	defer bar.Finish()
+
+	for _, key := range keys {
+		bar.Add(1)
+		numFollowers := rand.IntN(20) + 5
+		var ev model.Event
+		ev.Kind = nostr.KindFollowList
+		ev.CreatedAt = model.Timestamp(time.Now().UnixNano())
+		ev.Content = ""
+		selectedKeys := make(map[int]bool)
+		for range numFollowers {
+			idx := rand.IntN(len(keys))
+			if selectedKeys[idx] {
+				continue
+			}
+			selectedKeys[idx] = true
+
+			followerKey := keys[idx]
+			pubkey, err := model.GetPublicKey(followerKey)
+			if err != nil {
+				continue
+			}
+			ev.Tags = append(ev.Tags, model.Tag{"p", pubkey})
+		}
+		panicOnErr(ev.SignWithAlg(key, model.SignAlgEDDSA, model.KeyAlgCurve25519))
+		panicOnErr(query.AcceptEvents(ctx, &ev))
+	}
+}
+
 func createPostsReactions(ctx context.Context, keys []string, posts []*model.Event, n int) {
 	const (
 		reactionLike = iota

database/query/fuzzer/main.go

@@ -100,15 +100,21 @@ var (
 		"kind0>kind6400+kind3+group+p",
 		"kind0>kind6400+kind30175+expiration",
 	}
-	allSearchExtensions = []string{
+
+	kind0SearchExtensions = []string{
+		"keyword_lookup_strategy:prefix", "keyword_lookup_strategy:infix",
+		"followed_by:{{.Pubkey}}", "follower_of:{{.Pubkey}}",
+	}
+
+	otherKindsSearchExtensions = []string{
 		"media:true", "media:false",
 		"quotes:true", "quotes:false",
 		"references:true", "references:false",
 		"videos:true", "videos:false",
 		"images:true", "images:false",
 		"expiration:true", "expiration:false",
 	}
-	// allSearchExtensions + allSearchDependencies = 95,550,759 combinations.
+	// allSearchExtensions + allSearchDependencies = = 96,075,043 combinations.
 )
 
 func newContext() context.Context {
@@ -256,6 +262,7 @@ func main() {
 			Int("reactions", *reactionsCount).
 			Msg("creating dummy data ...")
 		keys := createUsers(ctx, *usersCount)
+		createFollowLists(ctx, keys)
 		posts := createPosts(ctx, keys, *postsCount)
 		createPostsReactions(ctx, keys, posts, *reactionsCount)
 		log.Info().Msg("dummy data created")
@@ -287,19 +294,31 @@ func main() {
 	}
 
 	log.Info().Msg("preparing combinations ...")
+	templateData := map[string]string{
+		"Pubkey": currentPublicKey,
+	}
+
 	for i := range allSearchDependencies {
 		var buf bytes.Buffer
 
 		buf.WriteString(`include:dependencies:`)
 		tpl := template.Must(template.New(strconv.Itoa(i)).Parse(allSearchDependencies[i]))
-		tpl.Execute(&buf, map[string]string{
-			"Pubkey": currentPublicKey,
-		})
+		tpl.Execute(&buf, templateData)
 		allSearchDependencies[i] = buf.String()
 		log.Info().Msgf("search dependency %d: %s", i, allSearchDependencies[i])
 	}
 
-	iterator, err := NewSearchCombinationsIterator(allSearchDependencies, allSearchExtensions, *positionToUse)
+	allExts := append(append([]string{}, otherKindsSearchExtensions...), kind0SearchExtensions...)
+	for i := range allExts {
+		if strings.Contains(allExts[i], "{{") {
+			var buf bytes.Buffer
+			tpl := template.Must(template.New("ext_" + strconv.Itoa(i)).Parse(allExts[i]))
+			tpl.Execute(&buf, templateData)
+			allExts[i] = buf.String()
+		}
+	}
+
+	iterator, err := NewSearchCombinationsIterator(allSearchDependencies, allExts, *positionToUse)
 	if err != nil {
 		log.Fatal().Err(err).Msg("failed to create combinations iterator")
 	}
@@ -345,12 +364,30 @@ func main() {
 
 		i := iterator.Position() - 1
 
+		hasTextSearch := false
+		for _, ext := range combination {
+			if strings.Contains(ext, "keyword_lookup_strategy") ||
+				strings.Contains(ext, "followed_by") ||
+				strings.Contains(ext, "follower_of") {
+				hasTextSearch = true
+
+				break
+			}
+		}
+
+		var kinds []int
+		if hasTextSearch {
+			kinds = []int{nostr.KindProfileMetadata}
+		} else {
+			kinds = []int{model.CustomIONKindEditableTextNote, nostr.KindArticle, nostr.KindGenericRepost}
+		}
+
 		for ctx.Err() == nil {
 			err = pool.Submit(func() {
 				defer bar.Add(1)
 
 				filter := model.Filter{
-					Kinds:  []int{model.CustomIONKindEditableTextNote, nostr.KindArticle, nostr.KindGenericRepost},
+					Kinds:  kinds,
 					Search: strings.Join(combination, " "),
 				}