diff --git a/app/models/search.rb b/app/models/search.rb index f81b9a9d81..3b8127ef18 100644 --- a/app/models/search.rb +++ b/app/models/search.rb @@ -1,4 +1,6 @@ module Search + CJK_PATTERN = /\p{Han}|\p{Hiragana}|\p{Katakana}|\p{Hangul}/ + def self.table_name_prefix "search_" end diff --git a/app/models/search/highlighter.rb b/app/models/search/highlighter.rb index d2b53d7720..2eed1778c8 100644 --- a/app/models/search/highlighter.rb +++ b/app/models/search/highlighter.rb @@ -1,6 +1,10 @@ class Search::Highlighter OPENING_MARK = "" CLOSING_MARK = "" + # Private-use characters mark matches during term passes, so a later term + # can't match inside HTML inserted for an earlier one + OPENING_SENTINEL = "\uE000" + CLOSING_SENTINEL = "\uE001" ELIPSIS = "..." attr_reader :query @@ -10,34 +14,37 @@ def initialize(query) end def highlight(text) - result = text.dup + result = text.delete(OPENING_SENTINEL + CLOSING_SENTINEL) terms.each do |term| - result.gsub!(/\b(#{Regexp.escape(term)}\w*)\b/i) do |match| - "#{OPENING_MARK}#{match}#{CLOSING_MARK}" + if term.match?(Search::CJK_PATTERN) + result.gsub!(/(#{Regexp.escape(term)})/i) do |match| + "#{OPENING_SENTINEL}#{match}#{CLOSING_SENTINEL}" + end + else + result.gsub!(/(? 0 - snippet_text = "#{snippet_text}..." if end_index < words.length - 1 + snippet_text = text[start_index...end_index] + snippet_text = "#{ELIPSIS}#{snippet_text}" if start_index > 0 + snippet_text = "#{snippet_text}#{ELIPSIS}" if end_index < text.length highlight(snippet_text) else - text.truncate_words(max_words, omission: "...") + "#{text[0, max_chars]}#{ELIPSIS}" end end @@ -52,17 +59,35 @@ def terms unquoted = query.gsub(/"[^"]+"/, "") unquoted.split(/\s+/).each do |word| - terms << word if word.present? + next unless word.present? + + if word.match?(Search::CJK_PATTERN) + terms << word + else + stemmed = Search::Stemmer.stem(word) + terms << stemmed + terms << word.downcase unless word.downcase.start_with?(stemmed) + end end terms.uniq end end + def first_match_position(text) + terms.filter_map do |term| + if term.match?(Search::CJK_PATTERN) + text =~ /#{Regexp.escape(term)}/i + else + text =~ /(? { with_rowid }, class_name: "Search::Record::SQLite::Fts", foreign_key: :rowid, primary_key: :id, dependent: :destroy @@ -12,19 +9,13 @@ module Search::Record::SQLite scope :matching, ->(query, account_id) { joins("INNER JOIN search_records_fts ON search_records_fts.rowid = #{table_name}.id") - .where("search_records_fts MATCH ?", query) + .where("search_records_fts MATCH ?", Search::Stemmer.stem_query(query.to_s)) } end class_methods do def search_fields(query) - opening_mark = connection.quote(Search::Highlighter::OPENING_MARK) - closing_mark = connection.quote(Search::Highlighter::CLOSING_MARK) - ellipsis = connection.quote(Search::Highlighter::ELIPSIS) - - [ "highlight(search_records_fts, 0, #{opening_mark}, #{closing_mark}) AS result_title", - "snippet(search_records_fts, 1, #{opening_mark}, #{closing_mark}, #{ellipsis}, 20) AS result_content", - "#{connection.quote(query.terms)} AS query" ] + "#{connection.quote(query.terms)} AS query" end def for(account_id) @@ -33,28 +24,35 @@ def for(account_id) end def card_title - escape_fts_highlight(result_title || card.title) + highlight_full(card.title) if card_id end def card_description - escape_fts_highlight(result_content) unless comment + highlight_snippet(card.description.to_plain_text) if card_id end def comment_body - escape_fts_highlight(result_content) if comment + highlight_snippet(comment.body.to_plain_text) if comment end private - def escape_fts_highlight(html) - return nil unless html.present? + def highlight_snippet(text) + if text.present? && attribute?(:query) + Search::Highlighter.new(query).snippet(text) + else + text + end + end - CGI.escapeHTML(html) - .gsub(CGI.escapeHTML(Search::Highlighter::OPENING_MARK), Search::Highlighter::OPENING_MARK) - .gsub(CGI.escapeHTML(Search::Highlighter::CLOSING_MARK), Search::Highlighter::CLOSING_MARK) - .html_safe + def highlight_full(text) + if text.present? && attribute?(:query) + Search::Highlighter.new(query).highlight(text) + else + text + end end def upsert_to_fts5_table - Fts.upsert(id, title, content) + Fts.upsert(id, Search::Stemmer.stem(title), Search::Stemmer.stem(content)) end end diff --git a/app/models/search/record/trilogy.rb b/app/models/search/record/trilogy.rb index 7705f3b716..6ae7c34bab 100644 --- a/app/models/search/record/trilogy.rb +++ b/app/models/search/record/trilogy.rb @@ -8,7 +8,7 @@ module Search::Record::Trilogy before_save :set_account_key, :stem_content scope :matching, ->(query, account_id) do - full_query = "+account#{account_id} +(#{Search::Stemmer.stem(query)})" + full_query = "+account#{account_id} +(#{Search::Stemmer.stem_query(query)})" where("MATCH(#{table_name}.account_key, #{table_name}.content, #{table_name}.title) AGAINST(? IN BOOLEAN MODE)", full_query) end @@ -38,20 +38,20 @@ def for(account_id) end def card_title - highlight(card.title, show: :full) if card_id + highlight_full(card.title) if card_id end def card_description - highlight(card.description.to_plain_text, show: :snippet) if card_id + highlight_snippet(card.description.to_plain_text) if card_id end def comment_body - highlight(comment.body.to_plain_text, show: :snippet) if comment + highlight_snippet(comment.body.to_plain_text) if comment end private def stem_content - self.title = Search::Stemmer.stem(title) if title_changed? + self.title = Search::Stemmer.stem(title).truncate(255, omission: "") if title_changed? self.content = Search::Stemmer.stem(content) if content_changed? end @@ -59,10 +59,17 @@ def set_account_key self.account_key = "account#{account_id}" end - def highlight(text, show:) + def highlight_snippet(text) if text.present? && attribute?(:query) - highlighter = Search::Highlighter.new(query) - show == :snippet ? highlighter.snippet(text) : highlighter.highlight(text) + Search::Highlighter.new(query).snippet(text) + else + text + end + end + + def highlight_full(text) + if text.present? && attribute?(:query) + Search::Highlighter.new(query).highlight(text) else text end diff --git a/app/models/search/stemmer.rb b/app/models/search/stemmer.rb index 21cb75f97c..30cf496f81 100644 --- a/app/models/search/stemmer.rb +++ b/app/models/search/stemmer.rb @@ -5,9 +5,62 @@ module Search::Stemmer def stem(value) if value.present? - value.gsub(/[^\w\s]/, " ").split(/\s+/).map { |word| STEMMER.stem(word.downcase) }.join(" ") + tokenize(value).join(" ") else value end end + + def stem_query(value) + if value.present? + value.gsub(/"[^"]*"|\S+/) do |token| + if token.start_with?('"') && token.end_with?('"') + "\"#{stem(token[1..-2])}\"" + else + stemmed = stem(token) + if stemmed.include?(" ") && token.match?(Search::CJK_PATTERN) + "\"#{stemmed}\"" + else + stemmed + end + end + end + else + value + end + end + + private + def tokenize(value) + tokens = [] + current_word = +"" + + value.each_char do |char| + if cjk_character?(char) + if current_word.present? + tokens << stem_word(current_word) + current_word = +"" + end + tokens << char + elsif char =~ /[\p{L}\p{M}\p{N}_]/ + current_word << char + else + if current_word.present? + tokens << stem_word(current_word) + current_word = +"" + end + end + end + + tokens << stem_word(current_word) if current_word.present? + tokens + end + + def cjk_character?(char) + char.match?(Search::CJK_PATTERN) + end + + def stem_word(word) + STEMMER.stem(word.downcase) + end end diff --git a/db/migrate/20260304120000_change_search_fts_tokenizer_to_unicode61.rb b/db/migrate/20260304120000_change_search_fts_tokenizer_to_unicode61.rb new file mode 100644 index 0000000000..dcbf00df37 --- /dev/null +++ b/db/migrate/20260304120000_change_search_fts_tokenizer_to_unicode61.rb @@ -0,0 +1,48 @@ +class ChangeSearchFtsTokenizerToUnicode61 < ActiveRecord::Migration[8.2] + def up + return unless connection.adapter_name == "SQLite" + + execute "DROP TABLE IF EXISTS search_records_fts" + execute <<~SQL + CREATE VIRTUAL TABLE search_records_fts USING fts5( + title, + content, + tokenize='unicode61' + ) + SQL + + reindex_fts_records + end + + def down + return unless connection.adapter_name == "SQLite" + + execute "DROP TABLE IF EXISTS search_records_fts" + execute <<~SQL + CREATE VIRTUAL TABLE search_records_fts USING fts5( + title, + content, + tokenize='porter' + ) + SQL + + # Insert original (unstemmed) text — porter tokenizer handles stemming itself + connection.select_all("SELECT id, title, content FROM search_records").each do |row| + connection.execute( + "INSERT INTO search_records_fts(rowid, title, content) VALUES (#{connection.quote(row['id'])}, #{connection.quote(row['title'])}, #{connection.quote(row['content'])})" + ) + end + end + + private + def reindex_fts_records + connection.select_all("SELECT id, title, content FROM search_records").each do |row| + stemmed_title = Search::Stemmer.stem(row["title"]) + stemmed_content = Search::Stemmer.stem(row["content"]) + + connection.execute( + "INSERT INTO search_records_fts(rowid, title, content) VALUES (#{connection.quote(row['id'])}, #{connection.quote(stemmed_title)}, #{connection.quote(stemmed_content)})" + ) + end + end +end diff --git a/db/schema_sqlite.rb b/db/schema_sqlite.rb index 9e4f050dc2..afe7dc01cf 100644 --- a/db/schema_sqlite.rb +++ b/db/schema_sqlite.rb @@ -630,6 +630,6 @@ t.index ["account_id"], name: "index_webhooks_on_account_id" t.index ["board_id", "subscribed_actions"], name: "index_webhooks_on_board_id_and_subscribed_actions" end - execute "CREATE VIRTUAL TABLE search_records_fts USING fts5(\n title,\n content,\n tokenize='porter'\n )" + execute "CREATE VIRTUAL TABLE search_records_fts USING fts5(\n title,\n content,\n tokenize='unicode61'\n)" end diff --git a/test/controllers/searches_controller_test.rb b/test/controllers/searches_controller_test.rb index b0172cdf51..0b723478df 100644 --- a/test/controllers/searches_controller_test.rb +++ b/test/controllers/searches_controller_test.rb @@ -83,6 +83,48 @@ class SearchesControllerTest < ActionDispatch::IntegrationTest assert_response :success end + test "search with CJK content" do + skip "MySQL fulltext requires tokens of 3+ characters" unless ActiveRecord::Base.connection.adapter_name == "SQLite" + + @board.cards.create!(title: "中文测试卡片", description: "这是中文描述", status: "published", creator: @user) + @board.cards.create!(title: "日本語テスト", description: "日本語の説明", status: "published", creator: @user) + @board.cards.create!(title: "한국어 테스트", description: "한국어 설명", status: "published", creator: @user) + @board.cards.create!(title: "Mixed 混合 content", description: "English and 中文 together", status: "published", creator: @user) + + # Chinese search + get search_path(q: "中文", script_name: "/#{@account.external_account_id}") + assert_select "li .search__title", text: /中文测试卡片/ + assert_match(/<\/span>中文<\/mark>/, response.body) + + # Japanese search + get search_path(q: "日本語", script_name: "/#{@account.external_account_id}") + assert_select "li .search__title", text: /日本語テスト/ + + # Korean search + get search_path(q: "한국어", script_name: "/#{@account.external_account_id}") + assert_select "li .search__title", text: /한국어 테스트/ + + # Mixed CJK/English search + get search_path(q: "混合", script_name: "/#{@account.external_account_id}") + assert_select "li .search__title", text: /Mixed 混合 content/ + end + + test "search with accented characters" do + @board.cards.create!(title: "Hälsa och friskvård", description: "Möte i Göteborg", status: "published", creator: @user) + + # Accented characters must survive query sanitization + get search_path(q: "hälsa", script_name: "/#{@account.external_account_id}") + assert_select "li .search__title", text: /Hälsa och friskvård/ + assert_match(/<\/span>Hälsa<\/mark>/, response.body) + + get search_path(q: "Göteborg", script_name: "/#{@account.external_account_id}") + assert_select "li .search__excerpt", text: /Möte i Göteborg/ + + # Decomposed form (a + combining diaeresis) must survive sanitization and still match + get search_path(q: "ha\u0308lsa", script_name: "/#{@account.external_account_id}") + assert_select "li .search__title", text: /Hälsa och friskvård/ + end + test "search preserves highlight marks but escapes surrounding HTML" do @board.cards.create!( title: "Bold testing content", diff --git a/test/models/card/searchable_test.rb b/test/models/card/searchable_test.rb index bf3de621f0..73450fbc00 100644 --- a/test/models/card/searchable_test.rb +++ b/test/models/card/searchable_test.rb @@ -72,7 +72,8 @@ class Card::SearchableTest < ActiveSupport::TestCase if search_record_class.connection.adapter_name == "SQLite" fts_entry = search_record.search_records_fts assert_not_nil fts_entry, "FTS entry should exist" - assert_equal card.title, fts_entry.title + # FTS stores stemmed content for search matching, not original text + assert_equal Search::Stemmer.stem(card.title), fts_entry.title end # Delete the card diff --git a/test/models/search/highlighter_test.rb b/test/models/search/highlighter_test.rb index a065c4da19..ee36e1f8f7 100644 --- a/test/models/search/highlighter_test.rb +++ b/test/models/search/highlighter_test.rb @@ -29,9 +29,9 @@ class Search::HighlighterTest < ActiveSupport::TestCase assert_equal "Say #{mark('hello world')} to everyone", result end - test "snippet returns full text with highlights when under max words" do + test "snippet returns full text with highlights when under limit" do highlighter = Search::Highlighter.new("ruby") - result = highlighter.snippet("Ruby is great", max_words: 20) + result = highlighter.snippet("Ruby is great", max_chars: 100) assert_equal "#{mark('Ruby')} is great", result end @@ -39,7 +39,7 @@ class Search::HighlighterTest < ActiveSupport::TestCase test "snippet creates excerpt around match" do highlighter = Search::Highlighter.new("match") text = "word " * 10 + "match " + "word " * 10 - result = highlighter.snippet(text, max_words: 10) + result = highlighter.snippet(text, max_chars: 50) assert result.start_with?("...") assert result.end_with?("...") @@ -49,7 +49,7 @@ class Search::HighlighterTest < ActiveSupport::TestCase test "snippet adds leading ellipsis when match is not at start" do highlighter = Search::Highlighter.new("middle") text = "word " * 20 + "middle" - result = highlighter.snippet(text, max_words: 10) + result = highlighter.snippet(text, max_chars: 50) assert result.start_with?("...") assert_not result.end_with?("...") @@ -59,7 +59,7 @@ class Search::HighlighterTest < ActiveSupport::TestCase test "snippet adds trailing ellipsis when text continues after excerpt" do highlighter = Search::Highlighter.new("start") text = "start " + "word " * 30 - result = highlighter.snippet(text, max_words: 10) + result = highlighter.snippet(text, max_chars: 50) assert result.end_with?("...") assert_not result.start_with?("...") @@ -69,7 +69,7 @@ class Search::HighlighterTest < ActiveSupport::TestCase test "snippet falls back to truncation when no match found" do highlighter = Search::Highlighter.new("nomatch") text = "This text does not contain the search term " + "word " * 50 - result = highlighter.snippet(text, max_words: 10) + result = highlighter.snippet(text, max_chars: 50) assert_includes result, "..." assert_not_includes result, Search::Highlighter::OPENING_MARK @@ -82,6 +82,95 @@ class Search::HighlighterTest < ActiveSupport::TestCase assert_equal "<script>#{mark('test')}</script>", result end + test "highlight CJK text" do + highlighter = Search::Highlighter.new("中文") + result = highlighter.highlight("这是中文测试") + + assert_equal "这是#{mark('中文')}测试", result + end + + test "highlight Japanese text" do + highlighter = Search::Highlighter.new("日本") + result = highlighter.highlight("これは日本語です") + + assert_equal "これは#{mark('日本')}語です", result + end + + test "highlight Korean text" do + highlighter = Search::Highlighter.new("한국") + result = highlighter.highlight("이것은 한국어입니다") + + assert_equal "이것은 #{mark('한국')}어입니다", result + end + + test "highlight mixed CJK and English" do + highlighter = Search::Highlighter.new("test 中文") + result = highlighter.highlight("This is a test about 中文内容") + + assert_equal "This is a #{mark('test')} about #{mark('中文')}内容", result + end + + test "snippet handles CJK text without spaces" do + highlighter = Search::Highlighter.new("中文") + text = "这是一段很长的中文文本用于测试摘要功能是否正常工作" + result = highlighter.snippet(text, max_chars: 100) + + assert_includes result, mark("中文") + end + + test "snippet truncates long CJK text around match" do + highlighter = Search::Highlighter.new("目标") + text = "前面有很多很多很多很多很多的文字内容" + "目标词汇" + "后面也有很多很多很多很多很多的文字内容" + result = highlighter.snippet(text, max_chars: 30) + + assert_includes result, mark("目标") + assert result.start_with?("...") + assert result.end_with?("...") + end + + test "highlight stems terms for better matching" do + highlighter = Search::Highlighter.new("running") + result = highlighter.highlight("I like to run every day") + + assert_equal "I like to #{mark('run')} every day", result + end + + test "snippet finds match case-insensitively" do + highlighter = Search::Highlighter.new("test") + text = "これは非常に長い日本語のテキストでTESTという単語を含む" * 3 + result = highlighter.snippet(text, max_chars: 30) + + assert_includes result, mark("TEST") + end + + test "highlight Latin terms adjacent to CJK characters" do + highlighter = Search::Highlighter.new("test") + result = highlighter.highlight("日本語TESTテスト") + + assert_equal "日本語#{mark('TEST')}テスト", result + end + + test "highlight mixed CJK and Latin term case-insensitively" do + highlighter = Search::Highlighter.new("日本語test") + result = highlighter.highlight("これは日本語TESTです") + + assert_equal "これは#{mark('日本語TEST')}です", result + end + + test "highlight does not corrupt markup when a term matches inserted HTML" do + highlighter = Search::Highlighter.new("test class") + result = highlighter.highlight("test class here") + + assert_equal "#{mark('test')} #{mark('class')} here", result + end + + test "highlight terms matching mark tag attributes" do + highlighter = Search::Highlighter.new("some text") + result = highlighter.highlight("some text to mark up") + + assert_equal "#{mark('some')} #{mark('text')} to mark up", result + end + private def mark(text) "#{Search::Highlighter::OPENING_MARK}#{text}#{Search::Highlighter::CLOSING_MARK}" diff --git a/test/models/search/stemmer_test.rb b/test/models/search/stemmer_test.rb index 33334ada4c..d2af254c31 100644 --- a/test/models/search/stemmer_test.rb +++ b/test/models/search/stemmer_test.rb @@ -24,4 +24,77 @@ class Search::StemmerTest < ActiveSupport::TestCase assert_equal "foo bar", result end + + test "stem CJK characters" do + result = Search::Stemmer.stem("测试中文") + + assert_equal "测 试 中 文", result + end + + test "stem Japanese characters" do + result = Search::Stemmer.stem("日本語テスト") + + assert_equal "日 本 語 テ ス ト", result + end + + test "stem Korean characters" do + result = Search::Stemmer.stem("한국어테스트") + + assert_equal "한 국 어 테 스 트", result + end + + test "stem mixed CJK and English" do + result = Search::Stemmer.stem("hello世界test") + + assert_equal "hello 世 界 test", result + end + + test "stem mixed with English stemming" do + result = Search::Stemmer.stem("running 测试 jumping") + + assert_equal "run 测 试 jump", result + end + + test "stem_query preserves quoted phrases" do + result = Search::Stemmer.stem_query('"hello world" running') + + assert_equal '"hello world" run', result + end + + test "stem_query stems words inside quotes" do + result = Search::Stemmer.stem_query('"running tests"') + + assert_equal '"run test"', result + end + + test "stem_query handles CJK with quotes" do + result = Search::Stemmer.stem_query('"测试" running') + + assert_equal '"测 试" run', result + end + + test "stem_query without quotes behaves like stem" do + result = Search::Stemmer.stem_query("running jumping") + + assert_equal "run jump", result + end + + test "stem_query wraps unquoted CJK tokens as phrase" do + result = Search::Stemmer.stem_query("中文 running") + + assert_equal '"中 文" run', result + end + + test "stem preserves accented characters" do + result = Search::Stemmer.stem("Hälsa i Göteborg") + + assert_equal "hälsa i göteborg", result + end + + test "stem preserves combining marks" do + # "cafe\u0301" in NFD form (e + combining acute accent) and Devanagari with matras + result = Search::Stemmer.stem("cafe\u0301 हिंदी") + + assert_equal "cafe\u0301 हिंदी", result + end end