From fd83c659ba2cf3506978669e870fc4165b86ae5a Mon Sep 17 00:00:00 2001 From: LU Date: Sat, 10 Jan 2026 13:02:44 +0900 Subject: [PATCH 1/6] Add CJK (Chinese, Japanese, Korean) search support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fix search functionality for CJK languages by addressing three issues: 1. Query sanitization: Switch from ASCII-only \w to Unicode-aware \p{L}\p{N} 2. Stemmer: Add character-level tokenization for CJK (e.g., "日本語" → "日 本 語") 3. Highlighter: Add CJK-aware highlighting without word boundaries For SQLite FTS5: Store original content in main table, stemmed content in FTS table. Use Search::Highlighter instead of FTS5's highlight() function to display original text in search results. --- app/models/search.rb | 2 + app/models/search/highlighter.rb | 74 ++++++++++++++++++++------ app/models/search/query.rb | 2 +- app/models/search/record/sqlite.rb | 35 +++++------- app/models/search/stemmer.rb | 36 ++++++++++++- test/models/card/searchable_test.rb | 3 +- test/models/search/highlighter_test.rb | 47 ++++++++++++++++ test/models/search/stemmer_test.rb | 30 +++++++++++ 8 files changed, 187 insertions(+), 42 deletions(-) diff --git a/app/models/search.rb b/app/models/search.rb index f81b9a9d81..3b8127ef18 100644 --- a/app/models/search.rb +++ b/app/models/search.rb @@ -1,4 +1,6 @@ module Search + CJK_PATTERN = /\p{Han}|\p{Hiragana}|\p{Katakana}|\p{Hangul}/ + def self.table_name_prefix "search_" end diff --git a/app/models/search/highlighter.rb b/app/models/search/highlighter.rb index d2b53d7720..1a0bfa2715 100644 --- a/app/models/search/highlighter.rb +++ b/app/models/search/highlighter.rb @@ -13,8 +13,14 @@ def highlight(text) result = text.dup terms.each do |term| - result.gsub!(/\b(#{Regexp.escape(term)}\w*)\b/i) do |match| - "#{OPENING_MARK}#{match}#{CLOSING_MARK}" + if term.match?(Search::CJK_PATTERN) + result.gsub!(/(#{Regexp.escape(term)})/) do |match| + "#{OPENING_MARK}#{match}#{CLOSING_MARK}" + end + else + result.gsub!(/\b(#{Regexp.escape(term)}\w*)\b/i) do |match| + "#{OPENING_MARK}#{match}#{CLOSING_MARK}" + end end end @@ -22,22 +28,10 @@ def highlight(text) end def snippet(text, max_words: 20) - words = text.split(/\s+/) - match_index = words.index { |word| terms.any? { |term| word.downcase.include?(term.downcase) } } - - if words.length <= max_words - highlight(text) - elsif match_index - start_index = [ 0, match_index - max_words / 2 ].max - end_index = [ words.length - 1, start_index + max_words - 1 ].min - - snippet_text = words[start_index..end_index].join(" ") - snippet_text = "...#{snippet_text}" if start_index > 0 - snippet_text = "#{snippet_text}..." if end_index < words.length - 1 - - highlight(snippet_text) + if cjk_dominant?(text) + snippet_for_cjk(text, max_chars: max_words * 3) else - text.truncate_words(max_words, omission: "...") + snippet_for_western(text, max_words: max_words) end end @@ -65,4 +59,50 @@ def escape_highlight_marks(html) .gsub(CGI.escapeHTML(CLOSING_MARK), CLOSING_MARK.html_safe) .html_safe end + + def cjk_dominant?(text) + return false if text.length < 3 + + cjk_chars = text.scan(Search::CJK_PATTERN).length + cjk_chars > text.length / 3 + end + + def snippet_for_cjk(text, max_chars:) + match_index = terms.map { |term| text.index(term) }.compact.min + + if text.length <= max_chars + highlight(text) + elsif match_index + start_index = [ 0, match_index - max_chars / 2 ].max + end_index = [ text.length, start_index + max_chars ].min + + snippet_text = text[start_index...end_index] + snippet_text = "...#{snippet_text}" if start_index > 0 + snippet_text = "#{snippet_text}..." if end_index < text.length + + highlight(snippet_text) + else + "#{text[0, max_chars]}..." + end + end + + def snippet_for_western(text, max_words:) + words = text.split(/\s+/) + match_index = words.index { |word| terms.any? { |term| word.downcase.include?(term.downcase) } } + + if words.length <= max_words + highlight(text) + elsif match_index + start_index = [ 0, match_index - max_words / 2 ].max + end_index = [ words.length - 1, start_index + max_words - 1 ].min + + snippet_text = words[start_index..end_index].join(" ") + snippet_text = "...#{snippet_text}" if start_index > 0 + snippet_text = "#{snippet_text}..." if end_index < words.length - 1 + + highlight(snippet_text) + else + text.truncate_words(max_words, omission: "...") + end + end end diff --git a/app/models/search/query.rb b/app/models/search/query.rb index 5fe0829e41..6bf06ef420 100644 --- a/app/models/search/query.rb +++ b/app/models/search/query.rb @@ -33,7 +33,7 @@ def sanitize(terms) end def remove_invalid_search_characters(terms) - terms.gsub(/[^\w"]/, " ") + terms.gsub(/[^\p{L}\p{N}_"]/, " ") end def remove_unbalanced_quotes(terms) diff --git a/app/models/search/record/sqlite.rb b/app/models/search/record/sqlite.rb index ae0d34281b..701de592b1 100644 --- a/app/models/search/record/sqlite.rb +++ b/app/models/search/record/sqlite.rb @@ -2,26 +2,17 @@ module Search::Record::SQLite extend ActiveSupport::Concern included do - attribute :result_title, :string - attribute :result_content, :string - has_one :search_records_fts, -> { with_rowid }, class_name: "Search::Record::SQLite::Fts", foreign_key: :rowid, primary_key: :id, dependent: :destroy after_save :upsert_to_fts5_table - scope :matching, ->(query, account_id) { joins(:search_records_fts).where("search_records_fts MATCH ?", query) } + scope :matching, ->(query, account_id) { joins(:search_records_fts).where("search_records_fts MATCH ?", Search::Stemmer.stem(query.to_s)) } end class_methods do def search_fields(query) - opening_mark = connection.quote(Search::Highlighter::OPENING_MARK) - closing_mark = connection.quote(Search::Highlighter::CLOSING_MARK) - ellipsis = connection.quote(Search::Highlighter::ELIPSIS) - - [ "highlight(search_records_fts, 0, #{opening_mark}, #{closing_mark}) AS result_title", - "snippet(search_records_fts, 1, #{opening_mark}, #{closing_mark}, #{ellipsis}, 20) AS result_content", - "#{connection.quote(query.terms)} AS query" ] + "#{connection.quote(query.terms)} AS query" end def for(account_id) @@ -30,28 +21,28 @@ def for(account_id) end def card_title - escape_fts_highlight(result_title || card.title) + highlight(card.title, show: :full) if card_id end def card_description - escape_fts_highlight(result_content) unless comment + highlight(card.description.to_plain_text, show: :snippet) if card_id end def comment_body - escape_fts_highlight(result_content) if comment + highlight(comment.body.to_plain_text, show: :snippet) if comment end private - def escape_fts_highlight(html) - return nil unless html.present? - - CGI.escapeHTML(html) - .gsub(CGI.escapeHTML(Search::Highlighter::OPENING_MARK), Search::Highlighter::OPENING_MARK) - .gsub(CGI.escapeHTML(Search::Highlighter::CLOSING_MARK), Search::Highlighter::CLOSING_MARK) - .html_safe + def highlight(text, show:) + if text.present? && attribute?(:query) + highlighter = Search::Highlighter.new(query) + show == :snippet ? highlighter.snippet(text) : highlighter.highlight(text) + else + text + end end def upsert_to_fts5_table - Fts.upsert(id, title, content) + Fts.upsert(id, Search::Stemmer.stem(title), Search::Stemmer.stem(content)) end end diff --git a/app/models/search/stemmer.rb b/app/models/search/stemmer.rb index 21cb75f97c..2ae0e1bf15 100644 --- a/app/models/search/stemmer.rb +++ b/app/models/search/stemmer.rb @@ -5,9 +5,43 @@ module Search::Stemmer def stem(value) if value.present? - value.gsub(/[^\w\s]/, " ").split(/\s+/).map { |word| STEMMER.stem(word.downcase) }.join(" ") + tokenize(value).join(" ") else value end end + + private + def tokenize(value) + tokens = [] + current_word = +"" + + value.each_char do |char| + if cjk_character?(char) + if current_word.present? + tokens << stem_word(current_word) + current_word = +"" + end + tokens << char + elsif char =~ /[\p{L}\p{N}_]/ + current_word << char + else + if current_word.present? + tokens << stem_word(current_word) + current_word = +"" + end + end + end + + tokens << stem_word(current_word) if current_word.present? + tokens + end + + def cjk_character?(char) + char.match?(Search::CJK_PATTERN) + end + + def stem_word(word) + STEMMER.stem(word.downcase) + end end diff --git a/test/models/card/searchable_test.rb b/test/models/card/searchable_test.rb index bf3de621f0..73450fbc00 100644 --- a/test/models/card/searchable_test.rb +++ b/test/models/card/searchable_test.rb @@ -72,7 +72,8 @@ class Card::SearchableTest < ActiveSupport::TestCase if search_record_class.connection.adapter_name == "SQLite" fts_entry = search_record.search_records_fts assert_not_nil fts_entry, "FTS entry should exist" - assert_equal card.title, fts_entry.title + # FTS stores stemmed content for search matching, not original text + assert_equal Search::Stemmer.stem(card.title), fts_entry.title end # Delete the card diff --git a/test/models/search/highlighter_test.rb b/test/models/search/highlighter_test.rb index a065c4da19..ce973d685d 100644 --- a/test/models/search/highlighter_test.rb +++ b/test/models/search/highlighter_test.rb @@ -82,6 +82,53 @@ class Search::HighlighterTest < ActiveSupport::TestCase assert_equal "<script>#{mark('test')}</script>", result end + test "highlight CJK text" do + highlighter = Search::Highlighter.new("中文") + result = highlighter.highlight("这是中文测试") + + assert_equal "这是#{mark('中文')}测试", result + end + + test "highlight Japanese text" do + highlighter = Search::Highlighter.new("日本") + result = highlighter.highlight("これは日本語です") + + assert_equal "これは#{mark('日本')}語です", result + end + + test "highlight Korean text" do + highlighter = Search::Highlighter.new("한국") + result = highlighter.highlight("이것은 한국어입니다") + + assert_equal "이것은 #{mark('한국')}어입니다", result + end + + test "highlight mixed CJK and English" do + highlighter = Search::Highlighter.new("test 中文") + result = highlighter.highlight("This is a test about 中文内容") + + assert_equal "This is a #{mark('test')} about #{mark('中文')}内容", result + end + + test "snippet handles CJK text without spaces" do + highlighter = Search::Highlighter.new("中文") + text = "这是一段很长的中文文本用于测试摘要功能是否正常工作" + result = highlighter.snippet(text, max_words: 20) + + assert_includes result, mark("中文") + end + + test "snippet truncates long CJK text around match" do + highlighter = Search::Highlighter.new("目标") + # 100+ characters, match in the middle + text = "前面有很多很多很多很多很多的文字内容" + "目标词汇" + "后面也有很多很多很多很多很多的文字内容" + result = highlighter.snippet(text, max_words: 10) # max_chars = 30 + + assert_includes result, mark("目标") + assert result.start_with?("...") + assert result.end_with?("...") + end + private def mark(text) "#{Search::Highlighter::OPENING_MARK}#{text}#{Search::Highlighter::CLOSING_MARK}" diff --git a/test/models/search/stemmer_test.rb b/test/models/search/stemmer_test.rb index 33334ada4c..6234c59348 100644 --- a/test/models/search/stemmer_test.rb +++ b/test/models/search/stemmer_test.rb @@ -24,4 +24,34 @@ class Search::StemmerTest < ActiveSupport::TestCase assert_equal "foo bar", result end + + test "stem CJK characters" do + result = Search::Stemmer.stem("测试中文") + + assert_equal "测 试 中 文", result + end + + test "stem Japanese characters" do + result = Search::Stemmer.stem("日本語テスト") + + assert_equal "日 本 語 テ ス ト", result + end + + test "stem Korean characters" do + result = Search::Stemmer.stem("한국어테스트") + + assert_equal "한 국 어 테 스 트", result + end + + test "stem mixed CJK and English" do + result = Search::Stemmer.stem("hello世界test") + + assert_equal "hello 世 界 test", result + end + + test "stem mixed with English stemming" do + result = Search::Stemmer.stem("running 测试 jumping") + + assert_equal "run 测 试 jump", result + end end From aa20528f9e2575afb06659e9c337114a3abb11c7 Mon Sep 17 00:00:00 2001 From: LU Date: Mon, 23 Feb 2026 21:06:28 +0900 Subject: [PATCH 2/6] Address review feedback: fix stemmed highlighting, phrase search, and CJK boundary matching - Stem highlighter terms so searches like "running" highlight "run" in results - Add stem_query to preserve quoted phrases in SQLite MATCH queries - Fix word boundary regex failing at CJK/Latin transitions (Ruby \b issue) - Use case-insensitive matching for CJK snippet positioning --- app/models/search/highlighter.rb | 14 +++++++++++--- app/models/search/record/sqlite.rb | 2 +- app/models/search/stemmer.rb | 14 ++++++++++++++ test/models/search/highlighter_test.rb | 22 ++++++++++++++++++++++ test/models/search/stemmer_test.rb | 24 ++++++++++++++++++++++++ 5 files changed, 72 insertions(+), 4 deletions(-) diff --git a/app/models/search/highlighter.rb b/app/models/search/highlighter.rb index 1a0bfa2715..51dacdfc96 100644 --- a/app/models/search/highlighter.rb +++ b/app/models/search/highlighter.rb @@ -18,7 +18,7 @@ def highlight(text) "#{OPENING_MARK}#{match}#{CLOSING_MARK}" end else - result.gsub!(/\b(#{Regexp.escape(term)}\w*)\b/i) do |match| + result.gsub!(/(?(query, account_id) { joins(:search_records_fts).where("search_records_fts MATCH ?", Search::Stemmer.stem(query.to_s)) } + scope :matching, ->(query, account_id) { joins(:search_records_fts).where("search_records_fts MATCH ?", Search::Stemmer.stem_query(query.to_s)) } end class_methods do diff --git a/app/models/search/stemmer.rb b/app/models/search/stemmer.rb index 2ae0e1bf15..cdc3812105 100644 --- a/app/models/search/stemmer.rb +++ b/app/models/search/stemmer.rb @@ -11,6 +11,20 @@ def stem(value) end end + def stem_query(value) + if value.present? + value.gsub(/"[^"]*"|\S+/) do |token| + if token.start_with?('"') && token.end_with?('"') + "\"#{stem(token[1..-2])}\"" + else + stem(token) + end + end + else + value + end + end + private def tokenize(value) tokens = [] diff --git a/test/models/search/highlighter_test.rb b/test/models/search/highlighter_test.rb index ce973d685d..3ad68198de 100644 --- a/test/models/search/highlighter_test.rb +++ b/test/models/search/highlighter_test.rb @@ -129,6 +129,28 @@ class Search::HighlighterTest < ActiveSupport::TestCase assert result.end_with?("...") end + test "highlight stems terms for better matching" do + highlighter = Search::Highlighter.new("running") + result = highlighter.highlight("I like to run every day") + + assert_equal "I like to #{mark('run')} every day", result + end + + test "snippet finds CJK match case-insensitively" do + highlighter = Search::Highlighter.new("test") + text = "これは非常に長い日本語のテキストでTESTという単語を含む" * 3 + result = highlighter.snippet(text, max_words: 10) + + assert_includes result, mark("TEST") + end + + test "highlight Latin terms adjacent to CJK characters" do + highlighter = Search::Highlighter.new("test") + result = highlighter.highlight("日本語TESTテスト") + + assert_equal "日本語#{mark('TEST')}テスト", result + end + private def mark(text) "#{Search::Highlighter::OPENING_MARK}#{text}#{Search::Highlighter::CLOSING_MARK}" diff --git a/test/models/search/stemmer_test.rb b/test/models/search/stemmer_test.rb index 6234c59348..74f1afb880 100644 --- a/test/models/search/stemmer_test.rb +++ b/test/models/search/stemmer_test.rb @@ -54,4 +54,28 @@ class Search::StemmerTest < ActiveSupport::TestCase assert_equal "run 测 试 jump", result end + + test "stem_query preserves quoted phrases" do + result = Search::Stemmer.stem_query('"hello world" running') + + assert_equal '"hello world" run', result + end + + test "stem_query stems words inside quotes" do + result = Search::Stemmer.stem_query('"running tests"') + + assert_equal '"run test"', result + end + + test "stem_query handles CJK with quotes" do + result = Search::Stemmer.stem_query('"测试" running') + + assert_equal '"测 试" run', result + end + + test "stem_query without quotes behaves like stem" do + result = Search::Stemmer.stem_query("running jumping") + + assert_equal "run jump", result + end end From e8bc4625e8d4567eccc07b933ccf5aa17188301f Mon Sep 17 00:00:00 2001 From: LU Date: Mon, 23 Feb 2026 21:28:26 +0900 Subject: [PATCH 3/6] Wrap CJK tokens as phrases in FTS5 queries and fix mixed-term highlighting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Unquoted CJK terms like 中文 now become "中 文" (phrase) in MATCH queries to preserve character order - CJK highlight regex is now case-insensitive for mixed CJK/Latin terms --- app/models/search/highlighter.rb | 2 +- app/models/search/stemmer.rb | 7 ++++++- test/models/search/highlighter_test.rb | 7 +++++++ test/models/search/stemmer_test.rb | 6 ++++++ 4 files changed, 20 insertions(+), 2 deletions(-) diff --git a/app/models/search/highlighter.rb b/app/models/search/highlighter.rb index 51dacdfc96..26d9316910 100644 --- a/app/models/search/highlighter.rb +++ b/app/models/search/highlighter.rb @@ -14,7 +14,7 @@ def highlight(text) terms.each do |term| if term.match?(Search::CJK_PATTERN) - result.gsub!(/(#{Regexp.escape(term)})/) do |match| + result.gsub!(/(#{Regexp.escape(term)})/i) do |match| "#{OPENING_MARK}#{match}#{CLOSING_MARK}" end else diff --git a/app/models/search/stemmer.rb b/app/models/search/stemmer.rb index cdc3812105..b1011155f4 100644 --- a/app/models/search/stemmer.rb +++ b/app/models/search/stemmer.rb @@ -17,7 +17,12 @@ def stem_query(value) if token.start_with?('"') && token.end_with?('"') "\"#{stem(token[1..-2])}\"" else - stem(token) + stemmed = stem(token) + if stemmed.include?(" ") && token.match?(Search::CJK_PATTERN) + "\"#{stemmed}\"" + else + stemmed + end end end else diff --git a/test/models/search/highlighter_test.rb b/test/models/search/highlighter_test.rb index 3ad68198de..1dec8220db 100644 --- a/test/models/search/highlighter_test.rb +++ b/test/models/search/highlighter_test.rb @@ -151,6 +151,13 @@ class Search::HighlighterTest < ActiveSupport::TestCase assert_equal "日本語#{mark('TEST')}テスト", result end + test "highlight mixed CJK and Latin term case-insensitively" do + highlighter = Search::Highlighter.new("日本語test") + result = highlighter.highlight("これは日本語TESTです") + + assert_equal "これは#{mark('日本語TEST')}です", result + end + private def mark(text) "#{Search::Highlighter::OPENING_MARK}#{text}#{Search::Highlighter::CLOSING_MARK}" diff --git a/test/models/search/stemmer_test.rb b/test/models/search/stemmer_test.rb index 74f1afb880..827527d4f8 100644 --- a/test/models/search/stemmer_test.rb +++ b/test/models/search/stemmer_test.rb @@ -78,4 +78,10 @@ class Search::StemmerTest < ActiveSupport::TestCase assert_equal "run jump", result end + + test "stem_query wraps unquoted CJK tokens as phrase" do + result = Search::Stemmer.stem_query("中文 running") + + assert_equal '"中 文" run', result + end end From 42c299fd27ea4a0a156f9a82aebf62a68c4b071c Mon Sep 17 00:00:00 2001 From: LU Date: Wed, 4 Mar 2026 13:30:25 +0900 Subject: [PATCH 4/6] Address review feedback: switch FTS5 to unicode61, unify snippet logic - Switch FTS5 tokenizer from porter to unicode61 to eliminate double-stemming (Ruby Mittens + SQLite porter). Added migration with full reindex. - Unify snippet into a single character-based method, removing the separate CJK/western branches and cjk_dominant? check. - Split highlight(text, show:) into highlight_snippet/highlight_full in both SQLite and Trilogy modules. - Add CJK integration test to searches_controller_test (SQLite only, MySQL fulltext requires 3+ character tokens). - Revert unrelated cable_schema.rb changes. --- app/models/search/highlighter.rb | 73 ++++++------------- app/models/search/record/sqlite.rb | 19 +++-- app/models/search/record/trilogy.rb | 23 ++++-- ...hange_search_fts_tokenizer_to_unicode61.rb | 48 ++++++++++++ db/schema.rb | 2 +- db/schema_sqlite.rb | 4 +- test/controllers/searches_controller_test.rb | 26 +++++++ test/models/search/highlighter_test.rb | 21 +++--- 8 files changed, 138 insertions(+), 78 deletions(-) create mode 100644 db/migrate/20260304120000_change_search_fts_tokenizer_to_unicode61.rb diff --git a/app/models/search/highlighter.rb b/app/models/search/highlighter.rb index 26d9316910..d897428a2b 100644 --- a/app/models/search/highlighter.rb +++ b/app/models/search/highlighter.rb @@ -27,11 +27,20 @@ def highlight(text) escape_highlight_marks(result) end - def snippet(text, max_words: 20) - if cjk_dominant?(text) - snippet_for_cjk(text, max_chars: max_words * 3) + def snippet(text, max_chars: 100) + if text.length <= max_chars + highlight(text) + elsif (match_index = first_match_position(text)) + start_index = [ 0, match_index - max_chars / 2 ].max + end_index = [ text.length, start_index + max_chars ].min + + snippet_text = text[start_index...end_index] + snippet_text = "#{ELIPSIS}#{snippet_text}" if start_index > 0 + snippet_text = "#{snippet_text}#{ELIPSIS}" if end_index < text.length + + highlight(snippet_text) else - snippet_for_western(text, max_words: max_words) + "#{text[0, max_chars]}#{ELIPSIS}" end end @@ -61,56 +70,20 @@ def terms end end + def first_match_position(text) + terms.filter_map do |term| + if term.match?(Search::CJK_PATTERN) + text =~ /#{Regexp.escape(term)}/i + else + text =~ /(? text.length / 3 - end - - def snippet_for_cjk(text, max_chars:) - match_index = terms.map { |term| text =~ /#{Regexp.escape(term)}/i }.compact.min - - if text.length <= max_chars - highlight(text) - elsif match_index - start_index = [ 0, match_index - max_chars / 2 ].max - end_index = [ text.length, start_index + max_chars ].min - - snippet_text = text[start_index...end_index] - snippet_text = "...#{snippet_text}" if start_index > 0 - snippet_text = "#{snippet_text}..." if end_index < text.length - - highlight(snippet_text) - else - "#{text[0, max_chars]}..." - end - end - - def snippet_for_western(text, max_words:) - words = text.split(/\s+/) - match_index = words.index { |word| terms.any? { |term| word.downcase.include?(term.downcase) } } - - if words.length <= max_words - highlight(text) - elsif match_index - start_index = [ 0, match_index - max_words / 2 ].max - end_index = [ words.length - 1, start_index + max_words - 1 ].min - - snippet_text = words[start_index..end_index].join(" ") - snippet_text = "...#{snippet_text}" if start_index > 0 - snippet_text = "#{snippet_text}..." if end_index < words.length - 1 - - highlight(snippet_text) - else - text.truncate_words(max_words, omission: "...") - end - end end diff --git a/app/models/search/record/sqlite.rb b/app/models/search/record/sqlite.rb index d1877e9f51..c4764f0cfa 100644 --- a/app/models/search/record/sqlite.rb +++ b/app/models/search/record/sqlite.rb @@ -21,22 +21,29 @@ def for(account_id) end def card_title - highlight(card.title, show: :full) if card_id + highlight_full(card.title) if card_id end def card_description - highlight(card.description.to_plain_text, show: :snippet) if card_id + highlight_snippet(card.description.to_plain_text) if card_id end def comment_body - highlight(comment.body.to_plain_text, show: :snippet) if comment + highlight_snippet(comment.body.to_plain_text) if comment end private - def highlight(text, show:) + def highlight_snippet(text) if text.present? && attribute?(:query) - highlighter = Search::Highlighter.new(query) - show == :snippet ? highlighter.snippet(text) : highlighter.highlight(text) + Search::Highlighter.new(query).snippet(text) + else + text + end + end + + def highlight_full(text) + if text.present? && attribute?(:query) + Search::Highlighter.new(query).highlight(text) else text end diff --git a/app/models/search/record/trilogy.rb b/app/models/search/record/trilogy.rb index 7705f3b716..6ae7c34bab 100644 --- a/app/models/search/record/trilogy.rb +++ b/app/models/search/record/trilogy.rb @@ -8,7 +8,7 @@ module Search::Record::Trilogy before_save :set_account_key, :stem_content scope :matching, ->(query, account_id) do - full_query = "+account#{account_id} +(#{Search::Stemmer.stem(query)})" + full_query = "+account#{account_id} +(#{Search::Stemmer.stem_query(query)})" where("MATCH(#{table_name}.account_key, #{table_name}.content, #{table_name}.title) AGAINST(? IN BOOLEAN MODE)", full_query) end @@ -38,20 +38,20 @@ def for(account_id) end def card_title - highlight(card.title, show: :full) if card_id + highlight_full(card.title) if card_id end def card_description - highlight(card.description.to_plain_text, show: :snippet) if card_id + highlight_snippet(card.description.to_plain_text) if card_id end def comment_body - highlight(comment.body.to_plain_text, show: :snippet) if comment + highlight_snippet(comment.body.to_plain_text) if comment end private def stem_content - self.title = Search::Stemmer.stem(title) if title_changed? + self.title = Search::Stemmer.stem(title).truncate(255, omission: "") if title_changed? self.content = Search::Stemmer.stem(content) if content_changed? end @@ -59,10 +59,17 @@ def set_account_key self.account_key = "account#{account_id}" end - def highlight(text, show:) + def highlight_snippet(text) if text.present? && attribute?(:query) - highlighter = Search::Highlighter.new(query) - show == :snippet ? highlighter.snippet(text) : highlighter.highlight(text) + Search::Highlighter.new(query).snippet(text) + else + text + end + end + + def highlight_full(text) + if text.present? && attribute?(:query) + Search::Highlighter.new(query).highlight(text) else text end diff --git a/db/migrate/20260304120000_change_search_fts_tokenizer_to_unicode61.rb b/db/migrate/20260304120000_change_search_fts_tokenizer_to_unicode61.rb new file mode 100644 index 0000000000..dcbf00df37 --- /dev/null +++ b/db/migrate/20260304120000_change_search_fts_tokenizer_to_unicode61.rb @@ -0,0 +1,48 @@ +class ChangeSearchFtsTokenizerToUnicode61 < ActiveRecord::Migration[8.2] + def up + return unless connection.adapter_name == "SQLite" + + execute "DROP TABLE IF EXISTS search_records_fts" + execute <<~SQL + CREATE VIRTUAL TABLE search_records_fts USING fts5( + title, + content, + tokenize='unicode61' + ) + SQL + + reindex_fts_records + end + + def down + return unless connection.adapter_name == "SQLite" + + execute "DROP TABLE IF EXISTS search_records_fts" + execute <<~SQL + CREATE VIRTUAL TABLE search_records_fts USING fts5( + title, + content, + tokenize='porter' + ) + SQL + + # Insert original (unstemmed) text — porter tokenizer handles stemming itself + connection.select_all("SELECT id, title, content FROM search_records").each do |row| + connection.execute( + "INSERT INTO search_records_fts(rowid, title, content) VALUES (#{connection.quote(row['id'])}, #{connection.quote(row['title'])}, #{connection.quote(row['content'])})" + ) + end + end + + private + def reindex_fts_records + connection.select_all("SELECT id, title, content FROM search_records").each do |row| + stemmed_title = Search::Stemmer.stem(row["title"]) + stemmed_content = Search::Stemmer.stem(row["content"]) + + connection.execute( + "INSERT INTO search_records_fts(rowid, title, content) VALUES (#{connection.quote(row['id'])}, #{connection.quote(stemmed_title)}, #{connection.quote(stemmed_content)})" + ) + end + end +end diff --git a/db/schema.rb b/db/schema.rb index 20cb0d1614..fa35eba1ac 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[8.2].define(version: 2026_02_18_120000) do +ActiveRecord::Schema[8.2].define(version: 2026_03_04_120000) do create_table "accesses", id: :uuid, charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci", force: :cascade do |t| t.datetime "accessed_at" t.uuid "account_id", null: false diff --git a/db/schema_sqlite.rb b/db/schema_sqlite.rb index c8ce19e070..59cbb54fc5 100644 --- a/db/schema_sqlite.rb +++ b/db/schema_sqlite.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[8.2].define(version: 2026_02_18_120000) do +ActiveRecord::Schema[8.2].define(version: 2026_03_04_120000) do create_table "accesses", id: :uuid, force: :cascade do |t| t.datetime "accessed_at" t.uuid "account_id", null: false @@ -613,6 +613,6 @@ t.index ["account_id"], name: "index_webhooks_on_account_id" t.index ["board_id", "subscribed_actions"], name: "index_webhooks_on_board_id_and_subscribed_actions" end - execute "CREATE VIRTUAL TABLE search_records_fts USING fts5(\n title,\n content,\n tokenize='porter'\n )" + execute "CREATE VIRTUAL TABLE search_records_fts USING fts5(\n title,\n content,\n tokenize='unicode61'\n)" end diff --git a/test/controllers/searches_controller_test.rb b/test/controllers/searches_controller_test.rb index 11bac3bc93..2b7691cd81 100644 --- a/test/controllers/searches_controller_test.rb +++ b/test/controllers/searches_controller_test.rb @@ -53,6 +53,32 @@ class SearchesControllerTest < ActionDispatch::IntegrationTest assert_response :success end + test "search with CJK content" do + skip "MySQL fulltext requires tokens of 3+ characters" unless ActiveRecord::Base.connection.adapter_name == "SQLite" + + @board.cards.create!(title: "中文测试卡片", description: "这是中文描述", status: "published", creator: @user) + @board.cards.create!(title: "日本語テスト", description: "日本語の説明", status: "published", creator: @user) + @board.cards.create!(title: "한국어 테스트", description: "한국어 설명", status: "published", creator: @user) + @board.cards.create!(title: "Mixed 混合 content", description: "English and 中文 together", status: "published", creator: @user) + + # Chinese search + get search_path(q: "中文", script_name: "/#{@account.external_account_id}") + assert_select "li .search__title", text: /中文测试卡片/ + assert_match(/<\/span>中文<\/mark>/, response.body) + + # Japanese search + get search_path(q: "日本語", script_name: "/#{@account.external_account_id}") + assert_select "li .search__title", text: /日本語テスト/ + + # Korean search + get search_path(q: "한국어", script_name: "/#{@account.external_account_id}") + assert_select "li .search__title", text: /한국어 테스트/ + + # Mixed CJK/English search + get search_path(q: "混合", script_name: "/#{@account.external_account_id}") + assert_select "li .search__title", text: /Mixed 混合 content/ + end + test "search preserves highlight marks but escapes surrounding HTML" do @board.cards.create!( title: "Bold testing content", diff --git a/test/models/search/highlighter_test.rb b/test/models/search/highlighter_test.rb index 1dec8220db..9d4313054a 100644 --- a/test/models/search/highlighter_test.rb +++ b/test/models/search/highlighter_test.rb @@ -29,9 +29,9 @@ class Search::HighlighterTest < ActiveSupport::TestCase assert_equal "Say #{mark('hello world')} to everyone", result end - test "snippet returns full text with highlights when under max words" do + test "snippet returns full text with highlights when under limit" do highlighter = Search::Highlighter.new("ruby") - result = highlighter.snippet("Ruby is great", max_words: 20) + result = highlighter.snippet("Ruby is great", max_chars: 100) assert_equal "#{mark('Ruby')} is great", result end @@ -39,7 +39,7 @@ class Search::HighlighterTest < ActiveSupport::TestCase test "snippet creates excerpt around match" do highlighter = Search::Highlighter.new("match") text = "word " * 10 + "match " + "word " * 10 - result = highlighter.snippet(text, max_words: 10) + result = highlighter.snippet(text, max_chars: 50) assert result.start_with?("...") assert result.end_with?("...") @@ -49,7 +49,7 @@ class Search::HighlighterTest < ActiveSupport::TestCase test "snippet adds leading ellipsis when match is not at start" do highlighter = Search::Highlighter.new("middle") text = "word " * 20 + "middle" - result = highlighter.snippet(text, max_words: 10) + result = highlighter.snippet(text, max_chars: 50) assert result.start_with?("...") assert_not result.end_with?("...") @@ -59,7 +59,7 @@ class Search::HighlighterTest < ActiveSupport::TestCase test "snippet adds trailing ellipsis when text continues after excerpt" do highlighter = Search::Highlighter.new("start") text = "start " + "word " * 30 - result = highlighter.snippet(text, max_words: 10) + result = highlighter.snippet(text, max_chars: 50) assert result.end_with?("...") assert_not result.start_with?("...") @@ -69,7 +69,7 @@ class Search::HighlighterTest < ActiveSupport::TestCase test "snippet falls back to truncation when no match found" do highlighter = Search::Highlighter.new("nomatch") text = "This text does not contain the search term " + "word " * 50 - result = highlighter.snippet(text, max_words: 10) + result = highlighter.snippet(text, max_chars: 50) assert_includes result, "..." assert_not_includes result, Search::Highlighter::OPENING_MARK @@ -113,16 +113,15 @@ class Search::HighlighterTest < ActiveSupport::TestCase test "snippet handles CJK text without spaces" do highlighter = Search::Highlighter.new("中文") text = "这是一段很长的中文文本用于测试摘要功能是否正常工作" - result = highlighter.snippet(text, max_words: 20) + result = highlighter.snippet(text, max_chars: 100) assert_includes result, mark("中文") end test "snippet truncates long CJK text around match" do highlighter = Search::Highlighter.new("目标") - # 100+ characters, match in the middle text = "前面有很多很多很多很多很多的文字内容" + "目标词汇" + "后面也有很多很多很多很多很多的文字内容" - result = highlighter.snippet(text, max_words: 10) # max_chars = 30 + result = highlighter.snippet(text, max_chars: 30) assert_includes result, mark("目标") assert result.start_with?("...") @@ -136,10 +135,10 @@ class Search::HighlighterTest < ActiveSupport::TestCase assert_equal "I like to #{mark('run')} every day", result end - test "snippet finds CJK match case-insensitively" do + test "snippet finds match case-insensitively" do highlighter = Search::Highlighter.new("test") text = "これは非常に長い日本語のテキストでTESTという単語を含む" * 3 - result = highlighter.snippet(text, max_words: 10) + result = highlighter.snippet(text, max_chars: 30) assert_includes result, mark("TEST") end From 38e4bfd605936e0c9b263b668be369891c0b7bda Mon Sep 17 00:00:00 2001 From: LU Date: Fri, 10 Jul 2026 15:34:36 +0900 Subject: [PATCH 5/6] Add test for accented character search MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Query sanitization used to strip non-ASCII letters, breaking search for accented characters (å, ä, ö, é, ...). The switch to \p{L}\p{N} in Search::Query fixed this alongside CJK; cover it with a test that runs on both SQLite and MySQL. Claude-Session: https://claude.ai/code/session_01AHkZVHPX1kNFJU5fYu7o41 --- test/controllers/searches_controller_test.rb | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/test/controllers/searches_controller_test.rb b/test/controllers/searches_controller_test.rb index 645b684d7d..26fc68fb77 100644 --- a/test/controllers/searches_controller_test.rb +++ b/test/controllers/searches_controller_test.rb @@ -109,6 +109,18 @@ class SearchesControllerTest < ActionDispatch::IntegrationTest assert_select "li .search__title", text: /Mixed 混合 content/ end + test "search with accented characters" do + @board.cards.create!(title: "Hälsa och friskvård", description: "Möte i Göteborg", status: "published", creator: @user) + + # Accented characters must survive query sanitization + get search_path(q: "hälsa", script_name: "/#{@account.external_account_id}") + assert_select "li .search__title", text: /Hälsa och friskvård/ + assert_match(/<\/span>Hälsa<\/mark>/, response.body) + + get search_path(q: "Göteborg", script_name: "/#{@account.external_account_id}") + assert_select "li .search__excerpt", text: /Möte i Göteborg/ + end + test "search preserves highlight marks but escapes surrounding HTML" do @board.cards.create!( title: "Bold testing content", From 1dbb5eb3e64d232f60bb3a508150b41508169c23 Mon Sep 17 00:00:00 2001 From: LU Date: Fri, 10 Jul 2026 16:23:19 +0900 Subject: [PATCH 6/6] Fix highlight markup corruption and preserve Unicode combining marks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit highlight() inserted real HTML during per-term gsub passes, so a later term matching the markup itself (e.g. "class", "text") corrupted the output. Wrap matches in private-use sentinel characters instead and swap in the real markup once all terms are processed. Query sanitization and stemmer tokenization also treated combining marks (\p{M}) as separators, splitting decomposed accents (NFD "café") and scripts like Devanagari into fragments. Keep marks with their base characters on both the indexing and query side. Claude-Session: https://claude.ai/code/session_01AHkZVHPX1kNFJU5fYu7o41 --- app/models/search/highlighter.rb | 14 +++++++++----- app/models/search/query.rb | 2 +- app/models/search/stemmer.rb | 2 +- test/controllers/searches_controller_test.rb | 4 ++++ test/models/search/highlighter_test.rb | 14 ++++++++++++++ test/models/search/stemmer_test.rb | 13 +++++++++++++ 6 files changed, 42 insertions(+), 7 deletions(-) diff --git a/app/models/search/highlighter.rb b/app/models/search/highlighter.rb index d897428a2b..2eed1778c8 100644 --- a/app/models/search/highlighter.rb +++ b/app/models/search/highlighter.rb @@ -1,6 +1,10 @@ class Search::Highlighter OPENING_MARK = "" CLOSING_MARK = "" + # Private-use characters mark matches during term passes, so a later term + # can't match inside HTML inserted for an earlier one + OPENING_SENTINEL = "\uE000" + CLOSING_SENTINEL = "\uE001" ELIPSIS = "..." attr_reader :query @@ -10,16 +14,16 @@ def initialize(query) end def highlight(text) - result = text.dup + result = text.delete(OPENING_SENTINEL + CLOSING_SENTINEL) terms.each do |term| if term.match?(Search::CJK_PATTERN) result.gsub!(/(#{Regexp.escape(term)})/i) do |match| - "#{OPENING_MARK}#{match}#{CLOSING_MARK}" + "#{OPENING_SENTINEL}#{match}#{CLOSING_SENTINEL}" end else result.gsub!(/(?