diff --git a/app/models/search.rb b/app/models/search.rb
index f81b9a9d81..3b8127ef18 100644
--- a/app/models/search.rb
+++ b/app/models/search.rb
@@ -1,4 +1,6 @@
module Search
+ CJK_PATTERN = /\p{Han}|\p{Hiragana}|\p{Katakana}|\p{Hangul}/
+
def self.table_name_prefix
"search_"
end
diff --git a/app/models/search/highlighter.rb b/app/models/search/highlighter.rb
index d2b53d7720..2eed1778c8 100644
--- a/app/models/search/highlighter.rb
+++ b/app/models/search/highlighter.rb
@@ -1,6 +1,10 @@
class Search::Highlighter
OPENING_MARK = ""
CLOSING_MARK = ""
+ # Private-use characters mark matches during term passes, so a later term
+ # can't match inside HTML inserted for an earlier one
+ OPENING_SENTINEL = "\uE000"
+ CLOSING_SENTINEL = "\uE001"
ELIPSIS = "..."
attr_reader :query
@@ -10,34 +14,37 @@ def initialize(query)
end
def highlight(text)
- result = text.dup
+ result = text.delete(OPENING_SENTINEL + CLOSING_SENTINEL)
terms.each do |term|
- result.gsub!(/\b(#{Regexp.escape(term)}\w*)\b/i) do |match|
- "#{OPENING_MARK}#{match}#{CLOSING_MARK}"
+ if term.match?(Search::CJK_PATTERN)
+ result.gsub!(/(#{Regexp.escape(term)})/i) do |match|
+ "#{OPENING_SENTINEL}#{match}#{CLOSING_SENTINEL}"
+ end
+ else
+ result.gsub!(/(? 0
- snippet_text = "#{snippet_text}..." if end_index < words.length - 1
+ snippet_text = text[start_index...end_index]
+ snippet_text = "#{ELIPSIS}#{snippet_text}" if start_index > 0
+ snippet_text = "#{snippet_text}#{ELIPSIS}" if end_index < text.length
highlight(snippet_text)
else
- text.truncate_words(max_words, omission: "...")
+ "#{text[0, max_chars]}#{ELIPSIS}"
end
end
@@ -52,17 +59,35 @@ def terms
unquoted = query.gsub(/"[^"]+"/, "")
unquoted.split(/\s+/).each do |word|
- terms << word if word.present?
+ next unless word.present?
+
+ if word.match?(Search::CJK_PATTERN)
+ terms << word
+ else
+ stemmed = Search::Stemmer.stem(word)
+ terms << stemmed
+ terms << word.downcase unless word.downcase.start_with?(stemmed)
+ end
end
terms.uniq
end
end
+ def first_match_position(text)
+ terms.filter_map do |term|
+ if term.match?(Search::CJK_PATTERN)
+ text =~ /#{Regexp.escape(term)}/i
+ else
+ text =~ /(? { with_rowid },
class_name: "Search::Record::SQLite::Fts", foreign_key: :rowid, primary_key: :id, dependent: :destroy
@@ -12,19 +9,13 @@ module Search::Record::SQLite
scope :matching, ->(query, account_id) {
joins("INNER JOIN search_records_fts ON search_records_fts.rowid = #{table_name}.id")
- .where("search_records_fts MATCH ?", query)
+ .where("search_records_fts MATCH ?", Search::Stemmer.stem_query(query.to_s))
}
end
class_methods do
def search_fields(query)
- opening_mark = connection.quote(Search::Highlighter::OPENING_MARK)
- closing_mark = connection.quote(Search::Highlighter::CLOSING_MARK)
- ellipsis = connection.quote(Search::Highlighter::ELIPSIS)
-
- [ "highlight(search_records_fts, 0, #{opening_mark}, #{closing_mark}) AS result_title",
- "snippet(search_records_fts, 1, #{opening_mark}, #{closing_mark}, #{ellipsis}, 20) AS result_content",
- "#{connection.quote(query.terms)} AS query" ]
+ "#{connection.quote(query.terms)} AS query"
end
def for(account_id)
@@ -33,28 +24,35 @@ def for(account_id)
end
def card_title
- escape_fts_highlight(result_title || card.title)
+ highlight_full(card.title) if card_id
end
def card_description
- escape_fts_highlight(result_content) unless comment
+ highlight_snippet(card.description.to_plain_text) if card_id
end
def comment_body
- escape_fts_highlight(result_content) if comment
+ highlight_snippet(comment.body.to_plain_text) if comment
end
private
- def escape_fts_highlight(html)
- return nil unless html.present?
+ def highlight_snippet(text)
+ if text.present? && attribute?(:query)
+ Search::Highlighter.new(query).snippet(text)
+ else
+ text
+ end
+ end
- CGI.escapeHTML(html)
- .gsub(CGI.escapeHTML(Search::Highlighter::OPENING_MARK), Search::Highlighter::OPENING_MARK)
- .gsub(CGI.escapeHTML(Search::Highlighter::CLOSING_MARK), Search::Highlighter::CLOSING_MARK)
- .html_safe
+ def highlight_full(text)
+ if text.present? && attribute?(:query)
+ Search::Highlighter.new(query).highlight(text)
+ else
+ text
+ end
end
def upsert_to_fts5_table
- Fts.upsert(id, title, content)
+ Fts.upsert(id, Search::Stemmer.stem(title), Search::Stemmer.stem(content))
end
end
diff --git a/app/models/search/record/trilogy.rb b/app/models/search/record/trilogy.rb
index 7705f3b716..6ae7c34bab 100644
--- a/app/models/search/record/trilogy.rb
+++ b/app/models/search/record/trilogy.rb
@@ -8,7 +8,7 @@ module Search::Record::Trilogy
before_save :set_account_key, :stem_content
scope :matching, ->(query, account_id) do
- full_query = "+account#{account_id} +(#{Search::Stemmer.stem(query)})"
+ full_query = "+account#{account_id} +(#{Search::Stemmer.stem_query(query)})"
where("MATCH(#{table_name}.account_key, #{table_name}.content, #{table_name}.title) AGAINST(? IN BOOLEAN MODE)", full_query)
end
@@ -38,20 +38,20 @@ def for(account_id)
end
def card_title
- highlight(card.title, show: :full) if card_id
+ highlight_full(card.title) if card_id
end
def card_description
- highlight(card.description.to_plain_text, show: :snippet) if card_id
+ highlight_snippet(card.description.to_plain_text) if card_id
end
def comment_body
- highlight(comment.body.to_plain_text, show: :snippet) if comment
+ highlight_snippet(comment.body.to_plain_text) if comment
end
private
def stem_content
- self.title = Search::Stemmer.stem(title) if title_changed?
+ self.title = Search::Stemmer.stem(title).truncate(255, omission: "") if title_changed?
self.content = Search::Stemmer.stem(content) if content_changed?
end
@@ -59,10 +59,17 @@ def set_account_key
self.account_key = "account#{account_id}"
end
- def highlight(text, show:)
+ def highlight_snippet(text)
if text.present? && attribute?(:query)
- highlighter = Search::Highlighter.new(query)
- show == :snippet ? highlighter.snippet(text) : highlighter.highlight(text)
+ Search::Highlighter.new(query).snippet(text)
+ else
+ text
+ end
+ end
+
+ def highlight_full(text)
+ if text.present? && attribute?(:query)
+ Search::Highlighter.new(query).highlight(text)
else
text
end
diff --git a/app/models/search/stemmer.rb b/app/models/search/stemmer.rb
index 21cb75f97c..30cf496f81 100644
--- a/app/models/search/stemmer.rb
+++ b/app/models/search/stemmer.rb
@@ -5,9 +5,62 @@ module Search::Stemmer
def stem(value)
if value.present?
- value.gsub(/[^\w\s]/, " ").split(/\s+/).map { |word| STEMMER.stem(word.downcase) }.join(" ")
+ tokenize(value).join(" ")
else
value
end
end
+
+ def stem_query(value)
+ if value.present?
+ value.gsub(/"[^"]*"|\S+/) do |token|
+ if token.start_with?('"') && token.end_with?('"')
+ "\"#{stem(token[1..-2])}\""
+ else
+ stemmed = stem(token)
+ if stemmed.include?(" ") && token.match?(Search::CJK_PATTERN)
+ "\"#{stemmed}\""
+ else
+ stemmed
+ end
+ end
+ end
+ else
+ value
+ end
+ end
+
+ private
+ def tokenize(value)
+ tokens = []
+ current_word = +""
+
+ value.each_char do |char|
+ if cjk_character?(char)
+ if current_word.present?
+ tokens << stem_word(current_word)
+ current_word = +""
+ end
+ tokens << char
+ elsif char =~ /[\p{L}\p{M}\p{N}_]/
+ current_word << char
+ else
+ if current_word.present?
+ tokens << stem_word(current_word)
+ current_word = +""
+ end
+ end
+ end
+
+ tokens << stem_word(current_word) if current_word.present?
+ tokens
+ end
+
+ def cjk_character?(char)
+ char.match?(Search::CJK_PATTERN)
+ end
+
+ def stem_word(word)
+ STEMMER.stem(word.downcase)
+ end
end
diff --git a/db/migrate/20260304120000_change_search_fts_tokenizer_to_unicode61.rb b/db/migrate/20260304120000_change_search_fts_tokenizer_to_unicode61.rb
new file mode 100644
index 0000000000..dcbf00df37
--- /dev/null
+++ b/db/migrate/20260304120000_change_search_fts_tokenizer_to_unicode61.rb
@@ -0,0 +1,48 @@
+class ChangeSearchFtsTokenizerToUnicode61 < ActiveRecord::Migration[8.2]
+ def up
+ return unless connection.adapter_name == "SQLite"
+
+ execute "DROP TABLE IF EXISTS search_records_fts"
+ execute <<~SQL
+ CREATE VIRTUAL TABLE search_records_fts USING fts5(
+ title,
+ content,
+ tokenize='unicode61'
+ )
+ SQL
+
+ reindex_fts_records
+ end
+
+ def down
+ return unless connection.adapter_name == "SQLite"
+
+ execute "DROP TABLE IF EXISTS search_records_fts"
+ execute <<~SQL
+ CREATE VIRTUAL TABLE search_records_fts USING fts5(
+ title,
+ content,
+ tokenize='porter'
+ )
+ SQL
+
+ # Insert original (unstemmed) text — porter tokenizer handles stemming itself
+ connection.select_all("SELECT id, title, content FROM search_records").each do |row|
+ connection.execute(
+ "INSERT INTO search_records_fts(rowid, title, content) VALUES (#{connection.quote(row['id'])}, #{connection.quote(row['title'])}, #{connection.quote(row['content'])})"
+ )
+ end
+ end
+
+ private
+ def reindex_fts_records
+ connection.select_all("SELECT id, title, content FROM search_records").each do |row|
+ stemmed_title = Search::Stemmer.stem(row["title"])
+ stemmed_content = Search::Stemmer.stem(row["content"])
+
+ connection.execute(
+ "INSERT INTO search_records_fts(rowid, title, content) VALUES (#{connection.quote(row['id'])}, #{connection.quote(stemmed_title)}, #{connection.quote(stemmed_content)})"
+ )
+ end
+ end
+end
diff --git a/db/schema_sqlite.rb b/db/schema_sqlite.rb
index 9e4f050dc2..afe7dc01cf 100644
--- a/db/schema_sqlite.rb
+++ b/db/schema_sqlite.rb
@@ -630,6 +630,6 @@
t.index ["account_id"], name: "index_webhooks_on_account_id"
t.index ["board_id", "subscribed_actions"], name: "index_webhooks_on_board_id_and_subscribed_actions"
end
- execute "CREATE VIRTUAL TABLE search_records_fts USING fts5(\n title,\n content,\n tokenize='porter'\n )"
+ execute "CREATE VIRTUAL TABLE search_records_fts USING fts5(\n title,\n content,\n tokenize='unicode61'\n)"
end
diff --git a/test/controllers/searches_controller_test.rb b/test/controllers/searches_controller_test.rb
index b0172cdf51..0b723478df 100644
--- a/test/controllers/searches_controller_test.rb
+++ b/test/controllers/searches_controller_test.rb
@@ -83,6 +83,48 @@ class SearchesControllerTest < ActionDispatch::IntegrationTest
assert_response :success
end
+ test "search with CJK content" do
+ skip "MySQL fulltext requires tokens of 3+ characters" unless ActiveRecord::Base.connection.adapter_name == "SQLite"
+
+ @board.cards.create!(title: "中文测试卡片", description: "这是中文描述", status: "published", creator: @user)
+ @board.cards.create!(title: "日本語テスト", description: "日本語の説明", status: "published", creator: @user)
+ @board.cards.create!(title: "한국어 테스트", description: "한국어 설명", status: "published", creator: @user)
+ @board.cards.create!(title: "Mixed 混合 content", description: "English and 中文 together", status: "published", creator: @user)
+
+ # Chinese search
+ get search_path(q: "中文", script_name: "/#{@account.external_account_id}")
+ assert_select "li .search__title", text: /中文测试卡片/
+ assert_match(/<\/span>中文<\/mark>/, response.body)
+
+ # Japanese search
+ get search_path(q: "日本語", script_name: "/#{@account.external_account_id}")
+ assert_select "li .search__title", text: /日本語テスト/
+
+ # Korean search
+ get search_path(q: "한국어", script_name: "/#{@account.external_account_id}")
+ assert_select "li .search__title", text: /한국어 테스트/
+
+ # Mixed CJK/English search
+ get search_path(q: "混合", script_name: "/#{@account.external_account_id}")
+ assert_select "li .search__title", text: /Mixed 混合 content/
+ end
+
+ test "search with accented characters" do
+ @board.cards.create!(title: "Hälsa och friskvård", description: "Möte i Göteborg", status: "published", creator: @user)
+
+ # Accented characters must survive query sanitization
+ get search_path(q: "hälsa", script_name: "/#{@account.external_account_id}")
+ assert_select "li .search__title", text: /Hälsa och friskvård/
+ assert_match(/<\/span>Hälsa<\/mark>/, response.body)
+
+ get search_path(q: "Göteborg", script_name: "/#{@account.external_account_id}")
+ assert_select "li .search__excerpt", text: /Möte i Göteborg/
+
+ # Decomposed form (a + combining diaeresis) must survive sanitization and still match
+ get search_path(q: "ha\u0308lsa", script_name: "/#{@account.external_account_id}")
+ assert_select "li .search__title", text: /Hälsa och friskvård/
+ end
+
test "search preserves highlight marks but escapes surrounding HTML" do
@board.cards.create!(
title: "Bold testing content",
diff --git a/test/models/card/searchable_test.rb b/test/models/card/searchable_test.rb
index bf3de621f0..73450fbc00 100644
--- a/test/models/card/searchable_test.rb
+++ b/test/models/card/searchable_test.rb
@@ -72,7 +72,8 @@ class Card::SearchableTest < ActiveSupport::TestCase
if search_record_class.connection.adapter_name == "SQLite"
fts_entry = search_record.search_records_fts
assert_not_nil fts_entry, "FTS entry should exist"
- assert_equal card.title, fts_entry.title
+ # FTS stores stemmed content for search matching, not original text
+ assert_equal Search::Stemmer.stem(card.title), fts_entry.title
end
# Delete the card
diff --git a/test/models/search/highlighter_test.rb b/test/models/search/highlighter_test.rb
index a065c4da19..ee36e1f8f7 100644
--- a/test/models/search/highlighter_test.rb
+++ b/test/models/search/highlighter_test.rb
@@ -29,9 +29,9 @@ class Search::HighlighterTest < ActiveSupport::TestCase
assert_equal "Say #{mark('hello world')} to everyone", result
end
- test "snippet returns full text with highlights when under max words" do
+ test "snippet returns full text with highlights when under limit" do
highlighter = Search::Highlighter.new("ruby")
- result = highlighter.snippet("Ruby is great", max_words: 20)
+ result = highlighter.snippet("Ruby is great", max_chars: 100)
assert_equal "#{mark('Ruby')} is great", result
end
@@ -39,7 +39,7 @@ class Search::HighlighterTest < ActiveSupport::TestCase
test "snippet creates excerpt around match" do
highlighter = Search::Highlighter.new("match")
text = "word " * 10 + "match " + "word " * 10
- result = highlighter.snippet(text, max_words: 10)
+ result = highlighter.snippet(text, max_chars: 50)
assert result.start_with?("...")
assert result.end_with?("...")
@@ -49,7 +49,7 @@ class Search::HighlighterTest < ActiveSupport::TestCase
test "snippet adds leading ellipsis when match is not at start" do
highlighter = Search::Highlighter.new("middle")
text = "word " * 20 + "middle"
- result = highlighter.snippet(text, max_words: 10)
+ result = highlighter.snippet(text, max_chars: 50)
assert result.start_with?("...")
assert_not result.end_with?("...")
@@ -59,7 +59,7 @@ class Search::HighlighterTest < ActiveSupport::TestCase
test "snippet adds trailing ellipsis when text continues after excerpt" do
highlighter = Search::Highlighter.new("start")
text = "start " + "word " * 30
- result = highlighter.snippet(text, max_words: 10)
+ result = highlighter.snippet(text, max_chars: 50)
assert result.end_with?("...")
assert_not result.start_with?("...")
@@ -69,7 +69,7 @@ class Search::HighlighterTest < ActiveSupport::TestCase
test "snippet falls back to truncation when no match found" do
highlighter = Search::Highlighter.new("nomatch")
text = "This text does not contain the search term " + "word " * 50
- result = highlighter.snippet(text, max_words: 10)
+ result = highlighter.snippet(text, max_chars: 50)
assert_includes result, "..."
assert_not_includes result, Search::Highlighter::OPENING_MARK
@@ -82,6 +82,95 @@ class Search::HighlighterTest < ActiveSupport::TestCase
assert_equal "<script>#{mark('test')}</script>", result
end
+ test "highlight CJK text" do
+ highlighter = Search::Highlighter.new("中文")
+ result = highlighter.highlight("这是中文测试")
+
+ assert_equal "这是#{mark('中文')}测试", result
+ end
+
+ test "highlight Japanese text" do
+ highlighter = Search::Highlighter.new("日本")
+ result = highlighter.highlight("これは日本語です")
+
+ assert_equal "これは#{mark('日本')}語です", result
+ end
+
+ test "highlight Korean text" do
+ highlighter = Search::Highlighter.new("한국")
+ result = highlighter.highlight("이것은 한국어입니다")
+
+ assert_equal "이것은 #{mark('한국')}어입니다", result
+ end
+
+ test "highlight mixed CJK and English" do
+ highlighter = Search::Highlighter.new("test 中文")
+ result = highlighter.highlight("This is a test about 中文内容")
+
+ assert_equal "This is a #{mark('test')} about #{mark('中文')}内容", result
+ end
+
+ test "snippet handles CJK text without spaces" do
+ highlighter = Search::Highlighter.new("中文")
+ text = "这是一段很长的中文文本用于测试摘要功能是否正常工作"
+ result = highlighter.snippet(text, max_chars: 100)
+
+ assert_includes result, mark("中文")
+ end
+
+ test "snippet truncates long CJK text around match" do
+ highlighter = Search::Highlighter.new("目标")
+ text = "前面有很多很多很多很多很多的文字内容" + "目标词汇" + "后面也有很多很多很多很多很多的文字内容"
+ result = highlighter.snippet(text, max_chars: 30)
+
+ assert_includes result, mark("目标")
+ assert result.start_with?("...")
+ assert result.end_with?("...")
+ end
+
+ test "highlight stems terms for better matching" do
+ highlighter = Search::Highlighter.new("running")
+ result = highlighter.highlight("I like to run every day")
+
+ assert_equal "I like to #{mark('run')} every day", result
+ end
+
+ test "snippet finds match case-insensitively" do
+ highlighter = Search::Highlighter.new("test")
+ text = "これは非常に長い日本語のテキストでTESTという単語を含む" * 3
+ result = highlighter.snippet(text, max_chars: 30)
+
+ assert_includes result, mark("TEST")
+ end
+
+ test "highlight Latin terms adjacent to CJK characters" do
+ highlighter = Search::Highlighter.new("test")
+ result = highlighter.highlight("日本語TESTテスト")
+
+ assert_equal "日本語#{mark('TEST')}テスト", result
+ end
+
+ test "highlight mixed CJK and Latin term case-insensitively" do
+ highlighter = Search::Highlighter.new("日本語test")
+ result = highlighter.highlight("これは日本語TESTです")
+
+ assert_equal "これは#{mark('日本語TEST')}です", result
+ end
+
+ test "highlight does not corrupt markup when a term matches inserted HTML" do
+ highlighter = Search::Highlighter.new("test class")
+ result = highlighter.highlight("test class here")
+
+ assert_equal "#{mark('test')} #{mark('class')} here", result
+ end
+
+ test "highlight terms matching mark tag attributes" do
+ highlighter = Search::Highlighter.new("some text")
+ result = highlighter.highlight("some text to mark up")
+
+ assert_equal "#{mark('some')} #{mark('text')} to mark up", result
+ end
+
private
def mark(text)
"#{Search::Highlighter::OPENING_MARK}#{text}#{Search::Highlighter::CLOSING_MARK}"
diff --git a/test/models/search/stemmer_test.rb b/test/models/search/stemmer_test.rb
index 33334ada4c..d2af254c31 100644
--- a/test/models/search/stemmer_test.rb
+++ b/test/models/search/stemmer_test.rb
@@ -24,4 +24,77 @@ class Search::StemmerTest < ActiveSupport::TestCase
assert_equal "foo bar", result
end
+
+ test "stem CJK characters" do
+ result = Search::Stemmer.stem("测试中文")
+
+ assert_equal "测 试 中 文", result
+ end
+
+ test "stem Japanese characters" do
+ result = Search::Stemmer.stem("日本語テスト")
+
+ assert_equal "日 本 語 テ ス ト", result
+ end
+
+ test "stem Korean characters" do
+ result = Search::Stemmer.stem("한국어테스트")
+
+ assert_equal "한 국 어 테 스 트", result
+ end
+
+ test "stem mixed CJK and English" do
+ result = Search::Stemmer.stem("hello世界test")
+
+ assert_equal "hello 世 界 test", result
+ end
+
+ test "stem mixed with English stemming" do
+ result = Search::Stemmer.stem("running 测试 jumping")
+
+ assert_equal "run 测 试 jump", result
+ end
+
+ test "stem_query preserves quoted phrases" do
+ result = Search::Stemmer.stem_query('"hello world" running')
+
+ assert_equal '"hello world" run', result
+ end
+
+ test "stem_query stems words inside quotes" do
+ result = Search::Stemmer.stem_query('"running tests"')
+
+ assert_equal '"run test"', result
+ end
+
+ test "stem_query handles CJK with quotes" do
+ result = Search::Stemmer.stem_query('"测试" running')
+
+ assert_equal '"测 试" run', result
+ end
+
+ test "stem_query without quotes behaves like stem" do
+ result = Search::Stemmer.stem_query("running jumping")
+
+ assert_equal "run jump", result
+ end
+
+ test "stem_query wraps unquoted CJK tokens as phrase" do
+ result = Search::Stemmer.stem_query("中文 running")
+
+ assert_equal '"中 文" run', result
+ end
+
+ test "stem preserves accented characters" do
+ result = Search::Stemmer.stem("Hälsa i Göteborg")
+
+ assert_equal "hälsa i göteborg", result
+ end
+
+ test "stem preserves combining marks" do
+ # "cafe\u0301" in NFD form (e + combining acute accent) and Devanagari with matras
+ result = Search::Stemmer.stem("cafe\u0301 हिंदी")
+
+ assert_equal "cafe\u0301 हिंदी", result
+ end
end