Add CJK (Chinese, Japanese, Korean) search support

app/models/search.rb

@@ -1,4 +1,6 @@
 module Search
+  CJK_PATTERN = /\p{Han}|\p{Hiragana}|\p{Katakana}|\p{Hangul}/
+
   def self.table_name_prefix
     "search_"
   end

app/models/search/highlighter.rb

@@ -1,6 +1,10 @@
 class Search::Highlighter
   OPENING_MARK = "<mark class=\"circled-text\"><span></span>"
   CLOSING_MARK = "</mark>"
+  # Private-use characters mark matches during term passes, so a later term
+  # can't match inside HTML inserted for an earlier one
+  OPENING_SENTINEL = "\uE000"
+  CLOSING_SENTINEL = "\uE001"
   ELIPSIS = "..."
 
   attr_reader :query
@@ -10,34 +14,37 @@ def initialize(query)
   end
 
   def highlight(text)
-    result = text.dup
+    result = text.delete(OPENING_SENTINEL + CLOSING_SENTINEL)
 
     terms.each do |term|
-      result.gsub!(/\b(#{Regexp.escape(term)}\w*)\b/i) do |match|
-        "#{OPENING_MARK}#{match}#{CLOSING_MARK}"
+      if term.match?(Search::CJK_PATTERN)
+        result.gsub!(/(#{Regexp.escape(term)})/i) do |match|
+          "#{OPENING_SENTINEL}#{match}#{CLOSING_SENTINEL}"
+        end
+      else
+        result.gsub!(/(?<![a-zA-Z0-9_])(#{Regexp.escape(term)}[a-zA-Z0-9_]*)(?![a-zA-Z0-9_])/i) do |match|
+          "#{OPENING_SENTINEL}#{match}#{CLOSING_SENTINEL}"
+        end
       end
     end
 
     escape_highlight_marks(result)
   end
 
-  def snippet(text, max_words: 20)
-    words = text.split(/\s+/)
-    match_index = words.index { |word| terms.any? { |term| word.downcase.include?(term.downcase) } }
-
-    if words.length <= max_words
+  def snippet(text, max_chars: 100)
+    if text.length <= max_chars
       highlight(text)
-    elsif match_index
-      start_index = [ 0, match_index - max_words / 2 ].max
-      end_index = [ words.length - 1, start_index + max_words - 1 ].min
+    elsif (match_index = first_match_position(text))
+      start_index = [ 0, match_index - max_chars / 2 ].max
+      end_index = [ text.length, start_index + max_chars ].min
 
-      snippet_text = words[start_index..end_index].join(" ")
-      snippet_text = "...#{snippet_text}" if start_index > 0
-      snippet_text = "#{snippet_text}..." if end_index < words.length - 1
+      snippet_text = text[start_index...end_index]
+      snippet_text = "#{ELIPSIS}#{snippet_text}" if start_index > 0
+      snippet_text = "#{snippet_text}#{ELIPSIS}" if end_index < text.length
 
       highlight(snippet_text)
     else
-      text.truncate_words(max_words, omission: "...")
+      "#{text[0, max_chars]}#{ELIPSIS}"
     end
   end
 
@@ -52,17 +59,35 @@ def terms
 
         unquoted = query.gsub(/"[^"]+"/, "")
         unquoted.split(/\s+/).each do |word|
-          terms << word if word.present?
+          next unless word.present?
+
+          if word.match?(Search::CJK_PATTERN)
+            terms << word
+          else
+            stemmed = Search::Stemmer.stem(word)
+            terms << stemmed
+            terms << word.downcase unless word.downcase.start_with?(stemmed)
+          end
         end
 
         terms.uniq
       end
     end
 
+    def first_match_position(text)
+      terms.filter_map do |term|
+        if term.match?(Search::CJK_PATTERN)
+          text =~ /#{Regexp.escape(term)}/i
+        else
+          text =~ /(?<![a-zA-Z0-9_])#{Regexp.escape(term)}/i
+        end
+      end.min
+    end
+
     def escape_highlight_marks(html)
       CGI.escapeHTML(html)
-        .gsub(CGI.escapeHTML(OPENING_MARK), OPENING_MARK.html_safe)
-        .gsub(CGI.escapeHTML(CLOSING_MARK), CLOSING_MARK.html_safe)
+        .gsub(OPENING_SENTINEL, OPENING_MARK)
+        .gsub(CLOSING_SENTINEL, CLOSING_MARK)
         .html_safe
     end
 end

app/models/search/query.rb

@@ -33,7 +33,7 @@ def sanitize(terms)
     end
 
     def remove_invalid_search_characters(terms)
-      terms.gsub(/[^\w"]/, " ")
+      terms.gsub(/[^\p{L}\p{M}\p{N}_"]/, " ")
     end
 
     def remove_unbalanced_quotes(terms)

app/models/search/record/sqlite.rb

@@ -2,29 +2,20 @@ module Search::Record::SQLite
   extend ActiveSupport::Concern
 
   included do
-    attribute :result_title, :string
-    attribute :result_content, :string
-
     has_one :search_records_fts, -> { with_rowid },
       class_name: "Search::Record::SQLite::Fts", foreign_key: :rowid, primary_key: :id, dependent: :destroy
 
     after_save :upsert_to_fts5_table
 
     scope :matching, ->(query, account_id) {
       joins("INNER JOIN search_records_fts ON search_records_fts.rowid = #{table_name}.id")
-        .where("search_records_fts MATCH ?", query)
+        .where("search_records_fts MATCH ?", Search::Stemmer.stem_query(query.to_s))
     }
   end
 
   class_methods do
     def search_fields(query)
-      opening_mark = connection.quote(Search::Highlighter::OPENING_MARK)
-      closing_mark = connection.quote(Search::Highlighter::CLOSING_MARK)
-      ellipsis = connection.quote(Search::Highlighter::ELIPSIS)
-
-      [ "highlight(search_records_fts, 0, #{opening_mark}, #{closing_mark}) AS result_title",
-        "snippet(search_records_fts, 1, #{opening_mark}, #{closing_mark}, #{ellipsis}, 20) AS result_content",
-        "#{connection.quote(query.terms)} AS query" ]
+      "#{connection.quote(query.terms)} AS query"
     end
 
     def for(account_id)
@@ -33,28 +24,35 @@ def for(account_id)
   end
 
   def card_title
-    escape_fts_highlight(result_title || card.title)
+    highlight_full(card.title) if card_id
   end
 
   def card_description
-    escape_fts_highlight(result_content) unless comment
+    highlight_snippet(card.description.to_plain_text) if card_id
   end
 
   def comment_body
-    escape_fts_highlight(result_content) if comment
+    highlight_snippet(comment.body.to_plain_text) if comment
   end
 
   private
-    def escape_fts_highlight(html)
-      return nil unless html.present?
+    def highlight_snippet(text)
+      if text.present? && attribute?(:query)
+        Search::Highlighter.new(query).snippet(text)
+      else
+        text
+      end
+    end
 
-      CGI.escapeHTML(html)
-        .gsub(CGI.escapeHTML(Search::Highlighter::OPENING_MARK), Search::Highlighter::OPENING_MARK)
-        .gsub(CGI.escapeHTML(Search::Highlighter::CLOSING_MARK), Search::Highlighter::CLOSING_MARK)
-        .html_safe
+    def highlight_full(text)
+      if text.present? && attribute?(:query)
+        Search::Highlighter.new(query).highlight(text)
+      else
+        text
+      end
     end
 
     def upsert_to_fts5_table
-      Fts.upsert(id, title, content)
+      Fts.upsert(id, Search::Stemmer.stem(title), Search::Stemmer.stem(content))
     end
 end

app/models/search/record/trilogy.rb

@@ -8,7 +8,7 @@ module Search::Record::Trilogy
     before_save :set_account_key, :stem_content
 
     scope :matching, ->(query, account_id) do
-      full_query = "+account#{account_id} +(#{Search::Stemmer.stem(query)})"
+      full_query = "+account#{account_id} +(#{Search::Stemmer.stem_query(query)})"
       where("MATCH(#{table_name}.account_key, #{table_name}.content, #{table_name}.title) AGAINST(? IN BOOLEAN MODE)", full_query)
     end
 
@@ -38,31 +38,38 @@ def for(account_id)
   end
 
   def card_title
-    highlight(card.title, show: :full) if card_id
+    highlight_full(card.title) if card_id
   end
 
   def card_description
-    highlight(card.description.to_plain_text, show: :snippet) if card_id
+    highlight_snippet(card.description.to_plain_text) if card_id
   end
 
   def comment_body
-    highlight(comment.body.to_plain_text, show: :snippet) if comment
+    highlight_snippet(comment.body.to_plain_text) if comment
   end
 
   private
     def stem_content
-      self.title = Search::Stemmer.stem(title) if title_changed?
+      self.title = Search::Stemmer.stem(title).truncate(255, omission: "") if title_changed?
       self.content = Search::Stemmer.stem(content) if content_changed?
     end
 
     def set_account_key
       self.account_key = "account#{account_id}"
     end
 
-    def highlight(text, show:)
+    def highlight_snippet(text)
       if text.present? && attribute?(:query)
-        highlighter = Search::Highlighter.new(query)
-        show == :snippet ? highlighter.snippet(text) : highlighter.highlight(text)
+        Search::Highlighter.new(query).snippet(text)
+      else
+        text
+      end
+    end
+
+    def highlight_full(text)
+      if text.present? && attribute?(:query)
+        Search::Highlighter.new(query).highlight(text)
       else
         text
       end

app/models/search/stemmer.rb

@@ -5,9 +5,62 @@ module Search::Stemmer
 
   def stem(value)
     if value.present?
-      value.gsub(/[^\w\s]/, " ").split(/\s+/).map { |word| STEMMER.stem(word.downcase) }.join(" ")
+      tokenize(value).join(" ")
     else
       value
     end
   end
+
+  def stem_query(value)
+    if value.present?
+      value.gsub(/"[^"]*"|\S+/) do |token|
+        if token.start_with?('"') && token.end_with?('"')
+          "\"#{stem(token[1..-2])}\""
+        else
+          stemmed = stem(token)
+          if stemmed.include?(" ") && token.match?(Search::CJK_PATTERN)
+            "\"#{stemmed}\""
+          else
+            stemmed
+          end
+        end
+      end
+    else
+      value
+    end
+  end
+
+  private
+    def tokenize(value)
+      tokens = []
+      current_word = +""
+
+      value.each_char do |char|
+        if cjk_character?(char)
+          if current_word.present?
+            tokens << stem_word(current_word)
+            current_word = +""
+          end
+          tokens << char
+        elsif char =~ /[\p{L}\p{M}\p{N}_]/
+          current_word << char
+        else
+          if current_word.present?
+            tokens << stem_word(current_word)
+            current_word = +""
+          end
+        end
+      end
+
+      tokens << stem_word(current_word) if current_word.present?
+      tokens
+    end
+
+    def cjk_character?(char)
+      char.match?(Search::CJK_PATTERN)
+    end
+
+    def stem_word(word)
+      STEMMER.stem(word.downcase)
+    end
 end

db/migrate/20260304120000_change_search_fts_tokenizer_to_unicode61.rb

@@ -0,0 +1,48 @@
+class ChangeSearchFtsTokenizerToUnicode61 < ActiveRecord::Migration[8.2]
+  def up
+    return unless connection.adapter_name == "SQLite"
+
+    execute "DROP TABLE IF EXISTS search_records_fts"
+    execute <<~SQL
+      CREATE VIRTUAL TABLE search_records_fts USING fts5(
+        title,
+        content,
+        tokenize='unicode61'
+      )
+    SQL
+
+    reindex_fts_records
+  end
+
+  def down
+    return unless connection.adapter_name == "SQLite"
+
+    execute "DROP TABLE IF EXISTS search_records_fts"
+    execute <<~SQL
+      CREATE VIRTUAL TABLE search_records_fts USING fts5(
+        title,
+        content,
+        tokenize='porter'
+      )
+    SQL
+
+    # Insert original (unstemmed) text — porter tokenizer handles stemming itself
+    connection.select_all("SELECT id, title, content FROM search_records").each do |row|
+      connection.execute(
+        "INSERT INTO search_records_fts(rowid, title, content) VALUES (#{connection.quote(row['id'])}, #{connection.quote(row['title'])}, #{connection.quote(row['content'])})"
+      )
+    end
+  end
+
+  private
+    def reindex_fts_records
+      connection.select_all("SELECT id, title, content FROM search_records").each do |row|
+        stemmed_title = Search::Stemmer.stem(row["title"])
+        stemmed_content = Search::Stemmer.stem(row["content"])
+
+        connection.execute(
+          "INSERT INTO search_records_fts(rowid, title, content) VALUES (#{connection.quote(row['id'])}, #{connection.quote(stemmed_title)}, #{connection.quote(stemmed_content)})"
+        )
+      end
+    end
+end

db/schema_sqlite.rb

@@ -630,6 +630,6 @@
     t.index ["account_id"], name: "index_webhooks_on_account_id"
     t.index ["board_id", "subscribed_actions"], name: "index_webhooks_on_board_id_and_subscribed_actions"
   end
-  execute "CREATE VIRTUAL TABLE search_records_fts USING fts5(\n        title,\n        content,\n        tokenize='porter'\n      )"
+  execute "CREATE VIRTUAL TABLE search_records_fts USING fts5(\n  title,\n  content,\n  tokenize='unicode61'\n)"
 
 end