From 207b3d8c4b18d1ef9a7dc3ab43c27152ac8ae1fd Mon Sep 17 00:00:00 2001 From: Tsubasa SEKIGUCHI Date: Sun, 12 Jul 2026 20:51:54 +0900 Subject: [PATCH 1/3] =?UTF-8?q?=E4=BA=AC=E7=8E=8B=E3=83=90=E3=82=B9?= =?UTF-8?q?=E3=83=BB=E6=9D=B1=E6=80=A5=E3=83=90=E3=82=B9=E3=81=AE=E8=8B=B1?= =?UTF-8?q?=E8=AA=9E=E8=A1=A8=E8=A8=98=E3=82=92=E3=82=AB=E3=83=8A=E8=AA=AD?= =?UTF-8?q?=E3=81=BF=E3=81=8B=E3=82=89=E8=A3=9C=E5=AE=8C=20(#1604)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 京王バス・東急バスの英語表記をカナ読みからヘボン式で補完 京王バスGTFSと東急バスODPT JSONは停留所ごとのカナ読みは持つが英語 (en)表記を持たないため、stop_name_r が空のままだった。gtfs_stops. stop_name_r は駅名projection・名称検索・ローマ字系統/行先名すべての 上流ソースであるため、ここが空だと英語表示面が一律に欠落する。 domain/romaji.rs を追加し、カナ読みを修正ヘボン式ローマ字へ変換する。 長音はrailデータの表記(Tōkyō / Kyōto / Shin-Ōsaka)に合わせマクロンを 用い、オウ/オオ→ō・拗音のシュウ/チュウ→ū・ーで前母音を伸ばす。形態素 境界の母音連続(ヤスウシ=Yasuushi 等)はカナだけでは長音と区別できない ため字義通り残す。変換不能な文字(漢字・ラテン文字・数字)を含む場合は None を返し、部分的な誤変換を出さない。 import.rs では en が無い場合のみ stop_name_r をこのローマ字で補完する (GTFS経路=京王等、Tokyu JSON経路の両方)。実際の en 値は上書きしない。 生成したマクロン付きローマ字は既存の romaji→IPA/TTS 経路 (normalize_name_char がマクロンを展開)と整合し、失敗時もカナからの IPA へ graceful fallback するため回帰は無い。 Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01VJ8JoyxRvZsX6XfEKuQEwZ * バス駅の station_name_rn をマクロンなしローマ字で補完 前コミットで station_name_r をマクロン付きヘボン式(Tōkyō)で補完したが、 railデータの規約では station_name_rn にマクロンなし版(Tokyo)が入る。バス projection は _rn を設定しておらず NULL のままだった。 domain/romaji.rs に strip_macrons() を追加(ō→o, ū→u, ā→a, ī→i, ē→e)。 integrate_gtfs_stops_to_stations で stop_name_r から _rn を導出して INSERT に追加する。en 由来・カナ由来いずれの _r でも同じ規約で _rn を生成する。 Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01VJ8JoyxRvZsX6XfEKuQEwZ --------- Co-authored-by: Claude --- AGENTS.md | 1 + stationapi/src/domain.rs | 1 + stationapi/src/domain/romaji.rs | 568 ++++++++++++++++++++++++++++++++ stationapi/src/import.rs | 35 +- 4 files changed, 599 insertions(+), 6 deletions(-) create mode 100644 stationapi/src/domain/romaji.rs diff --git a/AGENTS.md b/AGENTS.md index 765dcc6f..5122d1b4 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -55,6 +55,7 @@ This guide explains how automation agents and human contributors should work wit - **Routes** – `GetRoutes`, `GetRoutesMinimal`. The minimal variant returns `RouteMinimalResponse` with deduplicated `LineMinimal` data; paging tokens are currently empty (pagination not implemented). - **Train types** – `GetTrainTypesByStationId`, `GetRouteTypes`. Train types aggregate by line group and include related lines plus optional train type metadata. Rail variants use `TrainTypeKind::{Default, Branch, Rapid, Express, LimitedExpress, HighSpeedRapid, CommuterRapid}` (0-6); bus variants use `BusRoute` (7), which represents a `(route_id, shape_id)` operation pattern (e.g. 循環 / 短ターン / 支線) generated automatically from the configured GTFS bus feeds (Toei Bus, Seibu Bus, Keio Bus) and the converted Tokyu Bus JSON. - **GTFS bus integration** – At startup, `src/import.rs::integrate_gtfs_to_stations()` ingests GTFS feeds into `gtfs_*` tables and then projects them onto the shared `stations` / `lines` / `types` / `station_station_types` tables. Every configured GTFS feed is imported, including Seibu Bus and Keio Bus (both downloaded from ODPT with `ODPT_ACCESS_TOKEN`). Tokyu Bus ordinary-route `BusroutePattern`, `BusstopPole`, and `BusTimetable` JSON are converted into the same `gtfs_*` representation; pattern IDs become `shape_id` values so route variants remain queryable as bus TrainTypes. The Tokyu-operated Ota, Shinagawa, and Meguro community buses use their official GTFS feeds and matching JSON routes are excluded to prevent duplicates. `ODPT_ACCESS_TOKEN` is required for authenticated sources. Stops whose Tokyu JSON records omit coordinates remain available to name and route queries but not coordinate searches. `transport_type` (0: rail, 1: bus) on both `stations` and `lines` keeps rail and bus records queryable side by side. GTFS IDs are namespaced per feed before import to avoid cross-operator collisions. `line_cd` (100,000,000+), `station_cd` / `station_g_cd` (200,000,000+), and bus `type_cd` / `line_group_cd` (100,000,000+) are all deterministic fnv1a hashes that stay clear of the rail data ranges. Disable the entire bus pipeline with `DISABLE_BUS_FEATURE=true`. +- **Bus English-name fallback** – Keio Bus GTFS and Tokyu Bus JSON ship a kana reading for every stop but no English (`en`) name, so `gtfs_stops.stop_name_r` would otherwise be empty. `src/domain/romaji.rs::romaji_display_name()` derives a modified-Hepburn romanization (with macrons for long vowels, matching the curated rail style: Tōkyō / Kyōto / Shin-Ōsaka) from that reading, and `import.rs` fills `stop_name_r` with it whenever the feed provides no `en` value. Because `stop_name_r` is the single upstream source that fans out into the `stations` projection, `search_by_name`, and the romanized bus route/headsign names, this supplements every English-facing surface at once. The fallback never overwrites a real `en` value, and a reading with no convertible kana stays `NULL` rather than emitting a partial transcription. When projecting into `stations`, `station_name_rn` is filled with the plain-ASCII spelling via `romaji::strip_macrons()` (Tōkyō → Tokyo), mirroring the rail dataset's `_r` (macron) / `_rn` (macron-free) column pair. - **TTS metadata** – `Station`, `StationMinimal`, `Line`, and `TrainType` expose `name_ipa` / `name_roman_ipa` plus `name_tts_segments` for multi-segment pronunciation output. Use `name_tts_segments` when clients need per-token SSML construction for mixed-language names such as `Kasai-Rinkai Park`. - **Connected routes** – `GetConnectedRoutes`. `QueryInteractor::get_connected_stations` is not implemented yet and returns an empty vector; update the use-case and infrastructure layers together when adding real logic. - Changes to the service contract require coordinated updates to `proto/stationapi.proto`, regenerated code via `tonic-build`, and corresponding adjustments in both presentation and use-case layers. diff --git a/stationapi/src/domain.rs b/stationapi/src/domain.rs index 0e083171..7ac5bd9a 100644 --- a/stationapi/src/domain.rs +++ b/stationapi/src/domain.rs @@ -4,5 +4,6 @@ pub mod error; pub mod ipa; pub mod normalize; pub mod repository; +pub mod romaji; pub mod segment_speed_table; pub mod speed_table; diff --git a/stationapi/src/domain/romaji.rs b/stationapi/src/domain/romaji.rs new file mode 100644 index 00000000..48878ee4 --- /dev/null +++ b/stationapi/src/domain/romaji.rs @@ -0,0 +1,568 @@ +//! Katakana → Hepburn romaji conversion for bus-stop names that ship without an +//! official English reading. +//! +//! Keio Bus (GTFS) and Tokyu Bus (ODPT JSON) provide a kana reading for every +//! stop but no English (`en`) translation, so their `stop_name_r` would otherwise +//! stay empty. Because `gtfs_stops.stop_name_r` is the single upstream source that +//! fans out into the `stations` projection, name search, and the romanized bus +//! route/headsign names, filling it here supplements every English-facing surface +//! at once. +//! +//! The curated rail dataset (`data/3!stations.csv`) uses modified Hepburn with +//! macrons for long vowels (Tōkyō, Kyōto, Shin-Ōsaka), so this converter mirrors +//! that style: +//! - long o (オウ / オオ) → `ō`, palatalised long u (シュウ / チュウ) → `ū` +//! - the chōonpu `ー` lengthens the preceding vowel +//! - `ン` is always `n`, sokuon `ッ` geminates the next consonant (`ッチ` → `tch`) +//! +//! Cross-morpheme vowel runs that the hand-curated rail data keeps literal +//! (e.g. Yasuushi = ヤスウシ, Masuura = マスウラ) cannot be told apart from true long +//! vowels using the reading alone, so a plain `ウ` only lengthens after an o-sound +//! or a palatalised u-sound and otherwise stays literal — matching the rail style +//! for the cases it can, and erring toward the literal reading for the rest. +//! +//! Anything the tables cannot map (kanji, Latin letters, digits) makes the whole +//! conversion return `None`, so callers fall back to leaving the name empty rather +//! than emitting a partial, misleading transcription. + +/// One romanized mora, tracking enough state to apply long-vowel rules. +struct Syllable { + text: String, + /// Plain base vowel (`a`/`i`/`u`/`e`/`o`) of the mora, or `None` for `ン`. + /// Kept stable even after the text is lengthened to a macron. + base: Option, + /// Whether this mora came from a palatalised (小ャ/ュ/ョ) digraph, which is + /// what distinguishes a true long `ū` (シュウ) from a literal run (スウ). + is_yoon: bool, +} + +/// Convert a katakana (or hiragana) reading to lowercase modified-Hepburn romaji. +/// +/// Returns `None` when the input is empty or contains a character outside the +/// kana tables, so the caller can decline to supplement rather than emit a +/// partial transcription. +pub fn katakana_to_romaji(input: &str) -> Option { + if input.trim().is_empty() { + return None; + } + + let chars: Vec = normalize_kana(input).chars().collect(); + let len = chars.len(); + let mut syllables: Vec = Vec::new(); + let mut pending_sokuon = false; + let mut i = 0; + + while i < len { + let c = chars[i]; + + // Word separators inside a compound reading collapse to a single space. + if is_separator(c) { + pending_sokuon = false; + push_separator(&mut syllables); + i += 1; + continue; + } + + // Chōonpu: lengthen the previous vowel to a macron. + if c == 'ー' { + lengthen_last(&mut syllables)?; + i += 1; + continue; + } + + // Sokuon: geminate the following consonant. + if c == 'ッ' { + pending_sokuon = true; + i += 1; + continue; + } + + // A bare ウ / オ can act as a long-vowel marker instead of its own mora. + if !pending_sokuon && (c == 'ウ' || c == 'オ') { + if let Some(last) = syllables.last() { + let lengthen = match c { + // オ only lengthens a preceding o-sound (オオ / コオ → ō), + // leaving cross-morpheme オ literal (シズオカ → shizuoka). + 'オ' => last.base == Some('o'), + // ウ lengthens a preceding o-sound (トウ → tō) or a + // palatalised u-sound (シュウ → shū); a plain u stays literal + // (スウ → suu). + _ => last.base == Some('o') || (last.is_yoon && last.base == Some('u')), + }; + if lengthen { + lengthen_last(&mut syllables)?; + i += 1; + continue; + } + } + // Otherwise fall through and romanize ウ / オ as a normal mora (this + // also lets ウ still open a ウィ/ウェ/ウォ digraph below). + } + + // Two-character digraphs (palatalised キャ etc. and loanword ファ etc.). + if i + 1 < len { + if let Some(romaji) = lookup_digraph(c, chars[i + 1]) { + let is_yoon = matches!(chars[i + 1], 'ャ' | 'ュ' | 'ョ'); + push_syllable(&mut syllables, romaji, is_yoon, &mut pending_sokuon); + i += 2; + continue; + } + } + + // Single kana. + if let Some(romaji) = lookup_single(c) { + push_syllable(&mut syllables, romaji, false, &mut pending_sokuon); + i += 1; + continue; + } + + // Unknown character: refuse to emit a partial transcription. + return None; + } + + let result: String = syllables.iter().map(|s| s.text.as_str()).collect(); + let result = result.trim().to_string(); + if result.is_empty() { + None + } else { + Some(result) + } +} + +/// Convert a kana reading into a Title-Cased display name suitable for the +/// romanized `stop_name_r` / `station_name_r` fields. +pub fn romaji_display_name(input: &str) -> Option { + katakana_to_romaji(input).map(|romaji| title_case(&romaji)) +} + +/// Strip macron accents from a romanized name, mapping each long vowel back to +/// its plain ASCII letter (ō → o, ū → u, ā → a, ī → i, ē → e). +/// +/// The dataset stores two romanized forms per station: `station_name_r` keeps +/// the macrons (Tōkyō) while `station_name_rn` is the plain-ASCII spelling +/// (Tokyo). This produces the latter from the former. +pub fn strip_macrons(input: &str) -> String { + input + .chars() + .map(|c| match c { + 'ā' => 'a', + 'ī' => 'i', + 'ū' => 'u', + 'ē' => 'e', + 'ō' => 'o', + 'Ā' => 'A', + 'Ī' => 'I', + 'Ū' => 'U', + 'Ē' => 'E', + 'Ō' => 'O', + other => other, + }) + .collect() +} + +/// Capitalize the first letter of every whitespace/separator-delimited word. +fn title_case(s: &str) -> String { + let mut out = String::with_capacity(s.len() + 4); + let mut capitalize_next = true; + for c in s.chars() { + if c.is_alphabetic() { + if capitalize_next { + out.extend(c.to_uppercase()); + capitalize_next = false; + } else { + out.push(c); + } + } else { + out.push(c); + capitalize_next = true; + } + } + out +} + +/// Push a romanized mora, applying a pending sokuon gemination if present. +fn push_syllable( + syllables: &mut Vec, + romaji: &str, + is_yoon: bool, + pending_sokuon: &mut bool, +) { + let mut text = romaji.to_string(); + if *pending_sokuon { + *pending_sokuon = false; + if let Some(first) = text.chars().next() { + if is_romaji_vowel(first) { + // Gemination before a vowel is meaningless; drop the sokuon. + } else if text.starts_with("ch") { + // Hepburn spells っち as "tchi", not "cchi". + text = format!("t{text}"); + } else { + text = format!("{first}{text}"); + } + } + } + let base = base_vowel(romaji); + syllables.push(Syllable { + text, + base, + is_yoon, + }); +} + +/// Emit a single space separator, coalescing consecutive separators. +fn push_separator(syllables: &mut Vec) { + if syllables.last().map(|s| s.text.as_str()) == Some(" ") { + return; + } + syllables.push(Syllable { + text: " ".to_string(), + base: None, + is_yoon: false, + }); +} + +/// Replace the final vowel of the last mora with its macron form. +/// Returns `None` when there is nothing to lengthen or the mora ends in a +/// consonant (e.g. a stray `ー` after `ン`), failing the whole conversion. +fn lengthen_last(syllables: &mut [Syllable]) -> Option<()> { + let last = syllables.last_mut()?; + let mut chars: Vec = last.text.chars().collect(); + let final_c = *chars.last()?; + let macron = match final_c { + 'a' => 'ā', + 'i' => 'ī', + 'u' => 'ū', + 'e' => 'ē', + 'o' => 'ō', + // Already long: an extra ー/ウ is redundant, keep as-is. + 'ā' | 'ī' | 'ū' | 'ē' | 'ō' => return Some(()), + _ => return None, + }; + chars.pop(); + chars.push(macron); + last.text = chars.into_iter().collect(); + Some(()) +} + +/// Base vowel of a romaji mora (its last plain vowel), or `None` for `ン`. +fn base_vowel(romaji: &str) -> Option { + romaji.chars().rev().find(|c| is_romaji_vowel(*c)) +} + +fn is_romaji_vowel(c: char) -> bool { + matches!(c, 'a' | 'i' | 'u' | 'e' | 'o') +} + +fn is_separator(c: char) -> bool { + matches!(c, '\u{30FB}' | ' ' | '\u{3000}' | '/' | '\u{FF0F}') +} + +/// Normalize hiragana to katakana so both reading styles convert identically. +fn normalize_kana(input: &str) -> String { + input + .chars() + .map(|c| { + if ('\u{3041}'..='\u{3096}').contains(&c) { + char::from_u32(c as u32 + 0x60).unwrap_or(c) + } else { + c + } + }) + .collect() +} + +/// Two-character katakana combinations (palatalised sounds and loanword kana). +fn lookup_digraph(c1: char, c2: char) -> Option<&'static str> { + let romaji = match (c1, c2) { + // Palatalised (拗音) + ('キ', 'ャ') => "kya", + ('キ', 'ュ') => "kyu", + ('キ', 'ョ') => "kyo", + ('シ', 'ャ') => "sha", + ('シ', 'ュ') => "shu", + ('シ', 'ョ') => "sho", + ('チ', 'ャ') => "cha", + ('チ', 'ュ') => "chu", + ('チ', 'ョ') => "cho", + ('ニ', 'ャ') => "nya", + ('ニ', 'ュ') => "nyu", + ('ニ', 'ョ') => "nyo", + ('ヒ', 'ャ') => "hya", + ('ヒ', 'ュ') => "hyu", + ('ヒ', 'ョ') => "hyo", + ('ミ', 'ャ') => "mya", + ('ミ', 'ュ') => "myu", + ('ミ', 'ョ') => "myo", + ('リ', 'ャ') => "rya", + ('リ', 'ュ') => "ryu", + ('リ', 'ョ') => "ryo", + ('ギ', 'ャ') => "gya", + ('ギ', 'ュ') => "gyu", + ('ギ', 'ョ') => "gyo", + ('ジ', 'ャ') | ('ヂ', 'ャ') => "ja", + ('ジ', 'ュ') | ('ヂ', 'ュ') => "ju", + ('ジ', 'ョ') | ('ヂ', 'ョ') => "jo", + ('ビ', 'ャ') => "bya", + ('ビ', 'ュ') => "byu", + ('ビ', 'ョ') => "byo", + ('ピ', 'ャ') => "pya", + ('ピ', 'ュ') => "pyu", + ('ピ', 'ョ') => "pyo", + ('フ', 'ャ') => "fya", + ('フ', 'ュ') => "fyu", + ('フ', 'ョ') => "fyo", + // Loanword combinations (外来音) + ('シ', 'ェ') => "she", + ('ジ', 'ェ') => "je", + ('チ', 'ェ') => "che", + ('テ', 'ィ') => "ti", + ('テ', 'ュ') => "tyu", + ('デ', 'ィ') => "di", + ('デ', 'ュ') => "dyu", + ('ト', 'ゥ') => "tu", + ('ド', 'ゥ') => "du", + ('フ', 'ァ') => "fa", + ('フ', 'ィ') => "fi", + ('フ', 'ェ') => "fe", + ('フ', 'ォ') => "fo", + ('ウ', 'ィ') => "wi", + ('ウ', 'ェ') => "we", + ('ウ', 'ォ') => "wo", + ('ヴ', 'ァ') => "va", + ('ヴ', 'ィ') => "vi", + ('ヴ', 'ェ') => "ve", + ('ヴ', 'ォ') => "vo", + ('ヴ', 'ュ') => "vyu", + ('ツ', 'ァ') => "tsa", + ('ツ', 'ィ') => "tsi", + ('ツ', 'ェ') => "tse", + ('ツ', 'ォ') => "tso", + ('イ', 'ェ') => "ye", + ('ク', 'ァ') => "kwa", + ('グ', 'ァ') => "gwa", + _ => return None, + }; + Some(romaji) +} + +/// Single katakana → romaji. +fn lookup_single(c: char) -> Option<&'static str> { + let romaji = match c { + 'ア' => "a", + 'イ' => "i", + 'ウ' => "u", + 'エ' => "e", + 'オ' => "o", + 'カ' => "ka", + 'キ' => "ki", + 'ク' => "ku", + 'ケ' => "ke", + 'コ' => "ko", + 'サ' => "sa", + 'シ' => "shi", + 'ス' => "su", + 'セ' => "se", + 'ソ' => "so", + 'タ' => "ta", + 'チ' => "chi", + 'ツ' => "tsu", + 'テ' => "te", + 'ト' => "to", + 'ナ' => "na", + 'ニ' => "ni", + 'ヌ' => "nu", + 'ネ' => "ne", + 'ノ' => "no", + 'ハ' => "ha", + 'ヒ' => "hi", + 'フ' => "fu", + 'ヘ' => "he", + 'ホ' => "ho", + 'マ' => "ma", + 'ミ' => "mi", + 'ム' => "mu", + 'メ' => "me", + 'モ' => "mo", + 'ヤ' => "ya", + 'ユ' => "yu", + 'ヨ' => "yo", + 'ラ' => "ra", + 'リ' => "ri", + 'ル' => "ru", + 'レ' => "re", + 'ロ' => "ro", + 'ワ' => "wa", + 'ヰ' => "wi", + 'ヱ' => "we", + 'ヲ' => "o", + 'ン' => "n", + 'ガ' => "ga", + 'ギ' => "gi", + 'グ' => "gu", + 'ゲ' => "ge", + 'ゴ' => "go", + 'ザ' => "za", + 'ジ' => "ji", + 'ズ' => "zu", + 'ゼ' => "ze", + 'ゾ' => "zo", + 'ダ' => "da", + 'ヂ' => "ji", + 'ヅ' => "zu", + 'デ' => "de", + 'ド' => "do", + 'バ' => "ba", + 'ビ' => "bi", + 'ブ' => "bu", + 'ベ' => "be", + 'ボ' => "bo", + 'パ' => "pa", + 'ピ' => "pi", + 'プ' => "pu", + 'ペ' => "pe", + 'ポ' => "po", + 'ヴ' => "vu", + // Small kana appearing on their own (rare) fall back to their full sound. + 'ァ' => "a", + 'ィ' => "i", + 'ゥ' => "u", + 'ェ' => "e", + 'ォ' => "o", + 'ャ' => "ya", + 'ュ' => "yu", + 'ョ' => "yo", + 'ヮ' => "wa", + _ => return None, + }; + Some(romaji) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn romaji(input: &str) -> String { + katakana_to_romaji(input).expect("expected convertible katakana") + } + + #[test] + fn basic_gojuon() { + assert_eq!(romaji("シブヤ"), "shibuya"); + assert_eq!(romaji("シナガワ"), "shinagawa"); + assert_eq!(romaji("ナゴヤ"), "nagoya"); + assert_eq!(romaji("オダワラ"), "odawara"); + } + + #[test] + fn long_o_uses_macron_like_rail_data() { + assert_eq!(romaji("トウキョウ"), "tōkyō"); + assert_eq!(romaji("キョウト"), "kyōto"); + assert_eq!(romaji("シンオオサカ"), "shinōsaka"); + assert_eq!(romaji("コウベ"), "kōbe"); + } + + #[test] + fn palatalised_long_u_uses_macron() { + assert_eq!(romaji("カゴシマチュウオウ"), "kagoshimachūō"); + assert_eq!(romaji("マシュウ"), "mashū"); + assert_eq!(romaji("ジュウモンジ"), "jūmonji"); + } + + #[test] + fn cross_morpheme_u_run_stays_literal() { + // ヤスウシ (Yasuushi) / マスウラ (Masuura): the rail data keeps these literal. + assert_eq!(romaji("ヤスウシ"), "yasuushi"); + assert_eq!(romaji("マスウラ"), "masuura"); + } + + #[test] + fn cross_morpheme_o_stays_literal() { + // シズオカ (Shizuoka): オ after a u-sound is a separate mora, not a long vowel. + assert_eq!(romaji("シズオカ"), "shizuoka"); + } + + #[test] + fn ei_is_not_lengthened() { + assert_eq!(romaji("タイセイ"), "taisei"); + assert_eq!(romaji("ホウエイ"), "hōei"); + assert_eq!(romaji("シンメイ"), "shinmei"); + } + + #[test] + fn sokuon_geminates_next_consonant() { + assert_eq!(romaji("ロッポンギ"), "roppongi"); + assert_eq!(romaji("ハッサム"), "hassamu"); + // っち → tchi + assert_eq!(romaji("マッチャ"), "matcha"); + } + + #[test] + fn chouonpu_lengthens_preceding_vowel() { + assert_eq!(romaji("ビール"), "bīru"); + assert_eq!(romaji("パーク"), "pāku"); + assert_eq!(romaji("スーパー"), "sūpā"); + } + + #[test] + fn n_is_always_n() { + assert_eq!(romaji("シンバシ"), "shinbashi"); + assert_eq!(romaji("ホンマチ"), "honmachi"); + } + + #[test] + fn palatalised_and_loanword_digraphs() { + assert_eq!(romaji("トウキョウジョシダイ"), "tōkyōjoshidai"); + assert_eq!(romaji("フェリー"), "ferī"); + assert_eq!(romaji("ジェイアール"), "jeiāru"); + } + + #[test] + fn hiragana_is_accepted() { + assert_eq!(romaji("しぶや"), "shibuya"); + assert_eq!(romaji("とうきょう"), "tōkyō"); + } + + #[test] + fn separators_become_spaces() { + assert_eq!(romaji("シブヤ・エキ"), "shibuya eki"); + } + + #[test] + fn unconvertible_input_returns_none() { + assert_eq!(katakana_to_romaji("渋谷"), None); + assert_eq!(katakana_to_romaji("Shibuya"), None); + assert_eq!(katakana_to_romaji("シブヤ2"), None); + assert_eq!(katakana_to_romaji(""), None); + assert_eq!(katakana_to_romaji(" "), None); + } + + #[test] + fn strip_macrons_produces_plain_ascii_form() { + // Matches the rail dataset's _r → _rn convention. + assert_eq!(strip_macrons("Tōkyō"), "Tokyo"); + assert_eq!(strip_macrons("Kyōto"), "Kyoto"); + assert_eq!(strip_macrons("Shin-Ōsaka"), "Shin-Osaka"); + assert_eq!(strip_macrons("Mikawa-Anjō"), "Mikawa-Anjo"); + // Already plain names are unchanged. + assert_eq!(strip_macrons("Shinagawa"), "Shinagawa"); + // Derived bus romaji round-trips through both forms. + let r = romaji_display_name("トウキョウ").unwrap(); + assert_eq!(r, "Tōkyō"); + assert_eq!(strip_macrons(&r), "Tokyo"); + } + + #[test] + fn display_name_is_title_cased() { + assert_eq!(romaji_display_name("シブヤ").as_deref(), Some("Shibuya")); + assert_eq!(romaji_display_name("トウキョウ").as_deref(), Some("Tōkyō")); + assert_eq!( + romaji_display_name("シブヤエキマエ").as_deref(), + Some("Shibuyaekimae") + ); + assert_eq!( + romaji_display_name("シブヤ・エキマエ").as_deref(), + Some("Shibuya Ekimae") + ); + assert_eq!(romaji_display_name("渋谷"), None); + } +} diff --git a/stationapi/src/import.rs b/stationapi/src/import.rs index 04bff528..5770c740 100644 --- a/stationapi/src/import.rs +++ b/stationapi/src/import.rs @@ -5,6 +5,7 @@ use serde::de::{DeserializeOwned, DeserializeSeed, SeqAccess, Visitor}; use serde::{Deserialize, Serialize}; use sqlx::{Connection, PgConnection}; use stationapi::config::fetch_database_url; +use stationapi::domain::romaji::{romaji_display_name, strip_macrons}; use std::collections::{HashMap, HashSet}; use std::fmt; use std::fs::File; @@ -997,12 +998,21 @@ async fn import_tokyu_odpt_data( if stop.lat == 0.0 || stop.lon == 0.0 { missing_coordinates += 1; } + // Tokyu Bus ODPT JSON has no English field, so romanize the kana reading + // into a Hepburn stop_name_r (NULL when the reading has no convertible + // kana). This mirrors the GTFS `en` fallback and feeds the same + // English-facing station/search/route surfaces. + let katakana = hiragana_to_katakana(&stop.kana); + let stop_name_r = romaji_display_name(&katakana) + .map(|r| format!("'{}'", escape_sql_string(&r))) + .unwrap_or_else(|| "NULL".to_string()); stop_values.push(format!( - "('{}', '{}', '{}', '{}', NULL, {}, {})", + "('{}', '{}', '{}', '{}', {}, {}, {})", escape_sql_string(&stop_id), escape_sql_string(stop.number.as_deref().unwrap_or("")), escape_sql_string(&stop.title), - escape_sql_string(&hiragana_to_katakana(&stop.kana)), + escape_sql_string(&katakana), + stop_name_r, stop.lat, stop.lon )); @@ -1388,7 +1398,14 @@ async fn import_gtfs_stops( let translation = translations.get(&key); let stop_name_k = translation.and_then(|t| t.ja_hrkt.clone()); - let stop_name_r = translation.and_then(|t| t.en.clone()); + // Prefer the official English translation, but fall back to a Hepburn + // romanization of the kana reading when the feed ships no `en` row + // (e.g. Keio Bus). This keeps every English-facing surface — station + // name, name search, and romanized route/headsign names — populated. + let stop_name_r = translation + .and_then(|t| t.en.clone()) + .filter(|s| !s.is_empty()) + .or_else(|| stop_name_k.as_deref().and_then(romaji_display_name)); let stop_name_zh = translation.and_then(|t| t.zh.clone()); let stop_name_ko = translation.and_then(|t| t.ko.clone()); @@ -2895,6 +2912,11 @@ async fn integrate_gtfs_stops_to_stations( None => continue, // Skip stops not on any route }; + // station_name_rn is the plain-ASCII spelling of the romanized name + // (Tokyo), whereas station_name_r keeps the macrons (Tōkyō). Derive it + // from stop_name_r so both bus and rail rows follow the same convention. + let station_name_rn = stop.stop_name_r.as_deref().map(strip_macrons); + // Create a station record for each route this physical stop serves for (route_id, stop_sequence) in &routes { let station_cd = generate_bus_station_cd(&stop.stop_id, route_id); @@ -2903,12 +2925,12 @@ async fn integrate_gtfs_stops_to_stations( sqlx::query( r#"INSERT INTO stations ( station_cd, station_g_cd, station_name, station_name_k, - station_name_r, station_name_zh, station_name_ko, + station_name_r, station_name_rn, station_name_zh, station_name_ko, line_cd, pref_cd, post, address, lon, lat, open_ymd, close_ymd, e_status, e_sort, transport_type ) VALUES ( - $1, $2, $3, $4, $5, $6, $7, $8, 13, '', '', $9, $10, - '', '', 0, $11, 1 + $1, $2, $3, $4, $5, $6, $7, $8, $9, 13, '', '', $10, $11, + '', '', 0, $12, 1 ) ON CONFLICT (station_cd) DO NOTHING"#, ) @@ -2922,6 +2944,7 @@ async fn integrate_gtfs_stops_to_stations( .unwrap_or_else(|| stop.stop_name.clone()), ) .bind(&stop.stop_name_r) + .bind(&station_name_rn) .bind(&stop.stop_name_zh) .bind(&stop.stop_name_ko) .bind(line_cd) From 8251c5a0c26b9e1114708fd848e4eb965d840cc4 Mon Sep 17 00:00:00 2001 From: Tsubasa SEKIGUCHI Date: Sun, 12 Jul 2026 21:59:00 +0900 Subject: [PATCH 2/3] =?UTF-8?q?=E3=83=90=E3=82=B9=E5=81=9C=E3=81=AE?= =?UTF-8?q?=E7=BF=BB=E8=A8=B3=E3=82=92field=5Fvalue=E3=82=AD=E3=83=BC?= =?UTF-8?q?=E3=83=BB=E5=8D=8A=E8=A7=92=E3=82=AB=E3=83=8A=E3=81=AB=E3=82=82?= =?UTF-8?q?=E5=AF=BE=E5=BF=9C=E3=81=97=E8=8B=B1=E8=AA=9E/=E8=AA=AD?= =?UTF-8?q?=E3=81=BF=E3=82=92=E8=A3=9C=E5=AE=8C=20(#1606)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 前回のローマ字化フォールバックだけでは京王バス・東急コミュニティバスの 英語表記も読みも入らなかった。原因は GTFS-JP translations.txt のキー方式 とカラム構成がフィード毎に異なることだった: - 西武: record_id キー・6列(record_sub_idなし)・全角カナ → 従来動作 - 京王/東急(大田/品川/目黒): field_value(日本語駅名)キーで record_id 空・ 7列・ja-Hrkt が半角カナ → 従来ローダーが record.get(4) 決め打ち＆ record_id のみ参照だったため全取りこぼし。結果 station_name_k に漢字が 入り station_name_r は空になっていた。 対応: - load_gtfs_translations をヘッダー名でカラム解決するよう変更し、record_id と field_value の両方をキーとして索引。import_gtfs_stops は stop_id → 停留所名(field_value) の順で翻訳を引く。 - 半角カナ読み(ﾆｼﾊﾁｵｳｼﾞ 等、結合濁点含む)を to_fullwidth_katakana() で 全角カナに正規化してから格納。 - en があればそれを station_name_r に採用(本物の英語を優先)。en が無い フィード(東急一般路線JSON等)は従来どおりカナからローマ字生成。 実データ(ODPT)で全事業者の translations.txt 構成を確認済み。ローダーの 両キー方式・列数差・半角カナのユニットテストを追加。 Claude-Session: https://claude.ai/code/session_01VJ8JoyxRvZsX6XfEKuQEwZ Co-authored-by: Claude --- AGENTS.md | 3 +- stationapi/src/domain/romaji.rs | 172 +++++++++++++++++++++++++-- stationapi/src/import.rs | 202 ++++++++++++++++++++++---------- 3 files changed, 305 insertions(+), 72 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 5122d1b4..67127edb 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -55,7 +55,8 @@ This guide explains how automation agents and human contributors should work wit - **Routes** – `GetRoutes`, `GetRoutesMinimal`. The minimal variant returns `RouteMinimalResponse` with deduplicated `LineMinimal` data; paging tokens are currently empty (pagination not implemented). - **Train types** – `GetTrainTypesByStationId`, `GetRouteTypes`. Train types aggregate by line group and include related lines plus optional train type metadata. Rail variants use `TrainTypeKind::{Default, Branch, Rapid, Express, LimitedExpress, HighSpeedRapid, CommuterRapid}` (0-6); bus variants use `BusRoute` (7), which represents a `(route_id, shape_id)` operation pattern (e.g. 循環 / 短ターン / 支線) generated automatically from the configured GTFS bus feeds (Toei Bus, Seibu Bus, Keio Bus) and the converted Tokyu Bus JSON. - **GTFS bus integration** – At startup, `src/import.rs::integrate_gtfs_to_stations()` ingests GTFS feeds into `gtfs_*` tables and then projects them onto the shared `stations` / `lines` / `types` / `station_station_types` tables. Every configured GTFS feed is imported, including Seibu Bus and Keio Bus (both downloaded from ODPT with `ODPT_ACCESS_TOKEN`). Tokyu Bus ordinary-route `BusroutePattern`, `BusstopPole`, and `BusTimetable` JSON are converted into the same `gtfs_*` representation; pattern IDs become `shape_id` values so route variants remain queryable as bus TrainTypes. The Tokyu-operated Ota, Shinagawa, and Meguro community buses use their official GTFS feeds and matching JSON routes are excluded to prevent duplicates. `ODPT_ACCESS_TOKEN` is required for authenticated sources. Stops whose Tokyu JSON records omit coordinates remain available to name and route queries but not coordinate searches. `transport_type` (0: rail, 1: bus) on both `stations` and `lines` keeps rail and bus records queryable side by side. GTFS IDs are namespaced per feed before import to avoid cross-operator collisions. `line_cd` (100,000,000+), `station_cd` / `station_g_cd` (200,000,000+), and bus `type_cd` / `line_group_cd` (100,000,000+) are all deterministic fnv1a hashes that stay clear of the rail data ranges. Disable the entire bus pipeline with `DISABLE_BUS_FEATURE=true`. -- **Bus English-name fallback** – Keio Bus GTFS and Tokyu Bus JSON ship a kana reading for every stop but no English (`en`) name, so `gtfs_stops.stop_name_r` would otherwise be empty. `src/domain/romaji.rs::romaji_display_name()` derives a modified-Hepburn romanization (with macrons for long vowels, matching the curated rail style: Tōkyō / Kyōto / Shin-Ōsaka) from that reading, and `import.rs` fills `stop_name_r` with it whenever the feed provides no `en` value. Because `stop_name_r` is the single upstream source that fans out into the `stations` projection, `search_by_name`, and the romanized bus route/headsign names, this supplements every English-facing surface at once. The fallback never overwrites a real `en` value, and a reading with no convertible kana stays `NULL` rather than emitting a partial transcription. When projecting into `stations`, `station_name_rn` is filled with the plain-ASCII spelling via `romaji::strip_macrons()` (Tōkyō → Tokyo), mirroring the rail dataset's `_r` (macron) / `_rn` (macron-free) column pair. +- **Bus stop translations (readings & English)** – GTFS-JP `translations.txt` layouts differ per feed, so `load_gtfs_translations` resolves columns by header name (Seibu ships 6 columns without `record_sub_id`; Keio and the Tokyu community feeds ship 7) and indexes each `stop_name` translation under both keys it may use: `record_id` (== the stop_id, Seibu — with the "-NN" pole suffix also mapped to the parent stop_id) and `field_value` (== the Japanese stop_name, Keio / Tokyu community, where `record_id` is left empty). `import_gtfs_stops` then looks a stop's translation up by stop_id first, then by name. Keying only by `record_id` (the previous behavior) silently dropped every field_value-keyed feed, leaving `station_name_k` filled with the kanji stop_name and `station_name_r` empty. Readings arriving as half-width katakana (`ﾆｼﾊﾁｵｳｼﾞ`, Keio / Tokyu community) are folded to full-width via `romaji::to_fullwidth_katakana()` before storage. +- **Bus English-name fallback** – When a feed provides no English (`en`) translation for a stop — e.g. Tokyu Bus ordinary-route JSON, which carries only `dc:title` and `odpt:kana` — `src/domain/romaji.rs::romaji_display_name()` derives a modified-Hepburn romanization (with macrons for long vowels, matching the curated rail style: Tōkyō / Kyōto / Shin-Ōsaka) from the kana reading, and `import.rs` fills `stop_name_r` with it. The fallback never overwrites a real `en` value, and a reading with no convertible kana stays `NULL` rather than emitting a partial transcription. Because `stop_name_r` is the single upstream source that fans out into the `stations` projection, `search_by_name`, and the romanized bus route/headsign names, this supplements every English-facing surface at once. When projecting into `stations`, `station_name_rn` is filled with the plain-ASCII spelling via `romaji::strip_macrons()` (Tōkyō → Tokyo), mirroring the rail dataset's `_r` (macron) / `_rn` (macron-free) column pair. - **TTS metadata** – `Station`, `StationMinimal`, `Line`, and `TrainType` expose `name_ipa` / `name_roman_ipa` plus `name_tts_segments` for multi-segment pronunciation output. Use `name_tts_segments` when clients need per-token SSML construction for mixed-language names such as `Kasai-Rinkai Park`. - **Connected routes** – `GetConnectedRoutes`. `QueryInteractor::get_connected_stations` is not implemented yet and returns an empty vector; update the use-case and infrastructure layers together when adding real logic. - Changes to the service contract require coordinated updates to `proto/stationapi.proto`, regenerated code via `tonic-build`, and corresponding adjustments in both presentation and use-case layers. diff --git a/stationapi/src/domain/romaji.rs b/stationapi/src/domain/romaji.rs index 48878ee4..50f42d4b 100644 --- a/stationapi/src/domain/romaji.rs +++ b/stationapi/src/domain/romaji.rs @@ -257,18 +257,148 @@ fn is_separator(c: char) -> bool { matches!(c, '\u{30FB}' | ' ' | '\u{3000}' | '/' | '\u{FF0F}') } -/// Normalize hiragana to katakana so both reading styles convert identically. +/// Normalize a kana string to full-width katakana so every reading style +/// converts identically. Delegates to [`to_fullwidth_katakana`]. fn normalize_kana(input: &str) -> String { - input - .chars() - .map(|c| { - if ('\u{3041}'..='\u{3096}').contains(&c) { - char::from_u32(c as u32 + 0x60).unwrap_or(c) - } else { - c + to_fullwidth_katakana(input) +} + +/// Normalize hiragana and half-width katakana to full-width katakana. +/// +/// Some ODPT bus GTFS feeds ship their `ja-Hrkt` readings as half-width +/// katakana with combining voiced/semi-voiced marks (e.g. `ﾆｼﾊﾁｵｳｼﾞ`), which +/// must be folded to full-width before they can be stored in the katakana +/// column or romanized. Full-width katakana passes through unchanged, so the +/// function is idempotent. +pub fn to_fullwidth_katakana(input: &str) -> String { + let chars: Vec = input.chars().collect(); + let mut out = String::with_capacity(chars.len()); + let mut i = 0; + while i < chars.len() { + let c = chars[i]; + + // Hiragana → katakana (same +0x60 offset used across the module). + if ('\u{3041}'..='\u{3096}').contains(&c) { + out.push(char::from_u32(c as u32 + 0x60).unwrap_or(c)); + i += 1; + continue; + } + + // Half-width katakana → full-width, folding a trailing voiced (ﾞ) or + // semi-voiced (ﾟ) mark into the preceding kana. + if let Some(base) = halfwidth_katakana_to_fullwidth(c) { + match chars.get(i + 1).copied() { + Some('\u{FF9E}') => { + if let Some(voiced) = apply_dakuten(base) { + out.push(voiced); + i += 2; + continue; + } + } + Some('\u{FF9F}') => { + if let Some(semi) = apply_handakuten(base) { + out.push(semi); + i += 2; + continue; + } + } + _ => {} } - }) - .collect() + out.push(base); + i += 1; + continue; + } + + out.push(c); + i += 1; + } + out +} + +/// Map a half-width katakana code point to its full-width base form. +fn halfwidth_katakana_to_fullwidth(c: char) -> Option { + let full = match c { + '\u{FF65}' => '\u{30FB}', // ・ middle dot + 'ｦ' => 'ヲ', + 'ｧ' => 'ァ', + 'ｨ' => 'ィ', + 'ｩ' => 'ゥ', + 'ｪ' => 'ェ', + 'ｫ' => 'ォ', + 'ｬ' => 'ャ', + 'ｭ' => 'ュ', + 'ｮ' => 'ョ', + 'ｯ' => 'ッ', + 'ｰ' => 'ー', + 'ｱ' => 'ア', + 'ｲ' => 'イ', + 'ｳ' => 'ウ', + 'ｴ' => 'エ', + 'ｵ' => 'オ', + 'ｶ' => 'カ', + 'ｷ' => 'キ', + 'ｸ' => 'ク', + 'ｹ' => 'ケ', + 'ｺ' => 'コ', + 'ｻ' => 'サ', + 'ｼ' => 'シ', + 'ｽ' => 'ス', + 'ｾ' => 'セ', + 'ｿ' => 'ソ', + 'ﾀ' => 'タ', + 'ﾁ' => 'チ', + 'ﾂ' => 'ツ', + 'ﾃ' => 'テ', + 'ﾄ' => 'ト', + 'ﾅ' => 'ナ', + 'ﾆ' => 'ニ', + 'ﾇ' => 'ヌ', + 'ﾈ' => 'ネ', + 'ﾉ' => 'ノ', + 'ﾊ' => 'ハ', + 'ﾋ' => 'ヒ', + 'ﾌ' => 'フ', + 'ﾍ' => 'ヘ', + 'ﾎ' => 'ホ', + 'ﾏ' => 'マ', + 'ﾐ' => 'ミ', + 'ﾑ' => 'ム', + 'ﾒ' => 'メ', + 'ﾓ' => 'モ', + 'ﾔ' => 'ヤ', + 'ﾕ' => 'ユ', + 'ﾖ' => 'ヨ', + 'ﾗ' => 'ラ', + 'ﾘ' => 'リ', + 'ﾙ' => 'ル', + 'ﾚ' => 'レ', + 'ﾛ' => 'ロ', + 'ﾜ' => 'ワ', + 'ﾝ' => 'ン', + _ => return None, + }; + Some(full) +} + +/// Apply a dakuten (voiced mark) to a full-width katakana base, if voiceable. +fn apply_dakuten(c: char) -> Option { + let voiced = match c { + 'ウ' => 'ヴ', + 'カ' | 'キ' | 'ク' | 'ケ' | 'コ' | 'サ' | 'シ' | 'ス' | 'セ' | 'ソ' | 'タ' | 'チ' + | 'ツ' | 'テ' | 'ト' | 'ハ' | 'ヒ' | 'フ' | 'ヘ' | 'ホ' => { + char::from_u32(c as u32 + 1)? + } + _ => return None, + }; + Some(voiced) +} + +/// Apply a handakuten (semi-voiced mark) to a full-width katakana base. +fn apply_handakuten(c: char) -> Option { + match c { + 'ハ' | 'ヒ' | 'フ' | 'ヘ' | 'ホ' => char::from_u32(c as u32 + 2), + _ => None, + } } /// Two-character katakana combinations (palatalised sounds and loanword kana). @@ -522,6 +652,28 @@ mod tests { assert_eq!(romaji("とうきょう"), "tōkyō"); } + #[test] + fn halfwidth_katakana_is_folded_to_fullwidth() { + // Real Keio Bus ja-Hrkt readings (half-width with combining dakuten). + assert_eq!( + to_fullwidth_katakana("ﾆｼﾊﾁｵｳｼﾞｴｷﾐﾅﾐｸﾞﾁ"), + "ニシハチオウジエキミナミグチ" + ); + assert_eq!(to_fullwidth_katakana("ｱｷﾊﾞｼﾞﾝｼﾞｬ"), "アキバジンジャ"); + assert_eq!(to_fullwidth_katakana("ｱｻｶﾞﾔｼﾞｭｳﾀｸ"), "アサガヤジュウタク"); + // Handakuten and prolonged mark. + assert_eq!(to_fullwidth_katakana("ﾊﾟｰｸ"), "パーク"); + // Full-width katakana and hiragana are unchanged / folded (idempotent). + assert_eq!(to_fullwidth_katakana("トウキョウ"), "トウキョウ"); + assert_eq!(to_fullwidth_katakana("しぶや"), "シブヤ"); + } + + #[test] + fn halfwidth_reading_romanizes_end_to_end() { + // Keio Bus 阿佐ヶ谷住宅東 reading → Hepburn (used only when `en` is absent). + assert_eq!(romaji("ｱｻｶﾞﾔｼﾞｭｳﾀｸﾋｶﾞｼ"), "asagayajūtakuhigashi"); + } + #[test] fn separators_become_spaces() { assert_eq!(romaji("シブヤ・エキ"), "shibuya eki"); diff --git a/stationapi/src/import.rs b/stationapi/src/import.rs index 5770c740..f270bb0a 100644 --- a/stationapi/src/import.rs +++ b/stationapi/src/import.rs @@ -5,7 +5,7 @@ use serde::de::{DeserializeOwned, DeserializeSeed, SeqAccess, Visitor}; use serde::{Deserialize, Serialize}; use sqlx::{Connection, PgConnection}; use stationapi::config::fetch_database_url; -use stationapi::domain::romaji::{romaji_display_name, strip_macrons}; +use stationapi::domain::romaji::{romaji_display_name, strip_macrons, to_fullwidth_katakana}; use std::collections::{HashMap, HashSet}; use std::fmt; use std::fs::File; @@ -1151,72 +1151,84 @@ fn load_gtfs_translations( let mut rdr = ReaderBuilder::new().from_path(&translations_path)?; + // The GTFS-JP translations.txt column layout differs between feeds: Seibu + // omits `record_sub_id` (6 columns), while Keio and the Tokyu community feeds + // include it (7 columns). Resolve every column by header name so positions + // stay correct regardless of layout. + let headers = rdr.headers()?.clone(); + let column = |name: &str| headers.iter().position(|h| h == name); + let (Some(table_idx), Some(field_idx), Some(lang_idx), Some(text_idx)) = ( + column("table_name"), + column("field_name"), + column("language"), + column("translation"), + ) else { + warn!("translations.txt missing required columns; skipping translations."); + return Ok(translations); + }; + let record_id_idx = column("record_id"); + let field_value_idx = column("field_value"); + for result in rdr.records() { let record = result?; - // table_name,field_name,language,translation,record_id,record_sub_id,field_value - let table_name = record.get(0).unwrap_or(""); - let field_name = record.get(1).unwrap_or(""); - let language = record.get(2).unwrap_or(""); - let translation_text = record.get(3).unwrap_or(""); - let record_id = record.get(4).unwrap_or(""); + let cell = |idx: Option| idx.and_then(|i| record.get(i)).unwrap_or(""); // Only process stop_name translations for now - if table_name == "stops" && field_name == "stop_name" { - // Store translation for the exact record_id (e.g., "0001-01") - let key = ("stops".to_string(), record_id.to_string()); - let entry = translations.entry(key).or_default(); - - match language { - "ja" => entry.ja = Some(translation_text.to_string()), - "ja-Hrkt" => entry.ja_hrkt = Some(translation_text.to_string()), - "en" => entry.en = Some(translation_text.to_string()), - "zh-Hans" | "zh-Hant" | "zh" => entry.zh = Some(translation_text.to_string()), - "ko" => entry.ko = Some(translation_text.to_string()), - _ => {} - } + if record.get(table_idx) != Some("stops") || record.get(field_idx) != Some("stop_name") { + continue; + } - // Also store translation for parent stop_id (without suffix like "-01", "-02") - // This allows parent stops (location_type=1) to find translations + let language = record.get(lang_idx).unwrap_or(""); + let text = record.get(text_idx).unwrap_or(""); + let record_id = cell(record_id_idx); + let field_value = cell(field_value_idx); + + // Feeds key stop_name translations either by `record_id` (Seibu — the + // stop_id, sometimes with a "-01" pole suffix) or by `field_value` (Keio, + // Tokyu community — the Japanese stop_name itself, with record_id left + // empty). Index whichever the row provides so the stop importer can look + // the translation up by stop_id or by name. + if !record_id.is_empty() { + set_translation_language(&mut translations, record_id, language, text, true); + // Parent stop_id (drop the "-NN" pole suffix); first child wins. if let Some(parent_id) = record_id.rfind('-').map(|pos| &record_id[..pos]) { - let parent_key = ("stops".to_string(), parent_id.to_string()); - // Only insert if not already present (first child's translation wins) - translations - .entry(parent_key) - .or_insert_with(|| Translation { - ja: None, - ja_hrkt: None, - en: None, - zh: None, - ko: None, - }); - let parent_entry = translations - .get_mut(&("stops".to_string(), parent_id.to_string())) - .unwrap(); - match language { - "ja" if parent_entry.ja.is_none() => { - parent_entry.ja = Some(translation_text.to_string()) - } - "ja-Hrkt" if parent_entry.ja_hrkt.is_none() => { - parent_entry.ja_hrkt = Some(translation_text.to_string()) - } - "en" if parent_entry.en.is_none() => { - parent_entry.en = Some(translation_text.to_string()) - } - "zh-Hans" | "zh-Hant" | "zh" if parent_entry.zh.is_none() => { - parent_entry.zh = Some(translation_text.to_string()) - } - "ko" if parent_entry.ko.is_none() => { - parent_entry.ko = Some(translation_text.to_string()) - } - _ => {} - } + set_translation_language(&mut translations, parent_id, language, text, false); } } + if !field_value.is_empty() { + set_translation_language(&mut translations, field_value, language, text, true); + } } Ok(translations) } +/// Set one language field on the `("stops", key)` translation entry. When +/// `overwrite` is false the value is only filled if currently absent (used for +/// parent-stop aggregation where the first child wins). +fn set_translation_language( + translations: &mut HashMap<(String, String), Translation>, + key: &str, + language: &str, + text: &str, + overwrite: bool, +) { + let entry = translations + .entry(("stops".to_string(), key.to_string())) + .or_default(); + let slot = match language { + "ja" => &mut entry.ja, + "ja-Hrkt" => &mut entry.ja_hrkt, + "en" => &mut entry.en, + "zh-Hans" | "zh-Hant" | "zh" => &mut entry.zh, + "ko" => &mut entry.ko, + _ => return, + }; + if overwrite || slot.is_none() { + *slot = Some(text.to_string()); + } +} + /// Import agencies from agency.txt async fn import_gtfs_agencies( conn: &mut PgConnection, @@ -1393,15 +1405,22 @@ async fn import_gtfs_stops( .filter(|s| !s.is_empty()) .map(|s| s.to_string()); - // Get translations - let key = ("stops".to_string(), original_stop_id.to_string()); - let translation = translations.get(&key); - - let stop_name_k = translation.and_then(|t| t.ja_hrkt.clone()); + // Get translations. Feeds key stop_name translations by either record_id + // (== stop_id) or field_value (== the Japanese stop_name), so try both. + let translation = translations + .get(&("stops".to_string(), original_stop_id.to_string())) + .or_else(|| translations.get(&("stops".to_string(), stop_name.clone()))); + + // Some feeds (Keio, Tokyu community) provide ja-Hrkt readings as + // half-width katakana, so normalize to full-width for the katakana column. + let stop_name_k = translation + .and_then(|t| t.ja_hrkt.clone()) + .map(|k| to_fullwidth_katakana(&k)); // Prefer the official English translation, but fall back to a Hepburn // romanization of the kana reading when the feed ships no `en` row - // (e.g. Keio Bus). This keeps every English-facing surface — station - // name, name search, and romanized route/headsign names — populated. + // (e.g. Tokyu community, or any en-less stop). This keeps every + // English-facing surface — station name, name search, and romanized + // route/headsign names — populated. let stop_name_r = translation .and_then(|t| t.en.clone()) .filter(|s| !s.is_empty()) @@ -3570,6 +3589,67 @@ mod tests { assert_eq!(parse_gtfs_time("aa:bb:cc"), None); } + #[test] + fn test_load_gtfs_translations_field_value_keyed() { + // Keio and the Tokyu community feeds key stop_name translations by + // field_value (the Japanese stop_name) with an empty record_id, ship + // ja-Hrkt as half-width katakana, and use the 7-column layout that + // includes record_sub_id. + let dir = std::env::temp_dir().join("stationapi_test_translations_fv"); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::write( + dir.join("translations.txt"), + "table_name,field_name,language,translation,record_id,record_sub_id,field_value\n\ + stops,stop_name,ja,西八王子駅南口,,,西八王子駅南口\n\ + stops,stop_name,ja-Hrkt,ﾆｼﾊﾁｵｳｼﾞｴｷﾐﾅﾐｸﾞﾁ,,,西八王子駅南口\n\ + stops,stop_name,en,Nishi-hachioji Sta. South,,,西八王子駅南口\n", + ) + .unwrap(); + + let translations = load_gtfs_translations(&dir).unwrap(); + let t = translations + .get(&("stops".to_string(), "西八王子駅南口".to_string())) + .expect("field_value-keyed translation should be found by stop_name"); + assert_eq!(t.ja_hrkt.as_deref(), Some("ﾆｼﾊﾁｵｳｼﾞｴｷﾐﾅﾐｸﾞﾁ")); + assert_eq!(t.en.as_deref(), Some("Nishi-hachioji Sta. South")); + // The half-width reading normalizes to full-width katakana for storage. + assert_eq!( + to_fullwidth_katakana(t.ja_hrkt.as_deref().unwrap()), + "ニシハチオウジエキミナミグチ" + ); + + let _ = std::fs::remove_dir_all(&dir); + } + + #[test] + fn test_load_gtfs_translations_record_id_keyed() { + // Seibu keys translations by record_id (the stop_id, with a "-01" pole + // suffix) in a 6-column layout without record_sub_id, and ships ja-Hrkt + // as full-width katakana. The parent stop_id resolves too. + let dir = std::env::temp_dir().join("stationapi_test_translations_rid"); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::write( + dir.join("translations.txt"), + "table_name,field_name,language,translation,record_id,field_value\n\ + stops,stop_name,ja,西武百貨店前,20001-01,\n\ + stops,stop_name,ja-Hrkt,セイブヒャッカテンマエ,20001-01,\n\ + stops,stop_name,en,Seibu hyakkaten-mae,20001-01,\n", + ) + .unwrap(); + + let translations = load_gtfs_translations(&dir).unwrap(); + let t = translations + .get(&("stops".to_string(), "20001-01".to_string())) + .expect("record_id-keyed translation should be found by stop_id"); + assert_eq!(t.en.as_deref(), Some("Seibu hyakkaten-mae")); + // Parent stop_id (pole suffix dropped) also resolves to the reading. + assert!(translations.contains_key(&("stops".to_string(), "20001".to_string()))); + + let _ = std::fs::remove_dir_all(&dir); + } + #[test] fn test_parse_odpt_time_adds_gtfs_seconds() { assert_eq!(parse_odpt_time("08:30"), Some("08:30:00".to_string())); From 267aed55080ed6b6209f7a47f0df517188e90259 Mon Sep 17 00:00:00 2001 From: Tsubasa SEKIGUCHI Date: Sun, 12 Jul 2026 23:13:15 +0900 Subject: [PATCH 3/3] =?UTF-8?q?GTFS=E3=81=AE=E4=B8=8A=E4=B8=8B=E3=83=9D?= =?UTF-8?q?=E3=83=BC=E3=83=AB=E3=82=92=E3=82=B0=E3=83=AB=E3=83=BC=E3=83=97?= =?UTF-8?q?=E5=8C=96=E3=81=97=E3=83=90=E3=82=B9=E5=81=9C=E3=81=AE=E9=87=8D?= =?UTF-8?q?=E8=A4=87=E8=A1=A8=E7=A4=BA=E3=82=92=E4=BF=AE=E6=AD=A3=20(#1605?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * GTFSの上下ポールを同名+近接でグループ化しバス停の重複表示を修正 parent_stationの階層を持たないフィード(西武・東急コミュニティバス・京王等) では上下ポールが別々のトップレベルstopとして取り込まれ、同名なのに別 station_g_cdになるため同じバス停名が2件表示されていた。同名かつ250m以内の stopをunion-findでまとめ、クラスタ代表(最小stop_id)のstation_g_cdを共有させる。 単独stopは従来値と一致するため都営など正しく動くフィードには影響しない。 Co-Authored-By: Claude Opus 4.8 * fix: apply CodeRabbit auto-fixes バス停グループ化を決定的 greedy complete-linkage に変更しクラスタ直径を 250m以内に制限(推移閉包による遠距離統合を防止)。parent_stationとして参照 される親stopはグループ化対象外にして自身のstation_g_cdへ固定し、parent 階層を持つフィード(都営等)への影響をなくす。回帰テストを追加。 Co-Authored-By: Claude Opus 4.8 --------- Co-authored-by: Claude Opus 4.8 --- stationapi/src/import.rs | 194 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 193 insertions(+), 1 deletion(-) diff --git a/stationapi/src/import.rs b/stationapi/src/import.rs index f270bb0a..7a313527 100644 --- a/stationapi/src/import.rs +++ b/stationapi/src/import.rs @@ -5,6 +5,7 @@ use serde::de::{DeserializeOwned, DeserializeSeed, SeqAccess, Visitor}; use serde::{Deserialize, Serialize}; use sqlx::{Connection, PgConnection}; use stationapi::config::fetch_database_url; +use stationapi::domain::arrival_estimation::haversine_distance; use stationapi::domain::romaji::{romaji_display_name, strip_macrons, to_fullwidth_katakana}; use std::collections::{HashMap, HashSet}; use std::fmt; @@ -2246,6 +2247,88 @@ fn generate_bus_station_g_cd(stop_id: &str) -> i32 { 200_000_000 + (hash % 100_000_000) as i32 } +/// Distance within which two like-named bus stops are treated as direction +/// poles of a single physical stop and merged into one `station_g_cd`. +const BUS_STOP_GROUPING_RADIUS_METERS: f64 = 250.0; + +/// Build a `stop_id -> station_g_cd` map that collapses the separate direction +/// poles of one physical bus stop into a single group. +/// +/// Some GTFS feeds encode poles with a `parent_station` hierarchy (Toei), in +/// which case each imported stop is already one deduplicated physical stop. Such +/// stops — the ones listed in `parent_stop_ids` because a child references them — +/// are pinned to their own `station_g_cd` and never merged, so parent-hierarchy +/// feeds are left exactly as before. Other feeds (Seibu, the Tokyu community +/// buses, and typically Keio) expose every pole as its own top-level stop, so the +/// same bus stop name would otherwise appear multiple times in the app; those +/// hierarchy-less stops are grouped when they share a name *and* sit close +/// together, mirroring how rail stations collapse across companies. +/// +/// Grouping uses deterministic greedy complete-linkage: within a name, stops are +/// visited in `stop_id` order and each joins the first cluster whose *every* +/// member is within `BUS_STOP_GROUPING_RADIUS_METERS`. This bounds each cluster's +/// diameter to the radius (no transitive chaining that could swallow a distant +/// same-named stop) and is independent of DB row ordering. A lone stop yields the +/// exact same `station_g_cd` as [`generate_bus_station_g_cd`]. +fn build_bus_station_g_cd_map( + stops: &[GtfsStopRow], + parent_stop_ids: &HashSet, +) -> HashMap { + let mut map: HashMap = HashMap::with_capacity(stops.len()); + + // Pin parent stops to their own group; bucket the rest by name for grouping. + let mut by_name: HashMap<&str, Vec> = HashMap::new(); + for (idx, stop) in stops.iter().enumerate() { + if parent_stop_ids.contains(&stop.stop_id) { + map.insert( + stop.stop_id.clone(), + generate_bus_station_g_cd(&stop.stop_id), + ); + continue; + } + by_name.entry(stop.stop_name.trim()).or_default().push(idx); + } + + for indices in by_name.values() { + // Visit in a canonical order so the resulting clusters are deterministic. + let mut order = indices.clone(); + order.sort_by(|&a, &b| stops[a].stop_id.cmp(&stops[b].stop_id)); + + // Greedy complete-linkage: a stop joins the first cluster all of whose + // members are within the radius; otherwise it starts a new cluster. + let mut clusters: Vec> = Vec::new(); + for &i in &order { + let si = &stops[i]; + let joined = clusters.iter_mut().find(|cluster| { + cluster.iter().all(|&j| { + let sj = &stops[j]; + haversine_distance(si.stop_lat, si.stop_lon, sj.stop_lat, sj.stop_lon) + <= BUS_STOP_GROUPING_RADIUS_METERS + }) + }); + match joined { + Some(cluster) => cluster.push(i), + None => clusters.push(vec![i]), + } + } + + for cluster in &clusters { + // Representative = lexicographically smallest stop_id for stability. + let rep = cluster + .iter() + .map(|&j| stops[j].stop_id.as_str()) + .min() + .expect("cluster is non-empty"); + let g_cd = generate_bus_station_g_cd(rep); + for &j in cluster { + map.insert(stops[j].stop_id.clone(), g_cd); + } + } + } + + map +} + /// Generate deterministic type_cd from (route_id, shape_id). /// Uses range starting at 100,000,000 to avoid conflicts with existing rail types. fn generate_bus_type_cd(route_id: &str, shape_id: &str) -> i32 { @@ -2919,10 +3002,38 @@ async fn integrate_gtfs_stops_to_stations( .fetch_all(&mut *conn) .await?; + // Stops referenced as a parent_station already represent one deduplicated + // physical stop; they must stay pinned to their own group during grouping. + let parent_stop_ids: HashSet = sqlx::query_scalar::<_, String>( + "SELECT DISTINCT parent_station FROM gtfs_stops \ + WHERE parent_station IS NOT NULL AND parent_station <> ''", + ) + .fetch_all(&mut *conn) + .await? + .into_iter() + .collect(); + + // Collapse the separate direction poles of one physical stop (feeds without a + // parent_station hierarchy expose each pole as its own top-level stop). + let station_g_cd_by_stop = build_bus_station_g_cd_map(&stops, &parent_stop_ids); + let group_count = station_g_cd_by_stop + .values() + .copied() + .collect::>() + .len(); + info!( + "Grouped {} bus stops into {} station groups.", + stops.len(), + group_count + ); + let mut inserted_count = 0; for stop in &stops { - let station_g_cd = generate_bus_station_g_cd(&stop.stop_id); + let station_g_cd = station_g_cd_by_stop + .get(&stop.stop_id) + .copied() + .unwrap_or_else(|| generate_bus_station_g_cd(&stop.stop_id)); // Get routes for this parent stop (with stop_sequence) // The mapping now uses parent_stop_id as key @@ -3771,6 +3882,87 @@ mod tests { ); } + fn stop_row(stop_id: &str, name: &str, lat: f64, lon: f64) -> GtfsStopRow { + GtfsStopRow { + stop_id: stop_id.to_string(), + stop_code: None, + stop_name: name.to_string(), + stop_name_k: None, + stop_name_r: None, + stop_name_zh: None, + stop_name_ko: None, + stop_desc: None, + stop_lat: lat, + stop_lon: lon, + zone_id: None, + stop_url: None, + location_type: None, + parent_station: None, + stop_timezone: None, + wheelchair_boarding: None, + platform_code: None, + } + } + + #[test] + fn test_build_bus_station_g_cd_map_merges_nearby_poles() { + let stops = vec![ + // Two direction poles of the same physical stop (~17 m apart). + stop_row("100001-01", "南大通り【朝霞警察署】", 35.787006, 139.592117), + stop_row("100001-02", "南大通り【朝霞警察署】", 35.786852, 139.592150), + // A distinct stop that merely shares a common name, far away (~40 km). + stop_row("A", "新田", 35.79, 139.59), + stop_row("B", "新田", 35.65, 139.80), + ]; + + let map = build_bus_station_g_cd_map(&stops, &HashSet::new()); + + // The two nearby poles collapse into one group. + assert_eq!(map["100001-01"], map["100001-02"]); + // The far-apart namesakes stay separate. + assert_ne!(map["A"], map["B"]); + // The merged group is stable and equals the representative (min stop_id). + assert_eq!(map["100001-01"], generate_bus_station_g_cd("100001-01")); + // A lone stop keeps the exact station_g_cd it had before grouping. + assert_eq!(map["A"], generate_bus_station_g_cd("A")); + } + + #[test] + fn test_build_bus_station_g_cd_map_no_transitive_chain() { + // A-B and B-C are each within the radius, but A-C is not (~400 m apart). + // Single-linkage would chain all three; complete-linkage must keep the + // distant endpoints A and C in separate groups. + let stops = vec![ + stop_row("chain_a", "連鎖テスト", 35.0, 139.0), + stop_row("chain_b", "連鎖テスト", 35.0, 139.0022), // ~200 m east of A + stop_row("chain_c", "連鎖テスト", 35.0, 139.0044), // ~400 m east of A + ]; + + let map = build_bus_station_g_cd_map(&stops, &HashSet::new()); + + // The chain endpoints must not be merged despite the bridging middle stop. + assert_ne!(map["chain_a"], map["chain_c"]); + // Sanity: A-C really exceeds the radius while A-B stays within it. + assert!(haversine_distance(35.0, 139.0, 35.0, 139.0044) > BUS_STOP_GROUPING_RADIUS_METERS); + assert!(haversine_distance(35.0, 139.0, 35.0, 139.0022) <= BUS_STOP_GROUPING_RADIUS_METERS); + } + + #[test] + fn test_build_bus_station_g_cd_map_pins_parent_stops() { + // A parent stop and a hierarchy-less stop share a name and sit close by. + let stops = vec![ + stop_row("0001", "青戸車庫前", 35.744787, 139.843847), + stop_row("other-01", "青戸車庫前", 35.744900, 139.843900), + ]; + let parents: HashSet = ["0001".to_string()].into_iter().collect(); + + let map = build_bus_station_g_cd_map(&stops, &parents); + + // The parent stop keeps its own station_g_cd and is never merged. + assert_eq!(map["0001"], generate_bus_station_g_cd("0001")); + assert_ne!(map["0001"], map["other-01"]); + } + #[test] fn test_generate_bus_type_cd() { let type_cd = generate_bus_type_cd("route_001", "shape_A");