From 2ea802243f93f055727d705c306f60dd9be3c1c1 Mon Sep 17 00:00:00 2001 From: Tsubasa SEKIGUCHI Date: Sun, 12 Jul 2026 16:52:32 +0900 Subject: [PATCH 1/3] =?UTF-8?q?=E4=BA=AC=E7=8E=8B=E3=83=90=E3=82=B9?= =?UTF-8?q?=E3=81=AEGTFS=E3=83=95=E3=82=A3=E3=83=BC=E3=83=89=E3=82=92?= =?UTF-8?q?=E8=BF=BD=E5=8A=A0=20(#1601)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 2 ++ AGENTS.md | 6 +++--- README.md | 1 + data/1!companies.csv | 1 + docs/architecture.md | 4 ++-- stationapi/src/import.rs | 21 ++++++++++++++++++++- 6 files changed, 29 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index d2839402..1c1f88d3 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,8 @@ api_descriptor.pb tmp.sql .vscode/ data/ToeiBus-GTFS/ +data/SeibuBus-GTFS/ +data/KeioBus-GTFS/ scripts/.osm_cache/ scripts/.gtfs_cache/ __pycache__/ diff --git a/AGENTS.md b/AGENTS.md index f04add7e..d140c289 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -20,7 +20,7 @@ This guide explains how automation agents and human contributors should work wit - Environment variables: - `DATABASE_URL` – SQLx connection string (e.g., `postgres://stationapi:stationapi@localhost/stationapi`). - `DISABLE_GRPC_WEB` – `false` enables gRPC-Web; set to `true` for pure gRPC/HTTP2. - - `ODPT_ACCESS_TOKEN` – ODPT consumer key used to download authenticated GTFS feeds such as Seibu Bus. + - `ODPT_ACCESS_TOKEN` – ODPT consumer key used to download authenticated GTFS feeds such as Seibu Bus and Keio Bus. - `HOST` and `PORT` – listen address (defaults to `[::1]:50051`; Docker uses `0.0.0.0:50051`). - `.env.test` exports `TEST_DATABASE_URL`, `RUST_LOG`, `RUST_BACKTRACE`, and `RUST_TEST_THREADS=1` for integration tests. - Recommended: keep shared defaults in `.env`, copy overrides to `.env.local`, and rely on startup loading both files. @@ -53,8 +53,8 @@ This guide explains how automation agents and human contributors should work wit - **Stations** – `GetStationById`, `GetStationByIdList`, `GetStationsByGroupId`, `GetStationsByCoordinates`, `GetStationsByLineId`, `GetStationsByName`, `GetStationsByLineGroupId`. `QueryInteractor` enriches stations with lines, companies, station numbers, and train types. - **Lines** – `GetLineById`, `GetLinesByIdList`, `GetLinesByName`. Results include company data and computed line symbols based on repository helpers. - **Routes** – `GetRoutes`, `GetRoutesMinimal`. The minimal variant returns `RouteMinimalResponse` with deduplicated `LineMinimal` data; paging tokens are currently empty (pagination not implemented). -- **Train types** – `GetTrainTypesByStationId`, `GetRouteTypes`. Train types aggregate by line group and include related lines plus optional train type metadata. Rail variants use `TrainTypeKind::{Default, Branch, Rapid, Express, LimitedExpress, HighSpeedRapid, CommuterRapid}` (0-6); bus variants use `BusRoute` (7), which represents a `(route_id, shape_id)` operation pattern (e.g. 循環 / 短ターン / 支線) generated automatically from the Toei Bus GTFS feed. -- **GTFS bus integration** – At startup, `src/import.rs::integrate_gtfs_to_stations()` ingests GTFS feeds into `gtfs_*` tables and then projects them onto the shared `stations` / `lines` / `types` / `station_station_types` tables. Every configured feed is imported, including Seibu Bus (downloaded from ODPT with `ODPT_ACCESS_TOKEN`). `transport_type` (0: rail, 1: bus) on both `stations` and `lines` keeps the two worlds queryable side by side. GTFS IDs are namespaced per feed before import to avoid cross-operator collisions. `line_cd` (100,000,000+), `station_cd` / `station_g_cd` (200,000,000+), and bus `type_cd` / `line_group_cd` (100,000,000+) are all deterministic fnv1a hashes that stay clear of the rail data ranges. Disable the entire bus pipeline with `DISABLE_BUS_FEATURE=true`. +- **Train types** – `GetTrainTypesByStationId`, `GetRouteTypes`. Train types aggregate by line group and include related lines plus optional train type metadata. Rail variants use `TrainTypeKind::{Default, Branch, Rapid, Express, LimitedExpress, HighSpeedRapid, CommuterRapid}` (0-6); bus variants use `BusRoute` (7), which represents a `(route_id, shape_id)` operation pattern (e.g. 循環 / 短ターン / 支線) generated automatically from the configured GTFS bus feeds (Toei Bus, Seibu Bus, Keio Bus). +- **GTFS bus integration** – At startup, `src/import.rs::integrate_gtfs_to_stations()` ingests GTFS feeds into `gtfs_*` tables and then projects them onto the shared `stations` / `lines` / `types` / `station_station_types` tables. Every configured feed is imported, including Seibu Bus and Keio Bus (both downloaded from ODPT with `ODPT_ACCESS_TOKEN`). `transport_type` (0: rail, 1: bus) on both `stations` and `lines` keeps the two worlds queryable side by side. GTFS IDs are namespaced per feed before import to avoid cross-operator collisions. `line_cd` (100,000,000+), `station_cd` / `station_g_cd` (200,000,000+), and bus `type_cd` / `line_group_cd` (100,000,000+) are all deterministic fnv1a hashes that stay clear of the rail data ranges. Disable the entire bus pipeline with `DISABLE_BUS_FEATURE=true`. - **TTS metadata** – `Station`, `StationMinimal`, `Line`, and `TrainType` expose `name_ipa` / `name_roman_ipa` plus `name_tts_segments` for multi-segment pronunciation output. Use `name_tts_segments` when clients need per-token SSML construction for mixed-language names such as `Kasai-Rinkai Park`. - **Connected routes** – `GetConnectedRoutes`. `QueryInteractor::get_connected_stations` is not implemented yet and returns an empty vector; update the use-case and infrastructure layers together when adding real logic. - Changes to the service contract require coordinated updates to `proto/stationapi.proto`, regenerated code via `tonic-build`, and corresponding adjustments in both presentation and use-case layers. diff --git a/README.md b/README.md index 31f9bc5c..a1edaae6 100644 --- a/README.md +++ b/README.md @@ -111,6 +111,7 @@ We follow Rust best practices for testing: - Bus-related data provided by [Tokyo Metropolitan Bureau of Transportation (Toei)](https://www.kotsu.metro.tokyo.jp/), licensed under [CC BY 4.0](https://creativecommons.org/licenses/by/4.0/) - Bus-related data provided by [Seibu Bus Co., Ltd. (西武バス)](https://www.seibubus.co.jp/) via the [Public Transportation Open Data Center](https://ckan.odpt.org/), licensed under the [Public Transportation Open Data Basic License](https://developer.odpt.org/terms) +- Bus-related data provided by [Keio Dentetsu Bus Co., Ltd. (京王電鉄バス)](https://www.keio-bus.com/) via the [Public Transportation Open Data Center](https://ckan.odpt.org/), licensed under the [Public Transportation Open Data Basic License](https://developer.odpt.org/terms) - Station data provided by [駅データ.jp](https://www.ekidata.jp/) - Speed calibration data (`speed_table.rs`) derived from GTFS timetables provided by Kyoto City Transportation Bureau (京都市交通局), Yokohama City Transportation Bureau (横浜市交通局), Tokyo Metro (東京メトロ), Metropolitan Intercity Railway (首都圏新都市鉄道), Tokyo Tama Intercity Monorail (多摩都市モノレール), and Tokyo Waterfront Area Rapid Transit (東京臨海高速鉄道) via the [Public Transportation Open Data Center](https://ckan.odpt.org/), licensed under the [Public Transportation Open Data Basic License](https://developer.odpt.org/terms); by [Tokyo Metropolitan Bureau of Transportation (Toei)](https://www.kotsu.metro.tokyo.jp/), licensed under [CC BY 4.0](https://creativecommons.org/licenses/by/4.0/); and by Hakodate City Enterprise Bureau Transportation Department (函館市企業局交通部), licensed under [GTFS-RUL (ODPT)](https://gtfs-jp.org/GTFS-RUL(ODPT).pdf) - Average inter-station distances (`average_distance`) computed from railway track geometry © [OpenStreetMap contributors](https://www.openstreetmap.org/copyright), licensed under [ODbL](https://opendatacommons.org/licenses/odbl/) diff --git a/data/1!companies.csv b/data/1!companies.csv index 7879e36a..f9c931f8 100644 --- a/data/1!companies.csv +++ b/data/1!companies.csv @@ -8,6 +8,7 @@ company_cd,rr_cd,company_name,company_name_k,company_name_h,company_name_r,compa 11,21,東武鉄道,トウブテツドウ,東武鉄道株式会社,東武,Tobu,Tobu Railway,http://www.tobu.co.jp/,2,0,11 12,22,西武鉄道,セイブテツドウ,西武鉄道株式会社,西武,Seibu,Seibu Railway,http://www.seibu-group.co.jp/railways/,2,0,12 253,99,西武バス,セイブバス,西武バス株式会社,西武バス,Seibu Bus,Seibu Bus,https://www.seibubus.co.jp/,0,0,12 +254,99,京王バス,ケイオウバス,京王電鉄バス株式会社,京王バス,Keio Bus,Keio Bus,https://www.keio-bus.com/,0,0,14 13,23,京成電鉄,ケイセイデンテツ,京成電鉄株式会社,京成,Keisei,Keisei Electric Railway,http://www.keisei.co.jp/,2,0,13 14,24,京王電鉄,ケイオウデンテツ,京王電鉄株式会社,京王,Keio,Keio,http://www.keio.co.jp/,2,0,14 15,25,小田急電鉄,オダキュウデンテツ,小田急電鉄株式会社,小田急,Odakyu,Odakyu Electric Railway,http://www.odakyu.jp/,2,0,15 diff --git a/docs/architecture.md b/docs/architecture.md index 8ffdf295..d3200cc3 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -166,11 +166,11 @@ CREATE INDEX idx_performance_station_name_trgm ON stations ### バス (GTFS) データの統合 -`stations` / `lines` / `types` / `station_station_types` は鉄道とバスの両方を保持し、`stations.transport_type` / `lines.transport_type` (0: 鉄道, 1: バス) でフィルタリングします。バスデータの取り込みは `src/import.rs` の `integrate_gtfs_to_stations()` が起動時に実行し、ODPT 公開の GTFS を `gtfs_*` テーブルに展開してから既存テーブルへ統合します。Seibu Bus を含む設定済みの全GTFSフィードを使用します。Seibu Bus のダウンロードには `.env.local` の `ODPT_ACCESS_TOKEN` を使用します。 +`stations` / `lines` / `types` / `station_station_types` は鉄道とバスの両方を保持し、`stations.transport_type` / `lines.transport_type` (0: 鉄道, 1: バス) でフィルタリングします。バスデータの取り込みは `src/import.rs` の `integrate_gtfs_to_stations()` が起動時に実行し、ODPT 公開の GTFS を `gtfs_*` テーブルに展開してから既存テーブルへ統合します。Seibu Bus・Keio Bus を含む設定済みの全GTFSフィードを使用します。Seibu Bus・Keio Bus のダウンロードには `.env.local` の `ODPT_ACCESS_TOKEN` を使用します。 | 鉄道側の概念 | バス側の対応 | |---|---| -| `companies` | GTFS フィードごとに会社を決定。Toei Bus は東京都交通局 (`company_cd=119`)、Seibu Bus は西武バス (`company_cd=253`) | +| `companies` | GTFS フィードごとに会社を決定。Toei Bus は東京都交通局 (`company_cd=119`)、Seibu Bus は西武バス (`company_cd=253`)、Keio Bus は京王バス (`company_cd=254`) | | `lines` | GTFS `routes` を 1:1 で `lines` に登録。`line_cd` はフィード接頭辞付き `route_id` の fnv1a ハッシュで 100,000,000+ 空間に決定的に生成 | | `stations` | GTFS `stops` (親停留所) を `(stop_id, route_id)` 単位で `stations` に登録。`station_cd` / `station_g_cd` もフィード接頭辞付き ID から 200,000,000+ 空間にハッシュ生成 | | `types` | GTFS の `(route_id, shape_id)` バリエーション (フルループ / 短ターン / 支線など) を `kind = TrainTypeKind::BusRoute (= 7)` の TrainType として登録。**停留所集合が完全に同じ shape ペア (上下方向違いのみ) は 1 つの TrainType に畳み、`direction = Both` を設定**。`type_name` は循環なら ` (循環)`、双方向ペアなら ` ⇔ `、片方向なら `<始発停留所> → ` | diff --git a/stationapi/src/import.rs b/stationapi/src/import.rs index 5cb5f0dd..92a9d5db 100644 --- a/stationapi/src/import.rs +++ b/stationapi/src/import.rs @@ -286,6 +286,13 @@ const GTFS_FEEDS: &[GtfsFeed] = &[ url: "https://api.odpt.org/api/v4/files/SeibuBus/data/SeibuBus-GTFS.zip", requires_consumer_key: true, }, + GtfsFeed { + id: "keio", + name: "Keio Bus", + path: "data/KeioBus-GTFS", + url: "https://api.odpt.org/api/v4/files/odpt/KeioBus/AllLines.zip", + requires_consumer_key: true, + }, ]; const DEFAULT_GTFS_BUS_LINE_COLOR: &str = "#1f63c6"; @@ -1675,6 +1682,8 @@ fn company_cd_for_gtfs_route(route_id: &str) -> Option { Some(119) // Toei Transportation } else if route_id.starts_with("seibu:") { Some(253) // Seibu Bus + } else if route_id.starts_with("keio:") { + Some(254) // Keio Bus } else { None // Unknown/unsupported prefix } @@ -3040,6 +3049,7 @@ mod tests { fn test_company_cd_for_gtfs_route() { assert_eq!(company_cd_for_gtfs_route("toei:route_001"), Some(119)); assert_eq!(company_cd_for_gtfs_route("seibu:route_001"), Some(253)); + assert_eq!(company_cd_for_gtfs_route("keio:route_001"), Some(254)); assert_eq!(company_cd_for_gtfs_route("unknown:route_001"), None); } @@ -3128,8 +3138,17 @@ mod tests { fn test_gtfs_feeds() { assert_eq!( GTFS_FEEDS.iter().map(|feed| feed.id).collect::>(), - vec!["toei", "seibu"] + vec!["toei", "seibu", "keio"] + ); + + let keio = GTFS_FEEDS.iter().find(|feed| feed.id == "keio").unwrap(); + assert_eq!(keio.name, "Keio Bus"); + assert_eq!(keio.path, "data/KeioBus-GTFS"); + assert_eq!( + keio.url, + "https://api.odpt.org/api/v4/files/odpt/KeioBus/AllLines.zip" ); + assert!(keio.requires_consumer_key); } #[test] From a13b50aa657d846a38364e4c79337760b3ea8451 Mon Sep 17 00:00:00 2001 From: Tsubasa SEKIGUCHI Date: Sun, 12 Jul 2026 17:44:17 +0900 Subject: [PATCH 2/3] =?UTF-8?q?=E4=BA=AC=E7=8E=8B=E3=83=90=E3=82=B9GTFS?= =?UTF-8?q?=E3=81=AE=E3=83=80=E3=82=A6=E3=83=B3=E3=83=AD=E3=83=BC=E3=83=89?= =?UTF-8?q?404=E3=82=92=E4=BF=AE=E6=AD=A3=20(#1602)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- stationapi/src/import.rs | 99 +++++++++++++++++++++++++++++++++++----- 1 file changed, 87 insertions(+), 12 deletions(-) diff --git a/stationapi/src/import.rs b/stationapi/src/import.rs index 92a9d5db..10283f9b 100644 --- a/stationapi/src/import.rs +++ b/stationapi/src/import.rs @@ -287,10 +287,12 @@ const GTFS_FEEDS: &[GtfsFeed] = &[ requires_consumer_key: true, }, GtfsFeed { + // ODPT's versioned files endpoint requires a `date` selector; `current` + // always resolves to the timetable in effect today (omitting it 404s). id: "keio", name: "Keio Bus", path: "data/KeioBus-GTFS", - url: "https://api.odpt.org/api/v4/files/odpt/KeioBus/AllLines.zip", + url: "https://api.odpt.org/api/v4/files/odpt/KeioBus/AllLines.zip?date=current", requires_consumer_key: true, }, ]; @@ -306,6 +308,14 @@ fn scoped_gtfs_id_opt(feed: &GtfsFeed, id: Option<&str>) -> Option { .map(|s| scoped_gtfs_id(feed, s)) } +/// Append the ODPT `acl:consumerKey` query parameter to a feed URL, using the +/// correct separator (`?` for the first parameter, `&` when the URL already +/// carries a query string such as `?date=current`). +fn append_consumer_key(url: &str, token: &str) -> String { + let separator = if url.contains('?') { '&' } else { '?' }; + format!("{}{}acl:consumerKey={}", url, separator, token) +} + fn gtfs_download_url(feed: &GtfsFeed) -> Result> { if !feed.requires_consumer_key { return Ok(feed.url.to_string()); @@ -318,10 +328,14 @@ fn gtfs_download_url(feed: &GtfsFeed) -> Result Result<(), Box> { let gtfs_path = Path::new(feed.path); @@ -334,11 +348,33 @@ fn download_gtfs(feed: GtfsFeed) -> Result<(), Box Ok(()), + Err(e) => { + if gtfs_path.exists() { + if let Err(cleanup_err) = fs::remove_dir_all(gtfs_path) { + warn!( + "Failed to remove partial {} GTFS directory after error: {}", + feed.name, cleanup_err + ); + } + } + Err(e) + } + } +} + +fn download_and_extract_gtfs( + feed: &GtfsFeed, + gtfs_path: &Path, +) -> Result<(), Box> { info!("Downloading {} GTFS data from ODPT API...", feed.name); - // Download the ZIP file + // Download the ZIP file. `without_url()` strips the request URL (which carries + // the `acl:consumerKey` token) from any transport error so the token cannot + // leak into logs or propagated error messages. let request_start = std::time::Instant::now(); - let response = reqwest::blocking::get(gtfs_download_url(&feed)?)?; + let response = reqwest::blocking::get(gtfs_download_url(feed)?).map_err(|e| e.without_url())?; info!( "[gtfs-download:{}] response received in {:?} (status={})", feed.id, @@ -356,7 +392,7 @@ fn download_gtfs(feed: GtfsFeed) -> Result<(), Box Result<(), Box> { .join(", ") ); - // Download GTFS data if not present (use spawn_blocking to avoid blocking async runtime) + // Download GTFS data if not present (use spawn_blocking to avoid blocking async runtime). + // A single feed's download failure (e.g. an operator's ODPT outage or a moved URL) + // must not abort the whole pipeline: log it and continue so the other feeds still + // import. The per-feed import loop below skips any feed whose directory is missing. let download_start = std::time::Instant::now(); for feed in enabled_feeds.iter().copied() { - tokio::task::spawn_blocking(move || download_gtfs(feed)) - .await - .map_err(|e| format!("Failed to spawn blocking task: {}", e))? - .map_err(|e| -> Box { e })?; + match tokio::task::spawn_blocking(move || download_gtfs(feed)).await { + Ok(Ok(())) => {} + Ok(Err(e)) => { + warn!( + "Failed to download {} GTFS: {}. Skipping this feed; other feeds continue.", + feed.name, e + ); + } + // A panic or cancellation of the blocking task must not abort the whole + // pipeline either: log it and let the remaining feeds proceed. + Err(join_err) => { + warn!( + "{} GTFS download task did not complete ({}). Skipping this feed; other feeds continue.", + feed.name, join_err + ); + } + } } info!( "[gtfs] download/extract finished in {:?}", @@ -3144,13 +3196,36 @@ mod tests { let keio = GTFS_FEEDS.iter().find(|feed| feed.id == "keio").unwrap(); assert_eq!(keio.name, "Keio Bus"); assert_eq!(keio.path, "data/KeioBus-GTFS"); + // ODPT's versioned files endpoint 404s without a `date` selector; `current` + // keeps the URL pinned to the timetable in effect today. assert_eq!( keio.url, - "https://api.odpt.org/api/v4/files/odpt/KeioBus/AllLines.zip" + "https://api.odpt.org/api/v4/files/odpt/KeioBus/AllLines.zip?date=current" ); assert!(keio.requires_consumer_key); } + #[test] + fn test_append_consumer_key() { + // URL without an existing query string uses `?`. + assert_eq!( + append_consumer_key( + "https://api.odpt.org/api/v4/files/SeibuBus/data/SeibuBus-GTFS.zip", + "TOKEN" + ), + "https://api.odpt.org/api/v4/files/SeibuBus/data/SeibuBus-GTFS.zip?acl:consumerKey=TOKEN" + ); + // URL that already carries a query string (e.g. `?date=current`) uses `&` + // so the consumer key does not clobber the existing parameter. + assert_eq!( + append_consumer_key( + "https://api.odpt.org/api/v4/files/odpt/KeioBus/AllLines.zip?date=current", + "TOKEN" + ), + "https://api.odpt.org/api/v4/files/odpt/KeioBus/AllLines.zip?date=current&acl:consumerKey=TOKEN" + ); + } + #[test] fn test_escape_sql_string_single_quotes() { // Single quotes should be doubled From 25851acb65ef306223f83acdc8433e66edcf267f Mon Sep 17 00:00:00 2001 From: Tsubasa SEKIGUCHI Date: Sun, 12 Jul 2026 18:34:15 +0900 Subject: [PATCH 3/3] =?UTF-8?q?=E6=9D=B1=E6=80=A5=E3=83=90=E3=82=B9?= =?UTF-8?q?=E3=82=92GTFS=E3=83=BBODPT=20JSON=E3=81=8B=E3=82=89=E7=B5=B1?= =?UTF-8?q?=E5=90=88=20(#1600)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 4 + AGENTS.md | 6 +- Cargo.lock | 31 ++ README.md | 1 + data/1!companies.csv | 1 + docs/architecture.md | 4 +- stationapi/Cargo.toml | 2 +- stationapi/src/import.rs | 646 ++++++++++++++++++++++++++++++++++++++- 8 files changed, 686 insertions(+), 9 deletions(-) diff --git a/.gitignore b/.gitignore index 1c1f88d3..9f1d416e 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,10 @@ tmp.sql .vscode/ data/ToeiBus-GTFS/ data/SeibuBus-GTFS/ +data/TokyuBus-OtaCity-GTFS/ +data/TokyuBus-ShinagawaCity-GTFS/ +data/TokyuBus-MeguroCity-GTFS/ +data/TokyuBus-ODPT/ data/KeioBus-GTFS/ scripts/.osm_cache/ scripts/.gtfs_cache/ diff --git a/AGENTS.md b/AGENTS.md index d140c289..765dcc6f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -20,7 +20,7 @@ This guide explains how automation agents and human contributors should work wit - Environment variables: - `DATABASE_URL` – SQLx connection string (e.g., `postgres://stationapi:stationapi@localhost/stationapi`). - `DISABLE_GRPC_WEB` – `false` enables gRPC-Web; set to `true` for pure gRPC/HTTP2. - - `ODPT_ACCESS_TOKEN` – ODPT consumer key used to download authenticated GTFS feeds such as Seibu Bus and Keio Bus. + - `ODPT_ACCESS_TOKEN` – ODPT consumer key used to download authenticated data such as Seibu Bus GTFS, Keio Bus GTFS, Tokyu Bus JSON, and the Tokyu-operated Ota, Shinagawa, and Meguro community bus GTFS feeds. - `HOST` and `PORT` – listen address (defaults to `[::1]:50051`; Docker uses `0.0.0.0:50051`). - `.env.test` exports `TEST_DATABASE_URL`, `RUST_LOG`, `RUST_BACKTRACE`, and `RUST_TEST_THREADS=1` for integration tests. - Recommended: keep shared defaults in `.env`, copy overrides to `.env.local`, and rely on startup loading both files. @@ -53,8 +53,8 @@ This guide explains how automation agents and human contributors should work wit - **Stations** – `GetStationById`, `GetStationByIdList`, `GetStationsByGroupId`, `GetStationsByCoordinates`, `GetStationsByLineId`, `GetStationsByName`, `GetStationsByLineGroupId`. `QueryInteractor` enriches stations with lines, companies, station numbers, and train types. - **Lines** – `GetLineById`, `GetLinesByIdList`, `GetLinesByName`. Results include company data and computed line symbols based on repository helpers. - **Routes** – `GetRoutes`, `GetRoutesMinimal`. The minimal variant returns `RouteMinimalResponse` with deduplicated `LineMinimal` data; paging tokens are currently empty (pagination not implemented). -- **Train types** – `GetTrainTypesByStationId`, `GetRouteTypes`. Train types aggregate by line group and include related lines plus optional train type metadata. Rail variants use `TrainTypeKind::{Default, Branch, Rapid, Express, LimitedExpress, HighSpeedRapid, CommuterRapid}` (0-6); bus variants use `BusRoute` (7), which represents a `(route_id, shape_id)` operation pattern (e.g. 循環 / 短ターン / 支線) generated automatically from the configured GTFS bus feeds (Toei Bus, Seibu Bus, Keio Bus). -- **GTFS bus integration** – At startup, `src/import.rs::integrate_gtfs_to_stations()` ingests GTFS feeds into `gtfs_*` tables and then projects them onto the shared `stations` / `lines` / `types` / `station_station_types` tables. Every configured feed is imported, including Seibu Bus and Keio Bus (both downloaded from ODPT with `ODPT_ACCESS_TOKEN`). `transport_type` (0: rail, 1: bus) on both `stations` and `lines` keeps the two worlds queryable side by side. GTFS IDs are namespaced per feed before import to avoid cross-operator collisions. `line_cd` (100,000,000+), `station_cd` / `station_g_cd` (200,000,000+), and bus `type_cd` / `line_group_cd` (100,000,000+) are all deterministic fnv1a hashes that stay clear of the rail data ranges. Disable the entire bus pipeline with `DISABLE_BUS_FEATURE=true`. +- **Train types** – `GetTrainTypesByStationId`, `GetRouteTypes`. Train types aggregate by line group and include related lines plus optional train type metadata. Rail variants use `TrainTypeKind::{Default, Branch, Rapid, Express, LimitedExpress, HighSpeedRapid, CommuterRapid}` (0-6); bus variants use `BusRoute` (7), which represents a `(route_id, shape_id)` operation pattern (e.g. 循環 / 短ターン / 支線) generated automatically from the configured GTFS bus feeds (Toei Bus, Seibu Bus, Keio Bus) and the converted Tokyu Bus JSON. +- **GTFS bus integration** – At startup, `src/import.rs::integrate_gtfs_to_stations()` ingests GTFS feeds into `gtfs_*` tables and then projects them onto the shared `stations` / `lines` / `types` / `station_station_types` tables. Every configured GTFS feed is imported, including Seibu Bus and Keio Bus (both downloaded from ODPT with `ODPT_ACCESS_TOKEN`). Tokyu Bus ordinary-route `BusroutePattern`, `BusstopPole`, and `BusTimetable` JSON are converted into the same `gtfs_*` representation; pattern IDs become `shape_id` values so route variants remain queryable as bus TrainTypes. The Tokyu-operated Ota, Shinagawa, and Meguro community buses use their official GTFS feeds and matching JSON routes are excluded to prevent duplicates. `ODPT_ACCESS_TOKEN` is required for authenticated sources. Stops whose Tokyu JSON records omit coordinates remain available to name and route queries but not coordinate searches. `transport_type` (0: rail, 1: bus) on both `stations` and `lines` keeps rail and bus records queryable side by side. GTFS IDs are namespaced per feed before import to avoid cross-operator collisions. `line_cd` (100,000,000+), `station_cd` / `station_g_cd` (200,000,000+), and bus `type_cd` / `line_group_cd` (100,000,000+) are all deterministic fnv1a hashes that stay clear of the rail data ranges. Disable the entire bus pipeline with `DISABLE_BUS_FEATURE=true`. - **TTS metadata** – `Station`, `StationMinimal`, `Line`, and `TrainType` expose `name_ipa` / `name_roman_ipa` plus `name_tts_segments` for multi-segment pronunciation output. Use `name_tts_segments` when clients need per-token SSML construction for mixed-language names such as `Kasai-Rinkai Park`. - **Connected routes** – `GetConnectedRoutes`. `QueryInteractor::get_connected_stations` is not implemented yet and returns an empty vector; update the use-case and infrastructure layers together when adding real logic. - Changes to the service contract require coordinated updates to `proto/stationapi.proto`, regenerated code via `tonic-build`, and corresponding adjustments in both presentation and use-case layers. diff --git a/Cargo.lock b/Cargo.lock index fedd5421..662f9883 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -70,6 +70,18 @@ dependencies = [ "derive_arbitrary", ] +[[package]] +name = "async-compression" +version = "0.4.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e79b3f8a79cccc2898f31920fc69f304859b3bd567490f75ebf51ae1c792a9ac" +dependencies = [ + "compression-codecs", + "compression-core", + "pin-project-lite", + "tokio", +] + [[package]] name = "async-stream" version = "0.3.5" @@ -322,6 +334,23 @@ dependencies = [ "cc", ] +[[package]] +name = "compression-codecs" +version = "0.4.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce2548391e9c1929c21bf6aa2680af86fe4c1b33e6cea9ac1cfeec0bd11218cf" +dependencies = [ + "compression-core", + "flate2", + "memchr", +] + +[[package]] +name = "compression-core" +version = "0.4.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc14f565cf027a105f7a44ccf9e5b424348421a1d8952a8fc9d499d313107789" + [[package]] name = "concurrent-queue" version = "2.5.0" @@ -1943,6 +1972,7 @@ version = "0.12.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d19c46a6fdd48bc4dab94b6103fccc55d34c67cc0ad04653aad4ea2a07cd7bbb" dependencies = [ + "async-compression", "base64", "bytes", "futures-channel", @@ -1971,6 +2001,7 @@ dependencies = [ "sync_wrapper", "tokio", "tokio-rustls", + "tokio-util", "tower 0.5.2", "tower-service", "url", diff --git a/README.md b/README.md index a1edaae6..f4421695 100644 --- a/README.md +++ b/README.md @@ -112,6 +112,7 @@ We follow Rust best practices for testing: - Bus-related data provided by [Tokyo Metropolitan Bureau of Transportation (Toei)](https://www.kotsu.metro.tokyo.jp/), licensed under [CC BY 4.0](https://creativecommons.org/licenses/by/4.0/) - Bus-related data provided by [Seibu Bus Co., Ltd. (西武バス)](https://www.seibubus.co.jp/) via the [Public Transportation Open Data Center](https://ckan.odpt.org/), licensed under the [Public Transportation Open Data Basic License](https://developer.odpt.org/terms) - Bus-related data provided by [Keio Dentetsu Bus Co., Ltd. (京王電鉄バス)](https://www.keio-bus.com/) via the [Public Transportation Open Data Center](https://ckan.odpt.org/), licensed under the [Public Transportation Open Data Basic License](https://developer.odpt.org/terms) +- Bus data provided by [Tokyu Bus Corporation (東急バス)](https://www.tokyubus.co.jp/) via the [Public Transportation Open Data Center](https://ckan.odpt.org/), licensed under the [Public Transportation Open Data Basic License](https://developer.odpt.org/terms) - Station data provided by [駅データ.jp](https://www.ekidata.jp/) - Speed calibration data (`speed_table.rs`) derived from GTFS timetables provided by Kyoto City Transportation Bureau (京都市交通局), Yokohama City Transportation Bureau (横浜市交通局), Tokyo Metro (東京メトロ), Metropolitan Intercity Railway (首都圏新都市鉄道), Tokyo Tama Intercity Monorail (多摩都市モノレール), and Tokyo Waterfront Area Rapid Transit (東京臨海高速鉄道) via the [Public Transportation Open Data Center](https://ckan.odpt.org/), licensed under the [Public Transportation Open Data Basic License](https://developer.odpt.org/terms); by [Tokyo Metropolitan Bureau of Transportation (Toei)](https://www.kotsu.metro.tokyo.jp/), licensed under [CC BY 4.0](https://creativecommons.org/licenses/by/4.0/); and by Hakodate City Enterprise Bureau Transportation Department (函館市企業局交通部), licensed under [GTFS-RUL (ODPT)](https://gtfs-jp.org/GTFS-RUL(ODPT).pdf) - Average inter-station distances (`average_distance`) computed from railway track geometry © [OpenStreetMap contributors](https://www.openstreetmap.org/copyright), licensed under [ODbL](https://opendatacommons.org/licenses/odbl/) diff --git a/data/1!companies.csv b/data/1!companies.csv index f9c931f8..7984c1a4 100644 --- a/data/1!companies.csv +++ b/data/1!companies.csv @@ -9,6 +9,7 @@ company_cd,rr_cd,company_name,company_name_k,company_name_h,company_name_r,compa 12,22,西武鉄道,セイブテツドウ,西武鉄道株式会社,西武,Seibu,Seibu Railway,http://www.seibu-group.co.jp/railways/,2,0,12 253,99,西武バス,セイブバス,西武バス株式会社,西武バス,Seibu Bus,Seibu Bus,https://www.seibubus.co.jp/,0,0,12 254,99,京王バス,ケイオウバス,京王電鉄バス株式会社,京王バス,Keio Bus,Keio Bus,https://www.keio-bus.com/,0,0,14 +255,99,東急バス,トウキュウバス,東急バス株式会社,東急バス,Tokyu Bus,Tokyu Bus Corporation,https://www.tokyubus.co.jp/,0,0,16 13,23,京成電鉄,ケイセイデンテツ,京成電鉄株式会社,京成,Keisei,Keisei Electric Railway,http://www.keisei.co.jp/,2,0,13 14,24,京王電鉄,ケイオウデンテツ,京王電鉄株式会社,京王,Keio,Keio,http://www.keio.co.jp/,2,0,14 15,25,小田急電鉄,オダキュウデンテツ,小田急電鉄株式会社,小田急,Odakyu,Odakyu Electric Railway,http://www.odakyu.jp/,2,0,15 diff --git a/docs/architecture.md b/docs/architecture.md index d3200cc3..114d8148 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -166,11 +166,11 @@ CREATE INDEX idx_performance_station_name_trgm ON stations ### バス (GTFS) データの統合 -`stations` / `lines` / `types` / `station_station_types` は鉄道とバスの両方を保持し、`stations.transport_type` / `lines.transport_type` (0: 鉄道, 1: バス) でフィルタリングします。バスデータの取り込みは `src/import.rs` の `integrate_gtfs_to_stations()` が起動時に実行し、ODPT 公開の GTFS を `gtfs_*` テーブルに展開してから既存テーブルへ統合します。Seibu Bus・Keio Bus を含む設定済みの全GTFSフィードを使用します。Seibu Bus・Keio Bus のダウンロードには `.env.local` の `ODPT_ACCESS_TOKEN` を使用します。 +`stations` / `lines` / `types` / `station_station_types` は鉄道とバスの両方を保持し、`stations.transport_type` / `lines.transport_type` (0: 鉄道, 1: バス) でフィルタリングします。バスデータの取り込みは `src/import.rs` の `integrate_gtfs_to_stations()` が起動時に実行し、ODPT 公開のデータを `gtfs_*` テーブルに展開してから既存テーブルへ統合します。都営バス・西武バス・京王バスを含む設定済みの全GTFSフィードを直接読み込みます。東急バスの一般路線は `BusroutePattern` / `BusstopPole` / `BusTimetable` JSONをGTFS相当の路線・停留所・便・停車時刻へ変換し、路線パターンIDを `shape_id` として扱います。ODPTのJSON APIは全事業者分の大きな配列を返すため、レスポンスをファイルへストリーミングし、1件ずつ東急バス分だけ抽出した7日間キャッシュを `data/TokyuBus-ODPT/` に保持します。大田区（たまちゃんバス）・品川区（しなバス）・目黒区（さんまバス）は公式GTFSを使用し、JSON側の同一路線を除外して重複を防ぎます。認証付きデータ（西武バス・京王バスのGTFSや東急バスのJSON）のダウンロードには `.env.local` の `ODPT_ACCESS_TOKEN` を使用します。なお、ODPT JSONで座標が欠落している一般路線停留所は名称・路線検索には含まれますが、座標検索には含められません。 | 鉄道側の概念 | バス側の対応 | |---|---| -| `companies` | GTFS フィードごとに会社を決定。Toei Bus は東京都交通局 (`company_cd=119`)、Seibu Bus は西武バス (`company_cd=253`)、Keio Bus は京王バス (`company_cd=254`) | +| `companies` | データソースごとに会社を決定。Toei Bus は東京都交通局 (`company_cd=119`)、Seibu Bus は西武バス (`company_cd=253`)、Keio Bus は京王バス (`company_cd=254`)、東急バスのJSON変換データとコミュニティバス3フィードは東急バス (`company_cd=255`) | | `lines` | GTFS `routes` を 1:1 で `lines` に登録。`line_cd` はフィード接頭辞付き `route_id` の fnv1a ハッシュで 100,000,000+ 空間に決定的に生成 | | `stations` | GTFS `stops` (親停留所) を `(stop_id, route_id)` 単位で `stations` に登録。`station_cd` / `station_g_cd` もフィード接頭辞付き ID から 200,000,000+ 空間にハッシュ生成 | | `types` | GTFS の `(route_id, shape_id)` バリエーション (フルループ / 短ターン / 支線など) を `kind = TrainTypeKind::BusRoute (= 7)` の TrainType として登録。**停留所集合が完全に同じ shape ペア (上下方向違いのみ) は 1 つの TrainType に畳み、`direction = Both` を設定**。`type_name` は循環なら ` (循環)`、双方向ペアなら ` ⇔ `、片方向なら `<始発停留所> → ` | diff --git a/stationapi/Cargo.toml b/stationapi/Cargo.toml index 1578a638..efaac3dd 100644 --- a/stationapi/Cargo.toml +++ b/stationapi/Cargo.toml @@ -28,7 +28,7 @@ tonic-health = "0.12.3" tonic-reflection = "0.12.3" csv = "1.3.1" chrono = ">=0.4.20" -reqwest = { version = "0.12.12", default-features = false, features = ["blocking", "rustls-tls"] } +reqwest = { version = "0.12.12", default-features = false, features = ["blocking", "rustls-tls", "gzip"] } zip = ">=2.3.0" metrics = "0.24" metrics-exporter-prometheus = "0.16" diff --git a/stationapi/src/import.rs b/stationapi/src/import.rs index 10283f9b..04bff528 100644 --- a/stationapi/src/import.rs +++ b/stationapi/src/import.rs @@ -1,10 +1,14 @@ //! Data import module for CSV and GTFS data use csv::{ReaderBuilder, StringRecord}; +use serde::de::{DeserializeOwned, DeserializeSeed, SeqAccess, Visitor}; +use serde::{Deserialize, Serialize}; use sqlx::{Connection, PgConnection}; use stationapi::config::fetch_database_url; -use std::collections::HashMap; -use std::io::{Cursor, Read as _}; +use std::collections::{HashMap, HashSet}; +use std::fmt; +use std::fs::File; +use std::io::{BufReader, BufWriter, Cursor, Read, Write}; use std::path::Path; use std::{env, fs}; use tracing::{info, warn}; @@ -286,6 +290,27 @@ const GTFS_FEEDS: &[GtfsFeed] = &[ url: "https://api.odpt.org/api/v4/files/SeibuBus/data/SeibuBus-GTFS.zip", requires_consumer_key: true, }, + GtfsFeed { + id: "tokyu_ota", + name: "Tokyu Bus (Ota City Community Bus)", + path: "data/TokyuBus-OtaCity-GTFS", + url: "https://api.odpt.org/api/v4/files/odpt/TokyuBus/tokyubus_community_OtaCity.zip?date=current", + requires_consumer_key: true, + }, + GtfsFeed { + id: "tokyu_shinagawa", + name: "Tokyu Bus (Shinagawa City Community Bus)", + path: "data/TokyuBus-ShinagawaCity-GTFS", + url: "https://api.odpt.org/api/v4/files/odpt/TokyuBus/tokyubus_community_ShinagawaCity.zip?date=current", + requires_consumer_key: true, + }, + GtfsFeed { + id: "tokyu_meguro", + name: "Tokyu Bus (Meguro City Community Bus)", + path: "data/TokyuBus-MeguroCity-GTFS", + url: "https://api.odpt.org/api/v4/files/odpt/TokyuBus/tokyubus_community_MeguroCity.zip?date=current", + requires_consumer_key: true, + }, GtfsFeed { // ODPT's versioned files endpoint requires a `date` selector; `current` // always resolves to the timetable in effect today (omitting it 404s). @@ -298,6 +323,257 @@ const GTFS_FEEDS: &[GtfsFeed] = &[ ]; const DEFAULT_GTFS_BUS_LINE_COLOR: &str = "#1f63c6"; +const TOKYU_ODPT_PREFIX: &str = "tokyu_json"; +const TOKYU_ODPT_OPERATOR: &str = "odpt.Operator:TokyuBus"; +const TOKYU_ODPT_CACHE_MAX_AGE: std::time::Duration = + std::time::Duration::from_secs(7 * 24 * 60 * 60); + +#[derive(Debug, Deserialize, Serialize)] +struct OdptBusroutePattern { + #[serde(rename = "owl:sameAs")] + same_as: String, + #[serde(rename = "dc:title", default)] + title: String, + #[serde(rename = "odpt:operator")] + operator: String, + #[serde(rename = "odpt:busroute")] + busroute: String, + #[serde(rename = "odpt:direction")] + direction: Option, +} + +#[derive(Debug, Deserialize, Serialize)] +struct OdptBusstopPole { + #[serde(rename = "owl:sameAs")] + same_as: String, + #[serde(rename = "dc:title", default)] + title: String, + #[serde(rename = "odpt:kana", default)] + kana: String, + #[serde(rename = "odpt:busstopPoleNumber")] + number: Option, + #[serde(rename = "odpt:operator", default)] + operators: Vec, + #[serde(rename = "geo:lat", default)] + lat: f64, + #[serde(rename = "geo:long", default)] + lon: f64, +} + +#[derive(Debug, Deserialize, Serialize)] +struct OdptBusTimetable { + #[serde(rename = "owl:sameAs")] + same_as: String, + #[serde(rename = "odpt:operator")] + operator: String, + #[serde(rename = "odpt:busroutePattern")] + busroute_pattern: String, + #[serde(rename = "odpt:calendar")] + calendar: String, + #[serde(rename = "odpt:busTimetableObject", default)] + objects: Vec, +} + +#[derive(Debug, Deserialize, Serialize)] +struct OdptBusTimetableObject { + #[serde(rename = "odpt:index")] + index: i32, + #[serde(rename = "odpt:busstopPole")] + busstop_pole: String, + #[serde(rename = "odpt:arrivalTime")] + arrival_time: Option, + #[serde(rename = "odpt:departureTime")] + departure_time: Option, + #[serde(rename = "odpt:destinationSign")] + destination_sign: Option, + #[serde(rename = "odpt:canGetOn")] + can_get_on: Option, + #[serde(rename = "odpt:canGetOff")] + can_get_off: Option, +} + +struct TokyuOdptData { + patterns: Vec, + stops: Vec, + timetables: Vec, +} + +struct TokyuOnlySeed(std::marker::PhantomData); + +impl<'de, T> DeserializeSeed<'de> for TokyuOnlySeed +where + T: DeserializeOwned, +{ + type Value = Vec; + + fn deserialize(self, deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + deserializer.deserialize_seq(TokyuOnlyVisitor(std::marker::PhantomData)) + } +} + +struct TokyuOnlyVisitor(std::marker::PhantomData); + +impl<'de, T> Visitor<'de> for TokyuOnlyVisitor +where + T: DeserializeOwned, +{ + type Value = Vec; + + fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("an ODPT JSON array") + } + + fn visit_seq(self, mut sequence: A) -> Result + where + A: SeqAccess<'de>, + { + let mut selected = Vec::new(); + while let Some(value) = sequence.next_element::()? { + let belongs_to_tokyu = match value.get("odpt:operator") { + Some(serde_json::Value::String(operator)) => operator == TOKYU_ODPT_OPERATOR, + Some(serde_json::Value::Array(operators)) => operators + .iter() + .any(|operator| operator.as_str() == Some(TOKYU_ODPT_OPERATOR)), + _ => false, + }; + if belongs_to_tokyu { + selected.push(serde_json::from_value(value).map_err(|error| { + serde::de::Error::custom(format!("invalid Tokyu Bus record: {}", error)) + })?); + } + } + Ok(selected) + } +} + +fn read_tokyu_odpt_items(path: &Path) -> Result, Box> +where + T: DeserializeOwned, +{ + read_tokyu_odpt_items_from_reader(BufReader::new(File::open(path)?)) +} + +fn read_tokyu_odpt_items_from_reader( + reader: R, +) -> Result, Box> +where + T: DeserializeOwned, + R: Read, +{ + let mut deserializer = serde_json::Deserializer::from_reader(reader); + Ok(TokyuOnlySeed(std::marker::PhantomData).deserialize(&mut deserializer)?) +} + +fn strip_odpt_prefix(value: &str) -> &str { + value.split_once(':').map_or(value, |(_, id)| id) +} + +fn scoped_tokyu_odpt_id(value: &str) -> String { + format!("{}:{}", TOKYU_ODPT_PREFIX, strip_odpt_prefix(value)) +} + +fn odpt_direction_id(direction: Option<&str>) -> Option { + match direction { + Some("1") => Some(0), + Some("2") => Some(1), + _ => None, + } +} + +fn tokyu_route_name(title: &str) -> &str { + title.split_whitespace().next().unwrap_or(title) +} + +fn is_tokyu_community_route(busroute: &str) -> bool { + matches!( + strip_odpt_prefix(busroute), + "TokyuBus.Tamachan" | "TokyuBus.Shinabasu" | "TokyuBus.Sanma" + ) +} + +fn download_odpt_json( + client: &reqwest::blocking::Client, + resource: &str, + token: &str, +) -> Result, Box> +where + T: DeserializeOwned + Serialize, +{ + let cache_dir = Path::new(env!("CARGO_MANIFEST_DIR")).join("../data/TokyuBus-ODPT"); + let cache_path = cache_dir.join(format!( + "{}.tokyu.json", + resource.rsplit(':').next().unwrap_or(resource) + )); + if let Ok(metadata) = fs::metadata(&cache_path) { + if metadata + .modified() + .ok() + .and_then(|modified| modified.elapsed().ok()) + .is_some_and(|age| age < TOKYU_ODPT_CACHE_MAX_AGE) + { + info!("Using cached Tokyu Bus {} JSON.", resource); + return read_tokyu_odpt_items(&cache_path); + } + } + + let url = format!("https://api.odpt.org/api/v4/{}.json", resource); + let mut response = client + .get(url) + .query(&[ + ("odpt:operator", TOKYU_ODPT_OPERATOR), + ("acl:consumerKey", token), + ]) + .send()?; + if !response.status().is_success() { + return Err(format!( + "Failed to download {}: HTTP {}", + resource, + response.status() + ) + .into()); + } + fs::create_dir_all(&cache_dir)?; + let download_path = cache_path.with_extension("download.tmp"); + let mut download_file = BufWriter::new(File::create(&download_path)?); + std::io::copy(&mut response, &mut download_file)?; + download_file.flush()?; + drop(download_file); + + let selected = match read_tokyu_odpt_items(&download_path) { + Ok(selected) => selected, + Err(error) => { + let _ = fs::remove_file(&download_path); + return Err(error); + } + }; + fs::remove_file(&download_path)?; + let temporary_path = cache_path.with_extension("json.tmp"); + let mut cache_file = BufWriter::new(File::create(&temporary_path)?); + serde_json::to_writer(&mut cache_file, &selected)?; + cache_file.flush()?; + drop(cache_file); + if cache_path.exists() { + fs::remove_file(&cache_path)?; + } + fs::rename(temporary_path, &cache_path)?; + Ok(selected) +} + +fn download_tokyu_odpt_data() -> Result> { + let token = env::var("ODPT_ACCESS_TOKEN") + .map_err(|_| "Tokyu Bus ODPT JSON requires ODPT_ACCESS_TOKEN in the environment")?; + let client = reqwest::blocking::Client::builder() + .timeout(std::time::Duration::from_secs(300)) + .build()?; + Ok(TokyuOdptData { + patterns: download_odpt_json(&client, "odpt:BusroutePattern", &token)?, + stops: download_odpt_json(&client, "odpt:BusstopPole", &token)?, + timetables: download_odpt_json(&client, "odpt:BusTimetable", &token)?, + }) +} fn scoped_gtfs_id(feed: &GtfsFeed, id: &str) -> String { format!("{}:{}", feed.id, id) @@ -484,6 +760,17 @@ pub async fn import_gtfs() -> Result<(), Box> { } } } + let tokyu_odpt_data = tokio::task::spawn_blocking(download_tokyu_odpt_data) + .await + .map_err(|e| format!("Failed to spawn Tokyu Bus JSON download: {}", e))? + .map_err(|e| -> Box { e })?; + info!( + "[gtfs:{}] downloaded ODPT JSON: {} patterns, {} stops, {} timetables", + TOKYU_ODPT_PREFIX, + tokyu_odpt_data.patterns.len(), + tokyu_odpt_data.stops.len(), + tokyu_odpt_data.timetables.len() + ); info!( "[gtfs] download/extract finished in {:?}", download_start.elapsed() @@ -630,6 +917,8 @@ pub async fn import_gtfs() -> Result<(), Box> { ); } + import_tokyu_odpt_data(&mut tx, &tokyu_odpt_data).await?; + info!("[gtfs] committing transaction"); let commit_start = std::time::Instant::now(); tx.commit().await?; @@ -646,6 +935,199 @@ pub async fn import_gtfs() -> Result<(), Box> { Ok(()) } +async fn import_tokyu_odpt_data( + conn: &mut PgConnection, + data: &TokyuOdptData, +) -> Result<(), Box> { + let agency_id = format!("{}:TokyuBus", TOKYU_ODPT_PREFIX); + sqlx::query( + r#"INSERT INTO gtfs_agencies + (agency_id, agency_name, agency_name_k, agency_name_r, agency_url, + agency_timezone, agency_lang, company_cd) + VALUES ($1, '東急バス', 'トウキュウバス', 'Tokyu Bus', + 'https://www.tokyubus.co.jp/', 'Asia/Tokyo', 'ja', 255) + ON CONFLICT (agency_id) DO NOTHING"#, + ) + .bind(&agency_id) + .execute(&mut *conn) + .await?; + + let mut pattern_map: HashMap)> = HashMap::new(); + let mut inserted_routes = HashSet::new(); + for pattern in data.patterns.iter().filter(|pattern| { + pattern.operator == TOKYU_ODPT_OPERATOR && !is_tokyu_community_route(&pattern.busroute) + }) { + let route_id = scoped_tokyu_odpt_id(&pattern.busroute); + let pattern_id = scoped_tokyu_odpt_id(&pattern.same_as); + pattern_map.insert( + pattern_id, + ( + route_id.clone(), + odpt_direction_id(pattern.direction.as_deref()), + ), + ); + + if inserted_routes.insert(route_id.clone()) { + let route_name = tokyu_route_name(&pattern.title); + sqlx::query( + r#"INSERT INTO gtfs_routes + (route_id, agency_id, route_short_name, route_long_name, + route_long_name_r, route_type, route_color) + VALUES ($1, $2, $3, $3, NULL, 3, 'DD1133') + ON CONFLICT (route_id) DO NOTHING"#, + ) + .bind(&route_id) + .bind(&agency_id) + .bind(route_name) + .execute(&mut *conn) + .await?; + } + } + + let mut stop_ids = HashSet::new(); + let mut stop_values = Vec::new(); + let mut missing_coordinates = 0_usize; + for stop in data.stops.iter().filter(|stop| { + stop.operators + .iter() + .any(|operator| operator == TOKYU_ODPT_OPERATOR) + }) { + let stop_id = scoped_tokyu_odpt_id(&stop.same_as); + stop_ids.insert(stop_id.clone()); + if stop.lat == 0.0 || stop.lon == 0.0 { + missing_coordinates += 1; + } + stop_values.push(format!( + "('{}', '{}', '{}', '{}', NULL, {}, {})", + escape_sql_string(&stop_id), + escape_sql_string(stop.number.as_deref().unwrap_or("")), + escape_sql_string(&stop.title), + escape_sql_string(&hiragana_to_katakana(&stop.kana)), + stop.lat, + stop.lon + )); + if stop_values.len() == 500 { + insert_tokyu_stop_values(&mut *conn, &stop_values).await?; + stop_values.clear(); + } + } + insert_tokyu_stop_values(&mut *conn, &stop_values).await?; + if missing_coordinates > 0 { + warn!( + "Tokyu Bus ODPT JSON contains {} stops without coordinates; name/route queries work, but coordinate queries cannot return those stops", + missing_coordinates + ); + } + + let mut trips = Vec::with_capacity(2_000); + let mut stop_times = Vec::with_capacity(1_000); + let mut trip_count = 0_usize; + let mut stop_time_count = 0_usize; + for timetable in data + .timetables + .iter() + .filter(|timetable| timetable.operator == TOKYU_ODPT_OPERATOR) + { + let pattern_id = scoped_tokyu_odpt_id(&timetable.busroute_pattern); + let Some((route_id, direction_id)) = pattern_map.get(&pattern_id) else { + continue; + }; + let trip_id = scoped_tokyu_odpt_id(&timetable.same_as); + let service_id = scoped_tokyu_odpt_id(&timetable.calendar); + let headsign = timetable + .objects + .iter() + .find_map(|object| object.destination_sign.clone()); + trips.push(( + trip_id.clone(), + route_id.clone(), + service_id, + headsign, + None, + *direction_id, + None, + Some(pattern_id), + None, + None, + )); + trip_count += 1; + if trips.len() == 2_000 { + insert_trips_batch(&mut *conn, &trips).await?; + trips.clear(); + } + + for object in &timetable.objects { + let stop_id = scoped_tokyu_odpt_id(&object.busstop_pole); + if !stop_ids.contains(&stop_id) { + continue; + } + let arrival = object + .arrival_time + .as_deref() + .or(object.departure_time.as_deref()) + .and_then(parse_odpt_time); + let departure = object + .departure_time + .as_deref() + .or(object.arrival_time.as_deref()) + .and_then(parse_odpt_time); + if arrival.is_none() && departure.is_none() { + continue; + } + stop_times.push(( + trip_id.clone(), + arrival, + departure, + stop_id, + object.index, + object.destination_sign.clone(), + object.can_get_on.map(|allowed| if allowed { 0 } else { 1 }), + object + .can_get_off + .map(|allowed| if allowed { 0 } else { 1 }), + None, + None, + )); + stop_time_count += 1; + if stop_times.len() == 1_000 { + // stop_times has a foreign key to trips. Flush every pending trip + // before its stop-time batch, even when the trip batch is not full. + insert_trips_batch(&mut *conn, &trips).await?; + trips.clear(); + insert_stop_times_batch(&mut *conn, &stop_times).await?; + stop_times.clear(); + } + } + } + + insert_trips_batch(&mut *conn, &trips).await?; + insert_stop_times_batch(&mut *conn, &stop_times).await?; + + info!( + "Imported Tokyu Bus ODPT JSON as GTFS: {} routes, {} stops, {} trips, {} stop times.", + inserted_routes.len(), + stop_ids.len(), + trip_count, + stop_time_count + ); + Ok(()) +} + +async fn insert_tokyu_stop_values( + conn: &mut PgConnection, + values: &[String], +) -> Result<(), Box> { + if values.is_empty() { + return Ok(()); + } + let sql = format!( + "INSERT INTO gtfs_stops (stop_id, stop_code, stop_name, stop_name_k, stop_name_r, stop_lat, stop_lon) VALUES {} ON CONFLICT (stop_id) DO NOTHING", + values.join(",") + ); + sqlx::query(&sql).execute(&mut *conn).await?; + Ok(()) +} + /// Load translations from translations.txt fn load_gtfs_translations( gtfs_path: &Path, @@ -1507,6 +1989,21 @@ fn parse_gtfs_time(time_str: &str) -> Option { Some(time_str.to_string()) } +/// ODPT bus timetable values use HH:MM, while GTFS requires HH:MM:SS. +fn parse_odpt_time(time_str: &str) -> Option { + if let Some(gtfs_time) = parse_gtfs_time(time_str) { + return Some(gtfs_time); + } + + let (hours, minutes) = time_str.split_once(':')?; + let hours: u32 = hours.parse().ok()?; + let minutes: u32 = minutes.parse().ok()?; + if minutes >= 60 { + return None; + } + Some(format!("{hours:02}:{minutes:02}:00")) +} + async fn insert_stop_times_batch( conn: &mut PgConnection, batch: &[StopTimeBatchRow], @@ -1736,6 +2233,8 @@ fn company_cd_for_gtfs_route(route_id: &str) -> Option { Some(253) // Seibu Bus } else if route_id.starts_with("keio:") { Some(254) // Keio Bus + } else if route_id.starts_with("tokyu_") { + Some(255) // Tokyu Bus } else { None // Unknown/unsupported prefix } @@ -3048,6 +3547,15 @@ mod tests { assert_eq!(parse_gtfs_time("aa:bb:cc"), None); } + #[test] + fn test_parse_odpt_time_adds_gtfs_seconds() { + assert_eq!(parse_odpt_time("08:30"), Some("08:30:00".to_string())); + assert_eq!(parse_odpt_time("8:03"), Some("08:03:00".to_string())); + assert_eq!(parse_odpt_time("25:30:00"), Some("25:30:00".to_string())); + assert_eq!(parse_odpt_time("08:60"), None); + assert_eq!(parse_odpt_time("invalid"), None); + } + #[test] fn test_hiragana_to_katakana() { assert_eq!(hiragana_to_katakana("あいうえお"), "アイウエオ"); @@ -3102,6 +3610,19 @@ mod tests { assert_eq!(company_cd_for_gtfs_route("toei:route_001"), Some(119)); assert_eq!(company_cd_for_gtfs_route("seibu:route_001"), Some(253)); assert_eq!(company_cd_for_gtfs_route("keio:route_001"), Some(254)); + assert_eq!(company_cd_for_gtfs_route("tokyu_ota:route_001"), Some(255)); + assert_eq!( + company_cd_for_gtfs_route("tokyu_shinagawa:route_001"), + Some(255) + ); + assert_eq!( + company_cd_for_gtfs_route("tokyu_meguro:route_001"), + Some(255) + ); + assert_eq!( + company_cd_for_gtfs_route("tokyu_json:TokyuBus.Route"), + Some(255) + ); assert_eq!(company_cd_for_gtfs_route("unknown:route_001"), None); } @@ -3190,7 +3711,14 @@ mod tests { fn test_gtfs_feeds() { assert_eq!( GTFS_FEEDS.iter().map(|feed| feed.id).collect::>(), - vec!["toei", "seibu", "keio"] + vec![ + "toei", + "seibu", + "tokyu_ota", + "tokyu_shinagawa", + "tokyu_meguro", + "keio" + ] ); let keio = GTFS_FEEDS.iter().find(|feed| feed.id == "keio").unwrap(); @@ -3203,6 +3731,14 @@ mod tests { "https://api.odpt.org/api/v4/files/odpt/KeioBus/AllLines.zip?date=current" ); assert!(keio.requires_consumer_key); + + for feed in GTFS_FEEDS + .iter() + .filter(|feed| feed.id.starts_with("tokyu_")) + { + assert!(feed.url.contains("date=current")); + assert!(feed.requires_consumer_key); + } } #[test] @@ -3226,6 +3762,110 @@ mod tests { ); } + #[test] + fn test_gtfs_download_url_preserves_existing_query() { + let feed = GTFS_FEEDS + .iter() + .find(|feed| feed.id == "tokyu_ota") + .unwrap(); + assert_eq!( + append_consumer_key(feed.url, "test-token"), + "https://api.odpt.org/api/v4/files/odpt/TokyuBus/tokyubus_community_OtaCity.zip?date=current&acl:consumerKey=test-token" + ); + + let feed = GTFS_FEEDS.iter().find(|feed| feed.id == "seibu").unwrap(); + assert_eq!( + append_consumer_key(feed.url, "test-token"), + "https://api.odpt.org/api/v4/files/SeibuBus/data/SeibuBus-GTFS.zip?acl:consumerKey=test-token" + ); + } + + #[test] + fn test_tokyu_odpt_json_deserialization_and_id_conversion() { + let pattern: OdptBusroutePattern = serde_json::from_str( + r#"{ + "owl:sameAs":"odpt.BusroutePattern:TokyuBus.Sh11.1", + "dc:title":"渋11 田園調布駅行き", + "odpt:operator":"odpt.Operator:TokyuBus", + "odpt:busroute":"odpt.Busroute:TokyuBus.Sh11", + "odpt:direction":"2" + }"#, + ) + .unwrap(); + assert_eq!( + scoped_tokyu_odpt_id(&pattern.same_as), + "tokyu_json:TokyuBus.Sh11.1" + ); + assert_eq!(tokyu_route_name(&pattern.title), "渋11"); + assert_eq!(odpt_direction_id(pattern.direction.as_deref()), Some(1)); + + let stop: OdptBusstopPole = serde_json::from_str( + r#"{ + "owl:sameAs":"odpt.BusstopPole:TokyuBus.123.a", + "dc:title":"渋谷駅", + "odpt:kana":"しぶやえき", + "odpt:busstopPoleNumber":"01", + "odpt:operator":["odpt.Operator:TokyuBus"] + }"#, + ) + .unwrap(); + assert_eq!(stop.lat, 0.0); + assert_eq!(stop.lon, 0.0); + assert_eq!(stop.operators, vec![TOKYU_ODPT_OPERATOR]); + } + + #[test] + fn test_tokyu_odpt_streaming_filter_discards_other_operators() { + let json = r#"[ + { + "owl:sameAs":"odpt.BusroutePattern:TokyuBus.Sh11.1", + "dc:title":"渋11", + "odpt:operator":"odpt.Operator:TokyuBus", + "odpt:busroute":"odpt.Busroute:TokyuBus.Sh11" + }, + { + "owl:sameAs":"odpt.BusroutePattern:Other.1", + "dc:title":"Other", + "odpt:operator":"odpt.Operator:Other", + "odpt:busroute":"odpt.Busroute:Other.1" + } + ]"#; + let selected: Vec = + read_tokyu_odpt_items_from_reader(json.as_bytes()).unwrap(); + assert_eq!(selected.len(), 1); + assert_eq!(selected[0].operator, TOKYU_ODPT_OPERATOR); + } + + #[test] + #[ignore = "requires ODPT_ACCESS_TOKEN and downloads the large ODPT bus dumps"] + fn test_live_tokyu_odpt_download_and_filter() { + let _ = dotenv::from_filename(".env.local"); + let data = download_tokyu_odpt_data().unwrap(); + assert!(!data.patterns.is_empty()); + assert!(!data.stops.is_empty()); + assert!(!data.timetables.is_empty()); + assert!(data + .patterns + .iter() + .all(|pattern| pattern.operator == TOKYU_ODPT_OPERATOR)); + assert!(data.stops.iter().all(|stop| stop + .operators + .iter() + .any(|operator| operator == TOKYU_ODPT_OPERATOR))); + assert!(data + .timetables + .iter() + .all(|timetable| timetable.operator == TOKYU_ODPT_OPERATOR)); + } + + #[test] + fn test_tokyu_community_routes_are_excluded_from_json_conversion() { + assert!(is_tokyu_community_route("odpt.Busroute:TokyuBus.Tamachan")); + assert!(is_tokyu_community_route("odpt.Busroute:TokyuBus.Shinabasu")); + assert!(is_tokyu_community_route("odpt.Busroute:TokyuBus.Sanma")); + assert!(!is_tokyu_community_route("odpt.Busroute:TokyuBus.Sh11")); + } + #[test] fn test_escape_sql_string_single_quotes() { // Single quotes should be doubled