From 8fab28b1b50c0e6f2f3ca0d8e5c509284bba862c Mon Sep 17 00:00:00 2001 From: satyakwok <119509589+satyakwok@users.noreply.github.com> Date: Mon, 8 Jun 2026 05:12:16 +0200 Subject: [PATCH 1/2] feat(api): align stats/daily + whale/tx with the legacy indexer contract MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The explorer frontend consumes a fixed set of read endpoints and can be pointed at either indexer. Three gaps stopped indexer-rs from serving them: 1. /stats/daily returned an object-wrapped, never-populated payload ({"daily":[]} with day_bucket/block_count fields). It now returns a bare array [{date, blocks, transactions}] — calendar date derived from the day_bucket epoch in SQL (to_timestamp at UTC), numeric counts — matching the legacy indexer the frontend was built against. 2. stats_daily_mv was never refreshed (the refresh helper had no caller), so it stayed empty despite millions of indexed blocks. The indexer now refreshes it on an interval (INDEXER_STATS_REFRESH_SECS, default 300s): first tick does a plain REFRESH (CONCURRENTLY is rejected on a never-populated MV), subsequent ticks use CONCURRENTLY so reads aren't blocked. 3. Added /whale/tx as an alias of /whale/transfers — the legacy path name. /accounts/active + /contracts/* already share the legacy shape. Contract leaderboards (recent/pioneers) still need data population — tracked separately. --- bin/indexer.rs | 40 +++++++++++++ crates/api/src/routes/leaderboards.rs | 5 ++ crates/api/src/routes/stats.rs | 86 +++++++++++++++++---------- crates/db/src/stats.rs | 46 +++++++------- 4 files changed, 125 insertions(+), 52 deletions(-) diff --git a/bin/indexer.rs b/bin/indexer.rs index b82c936..b57f123 100644 --- a/bin/indexer.rs +++ b/bin/indexer.rs @@ -50,6 +50,8 @@ struct IndexerConfig { indexer_backfill_loop_secs: u64, #[serde(default = "default_analytics_flush_secs")] indexer_analytics_flush_secs: u64, + #[serde(default = "default_stats_refresh_secs")] + indexer_stats_refresh_secs: u64, } fn default_network() -> String { @@ -67,6 +69,9 @@ fn default_clickhouse_table() -> String { fn default_analytics_flush_secs() -> u64 { 15 } +fn default_stats_refresh_secs() -> u64 { + 300 +} #[tokio::main] async fn main() -> anyhow::Result<()> { @@ -221,10 +226,45 @@ async fn main() -> anyhow::Result<()> { }) }; + // Stats MV refresh loop. `stats_daily_mv` (migration 0002) has no + // auto-refresh; without this it stays empty and `/stats/daily` returns + // nothing. The first tick fires immediately and does a plain (blocking) + // refresh — Postgres rejects `REFRESH ... CONCURRENTLY` on a + // never-populated MV — then every subsequent tick uses CONCURRENTLY so + // reads are never locked out. + let stats_refresh_handle = { + let pool = pool.clone(); + let cancel = cancel.clone(); + let interval = Duration::from_secs(cfg.indexer_stats_refresh_secs); + tokio::spawn(async move { + let mut populated = false; + let mut tick = tokio::time::interval(interval); + loop { + tokio::select! { + _ = cancel.cancelled() => return Ok::<(), anyhow::Error>(()), + _ = tick.tick() => { + let res = if populated { + indexer_db::stats::refresh(&pool).await + } else { + indexer_db::stats::refresh_full(&pool).await + }; + match res { + Ok(()) => populated = true, + Err(e) => { + tracing::warn!(error = %e, "stats_daily_mv refresh failed"); + } + } + } + } + } + }) + }; + shutdown_signal().await; tracing::info!("indexer: shutdown signal received; cancelling workers"); cancel.cancel(); + let _ = stats_refresh_handle.await?; let _ = backfill_handle.await?; let _ = coinblast_handle.await?; if let Some(t) = tail_handle { diff --git a/crates/api/src/routes/leaderboards.rs b/crates/api/src/routes/leaderboards.rs index 8b21d36..640176d 100644 --- a/crates/api/src/routes/leaderboards.rs +++ b/crates/api/src/routes/leaderboards.rs @@ -101,8 +101,13 @@ async fn whale_transfers( } /// Router for `/accounts/active` + `/whale/transfers`. +/// +/// `/whale/tx` is an alias for `/whale/transfers` — the legacy TS indexer (and +/// the explorer frontend wired to it) names this path `/whale/tx`; serve both +/// so the frontend is indexer-agnostic. pub fn router() -> Router { Router::new() .route("/accounts/active", get(accounts_active)) .route("/whale/transfers", get(whale_transfers)) + .route("/whale/tx", get(whale_transfers)) } diff --git a/crates/api/src/routes/stats.rs b/crates/api/src/routes/stats.rs index 12d9161..f11bd76 100644 --- a/crates/api/src/routes/stats.rs +++ b/crates/api/src/routes/stats.rs @@ -1,5 +1,7 @@ -//! `/stats/daily` — chain-wide aggregates per day_bucket from the -//! `stats_daily_mv` materialised view (db migration 0002). +//! `/stats/daily` — chain-wide aggregates per day from the `stats_daily_mv` +//! materialised view (db migration 0002). The response is a bare array +//! `[{date, blocks, transactions}]`, matching the legacy TS indexer so the +//! explorer frontend can consume either indexer interchangeably. use crate::error::{ApiError, ApiResult}; use crate::routes::clamp_limit; @@ -17,54 +19,74 @@ struct ListQuery { #[derive(Debug, Serialize, Deserialize)] struct DailyRow { - /// Decimal-string `floor(timestamp / 86400)`. - day_bucket: String, - /// Decimal-string block count for the bucket. - block_count: String, - /// Decimal-string sum of tx_count. - tx_count: String, - /// Decimal-string sum of gas_used. - gas_used: String, - /// Decimal-string lowest height in the bucket. - first_height: String, - /// Decimal-string highest height in the bucket. - last_height: String, -} - -#[derive(Debug, Serialize, Deserialize)] -struct DailyResponse { - daily: Vec, + /// `YYYY-MM-DD` (UTC). + date: String, + /// Block count for the day. + blocks: i64, + /// Transaction count for the day. + transactions: i64, } async fn daily( State(state): State, Query(q): Query, -) -> ApiResult> { +) -> ApiResult>> { let limit = clamp_limit(q.limit.as_deref()); // Cache-aside: chain tier (60s TTL). MV refresh cadence is 5 min; // even short cache TTL collapses 60s of bursts into 1 PG read. let key = format!("stats:daily:{limit}"); - let response: DailyResponse = cached::get_or_load(&state, &key, CacheTier::Chain, || async { + let rows: Vec = cached::get_or_load(&state, &key, CacheTier::Chain, || async { let rows = stats::daily(&state.pool, limit).await?; - Ok::<_, ApiError>(DailyResponse { - daily: rows - .into_iter() + Ok::<_, ApiError>( + rows.into_iter() .map(|r| DailyRow { - day_bucket: r.day_bucket.to_string(), - block_count: r.block_count.to_string(), - tx_count: r.tx_count.to_string(), - gas_used: r.gas_used.to_string(), - first_height: r.first_height.to_string(), - last_height: r.last_height.to_string(), + date: r.date, + blocks: r.blocks, + transactions: r.transactions, }) .collect(), - }) + ) }) .await?; - Ok(Json(response)) + Ok(Json(rows)) } /// Router for `/stats/daily`. pub fn router() -> Router { Router::new().route("/stats/daily", get(daily)) } + +#[cfg(test)] +mod tests { + use super::DailyRow; + + #[test] + fn daily_row_is_flat_legacy_shape() { + // Each element must be a flat {date, blocks, transactions} object with + // numeric counts — the legacy TS indexer shape the explorer consumes. + // No `daily` wrapper, no `day_bucket`/`block_count` field names. + let row = DailyRow { + date: "2026-04-24".into(), + blocks: 102108, + transactions: 102110, + }; + let v = serde_json::to_value(&row).unwrap(); + assert_eq!(v["date"], "2026-04-24"); + assert_eq!(v["blocks"], 102108); + assert_eq!(v["transactions"], 102110); + assert!(v.get("day_bucket").is_none()); + assert!(v.get("block_count").is_none()); + } + + #[test] + fn daily_response_serialises_as_bare_array() { + let rows = vec![DailyRow { + date: "2026-04-24".into(), + blocks: 1, + transactions: 2, + }]; + let v = serde_json::to_value(&rows).unwrap(); + assert!(v.is_array(), "response must be a bare array, not an object"); + assert!(v.get("daily").is_none()); + } +} diff --git a/crates/db/src/stats.rs b/crates/db/src/stats.rs index 98993a2..4713d04 100644 --- a/crates/db/src/stats.rs +++ b/crates/db/src/stats.rs @@ -3,27 +3,27 @@ use crate::{DbResult, PgPool}; use sqlx::Row; -/// One row of `/stats/daily` — pre-aggregated per day_bucket. +/// One row of `/stats/daily`. Field names + types mirror the legacy TS +/// indexer's response (`date` ISO-8601 day, numeric `blocks`/`transactions`) +/// so the explorer frontend consumes either indexer interchangeably. The +/// calendar date is derived from the `day_bucket` (epoch-day) in SQL via +/// `to_timestamp(day_bucket * 86400)` at UTC. #[derive(Debug, Clone)] pub struct StatsDailyRow { - /// `floor(timestamp / 86400)` — chain-day bucket. - pub day_bucket: i64, + /// `YYYY-MM-DD` (UTC) for the bucket. + pub date: String, /// Blocks that landed in this bucket. - pub block_count: i64, + pub blocks: i64, /// Sum of `blocks.tx_count` over the bucket. - pub tx_count: i64, - /// Sum of `blocks.gas_used` over the bucket. - pub gas_used: i64, - /// First (lowest) block height in the bucket. - pub first_height: i64, - /// Last (highest) block height in the bucket. - pub last_height: i64, + pub transactions: i64, } /// Read the last `limit` daily buckets, newest-first. pub async fn daily(pool: &PgPool, limit: i64) -> DbResult> { let rows = sqlx::query( - "SELECT day_bucket, block_count, tx_count, gas_used, first_height, last_height \ + "SELECT to_char(to_timestamp(day_bucket * 86400) AT TIME ZONE 'UTC', 'YYYY-MM-DD') AS date, \ + block_count AS blocks, \ + tx_count AS transactions \ FROM stats_daily_mv ORDER BY day_bucket DESC LIMIT $1", ) .bind(limit) @@ -32,23 +32,29 @@ pub async fn daily(pool: &PgPool, limit: i64) -> DbResult> { rows.into_iter() .map(|r| { Ok(StatsDailyRow { - day_bucket: r.try_get("day_bucket")?, - block_count: r.try_get("block_count")?, - tx_count: r.try_get("tx_count")?, - gas_used: r.try_get("gas_used")?, - first_height: r.try_get("first_height")?, - last_height: r.try_get("last_height")?, + date: r.try_get("date")?, + blocks: r.try_get("blocks")?, + transactions: r.try_get("transactions")?, }) }) .collect::>() .map_err(Into::into) } -/// Trigger a CONCURRENTLY refresh of the MV. Called by the route on cache -/// miss for the most-recent bucket, OR by the operator on demand. +/// CONCURRENTLY refresh — does not lock out reads, but Postgres rejects it on +/// a never-populated MV. Use for the periodic refresh once populated. pub async fn refresh(pool: &PgPool) -> DbResult<()> { sqlx::query("REFRESH MATERIALIZED VIEW CONCURRENTLY stats_daily_mv") .execute(pool) .await?; Ok(()) } + +/// Plain (blocking) refresh — the only form that works on a never-populated +/// MV. Run once at startup before switching to `refresh` on the interval. +pub async fn refresh_full(pool: &PgPool) -> DbResult<()> { + sqlx::query("REFRESH MATERIALIZED VIEW stats_daily_mv") + .execute(pool) + .await?; + Ok(()) +} From 8863d6aea60cb85dc256bb5e552761879e527ae6 Mon Sep 17 00:00:00 2001 From: satyakwok <119509589+satyakwok@users.noreply.github.com> Date: Mon, 8 Jun 2026 05:40:29 +0200 Subject: [PATCH 2/2] test(smoke): assert new stats/daily array shape + /whale/tx alias --- scripts/smoke.sh | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/scripts/smoke.sh b/scripts/smoke.sh index a3a5016..ec2ebf5 100755 --- a/scripts/smoke.sh +++ b/scripts/smoke.sh @@ -183,6 +183,12 @@ v=$(curl -fsS "$API_BASE/whale/transfers" | jq -r '.transfers[0].hash') fail "/whale/transfers top != txaaaa (got '$v')" ok "/whale/transfers (sorted by value)" +# /whale/tx -> alias of /whale/transfers (legacy path name the frontend uses) +v=$(curl -fsS "$API_BASE/whale/tx" | jq -r '.transfers[0].hash') +[[ "$v" == "0xtxaaaa00000000000000000000000000000000000000000000000000000000aa" ]] || \ + fail "/whale/tx alias top != txaaaa (got '$v')" +ok "/whale/tx (alias of /whale/transfers)" + # /coinblast/tokens -> 1 curve v=$(curl -fsS "$API_BASE/coinblast/tokens" | jq -r '.tokens | length') [[ "$v" == "1" ]] || fail "/coinblast/tokens len != 1 (got $v)" @@ -202,14 +208,18 @@ v=$(curl -fsS "$API_BASE/coinblast/trades" | jq -r '.trades[0].type') [[ "$v" == "sell" ]] || fail "/coinblast/trades[0].type != sell (got '$v')" ok "/coinblast/trades (newest first)" -# /stats/daily -> 3 day rows (each fixture block is 86400s apart = distinct day) -v=$(curl -fsS "$API_BASE/stats/daily" | jq -r '.daily | length') +# /stats/daily -> bare array of 3 day rows (each fixture block 86400s apart = +# distinct day), newest first. Shape: [{date, blocks, transactions}]. +v=$(curl -fsS "$API_BASE/stats/daily" | jq -r 'length') [[ "$v" == "3" ]] || fail "/stats/daily len != 3 (got $v)" -# Highest bucket should be the newest (block 3 day). -v=$(curl -fsS "$API_BASE/stats/daily" | jq -r '.daily[0].day_bucket | tonumber') -prev=$(curl -fsS "$API_BASE/stats/daily" | jq -r '.daily[1].day_bucket | tonumber') -[[ "$v" -gt "$prev" ]] || fail "/stats/daily not ordered DESC (got $v <= $prev)" -ok "/stats/daily (3 day buckets, ordered DESC)" +# Newest day first — ISO dates sort lexically. +v=$(curl -fsS "$API_BASE/stats/daily" | jq -r '.[0].date') +prev=$(curl -fsS "$API_BASE/stats/daily" | jq -r '.[1].date') +[[ "$v" > "$prev" ]] || fail "/stats/daily not ordered DESC by date (got $v <= $prev)" +# Counts are numbers, not decimal strings. +v=$(curl -fsS "$API_BASE/stats/daily" | jq -r '.[0].blocks | type') +[[ "$v" == "number" ]] || fail "/stats/daily blocks not numeric (got $v)" +ok "/stats/daily (bare array, 3 day buckets, ordered DESC)" # /api?module=account&action=txlist (etherscan compat) v=$(curl -fsS "$API_BASE/api?module=account&action=txlist&address=0xdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef" | jq -r '.status')