From 7c9337843ed83e84504c8636256b2b48c0e797e3 Mon Sep 17 00:00:00 2001 From: Mischa0x Date: Mon, 20 Apr 2026 10:32:26 +0000 Subject: [PATCH] fix(cache): restore per-chain isolation + validation to vault data pipeline The vaultsData.json cache has been frozen at 2025-11-24 12:20 UTC because a latent TypeError in getSafeVaultApy crashes the whole pipeline via a brittle top-level Promise.all, which in turn skips the commit step. Changes: - lib/vault/prepareVaultData.ts: guard filteredLogs[0].blockNumber access in getSafeVaultApy when no oracle PriceUpdated events match the vault in the last 10k-block window. The guard at line 368 only checked the unfiltered logs; this adds the missing check on the filtered list. - lib/scripts/cacheVaultsData.ts: wrap each chain's fetch in try/catch inside Promise.allSettled so one chain cannot kill the others. Validate each chain's vault addresses/assets before inclusion; drop chains with malformed data rather than propagating corruption. Always emit every supported chain in the output (empty array for failures) so client vaultsData[chain.id] access remains shape-compatible. Add generatedAt, chains, failedChains, stakingTVLFromFallback manifest fields for observability. Log DefiLlama staking TVL fallback (previously silent). - .github/workflows/update-cache.yml: add Validate cache output step between run and commit (freshness <30min, mainnet required, >=6 chains, address hex format, vaultTVL numeric). Skip commit when no diff. Push to current ref instead of hardcoded main to make workflow_dispatch on feature branches safe. Add optional failure alert via CACHE_ALERT_WEBHOOK secret. Does not modify next.config.js env block. Secrets surface is unchanged from current main. No new process.env.* reads added. --- .github/workflows/update-cache.yml | 76 +++++++++- lib/scripts/cacheVaultsData.ts | 221 ++++++++++++++++++++--------- lib/vault/prepareVaultData.ts | 2 + 3 files changed, 227 insertions(+), 72 deletions(-) diff --git a/.github/workflows/update-cache.yml b/.github/workflows/update-cache.yml index 9fa7f53d..8835e27c 100644 --- a/.github/workflows/update-cache.yml +++ b/.github/workflows/update-cache.yml @@ -32,10 +32,82 @@ jobs: - name: Run cache script run: yarn run cache - - name: Commit & push changes + - name: Validate cache output + run: | + FILE=cache/vaultsData.json + if [ ! -f "$FILE" ]; then + echo "::error::$FILE not written by cache script" + exit 1 + fi + # Must be valid JSON + jq empty "$FILE" || { echo "::error::$FILE is not valid JSON"; exit 1; } + # Must have generatedAt within last 30 minutes + GEN=$(jq -r '.generatedAt // empty' "$FILE") + if [ -z "$GEN" ]; then + echo "::error::generatedAt missing from $FILE" + exit 1 + fi + GEN_EPOCH=$(date -d "$GEN" +%s) + NOW_EPOCH=$(date +%s) + AGE=$(( NOW_EPOCH - GEN_EPOCH )) + if [ "$AGE" -gt 1800 ] || [ "$AGE" -lt -60 ]; then + echo "::error::generatedAt=$GEN is ${AGE}s from now (outside -60..1800 window)" + exit 1 + fi + # Must have mainnet (chain 1) data + CHAINS=$(jq -r '.chains[]?' "$FILE" | tr '\n' ' ') + echo "chains present: $CHAINS" + if ! echo "$CHAINS" | grep -qE '(^| )1( |$)'; then + echo "::error::mainnet (chain 1) missing from output" + exit 1 + fi + # Require at least 6 of the 9 supported chains + CHAIN_COUNT=$(jq -r '.chains | length' "$FILE") + if [ "$CHAIN_COUNT" -lt 6 ]; then + echo "::error::only $CHAIN_COUNT chains succeeded (need >=6)" + exit 1 + fi + # vaultTVL must be a non-negative number + TVL=$(jq -r '.vaultTVL' "$FILE") + case "$TVL" in + ''|*[!0-9.]*) echo "::error::vaultTVL=$TVL is not numeric"; exit 1 ;; + esac + # Every vault.address must match 0x + 40 hex + BAD=$(jq -r '[.vaultsData[][] | .address] | map(select(test("^0x[a-fA-F0-9]{40}$") | not)) | length' "$FILE") + if [ "$BAD" -gt 0 ]; then + echo "::error::$BAD vault addresses failed hex validation" + exit 1 + fi + # Every vault.asset must match 0x + 40 hex + BAD=$(jq -r '[.vaultsData[][] | .asset] | map(select(test("^0x[a-fA-F0-9]{40}$") | not)) | length' "$FILE") + if [ "$BAD" -gt 0 ]; then + echo "::error::$BAD vault assets failed hex validation" + exit 1 + fi + echo "::notice::cache validated: $CHAIN_COUNT chains, vaultTVL=$TVL, generated $GEN" + + - name: Commit & push changes run: | git config user.name "github-actions[bot]" git config user.email "41898282+github-actions[bot]@users.noreply.github.com" git add -A + if git diff --cached --quiet; then + echo "No changes to commit" + exit 0 + fi git commit -m "Update vaults data cache" - git push origin main + git push origin HEAD:${{ github.ref_name }} + + - name: Alert on failure + if: failure() + env: + WEBHOOK: ${{ secrets.CACHE_ALERT_WEBHOOK }} + run: | + if [ -z "$WEBHOOK" ]; then + echo "CACHE_ALERT_WEBHOOK secret not set — skipping alert" + exit 0 + fi + RUN_URL="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" + curl -sS -X POST -H 'Content-Type: application/json' \ + -d "{\"text\":\"vaultcraft cache update failed: $RUN_URL\"}" \ + "$WEBHOOK" || true diff --git a/lib/scripts/cacheVaultsData.ts b/lib/scripts/cacheVaultsData.ts index 05ad9253..f1a9251a 100644 --- a/lib/scripts/cacheVaultsData.ts +++ b/lib/scripts/cacheVaultsData.ts @@ -24,91 +24,172 @@ function deepStringify(obj: any): any { ); } +function isValidAddress(s: unknown): boolean { + return typeof s === "string" && /^0x[a-fA-F0-9]{40}$/.test(s); +} + +function validateChainVaults(chainId: number, vaults: VaultData[]): string[] { + const errors: string[] = []; + vaults.forEach((v, i) => { + if (!isValidAddress(v?.address)) + errors.push(`chain ${chainId} vault #${i}: invalid address ${v?.address}`); + if (!isValidAddress(v?.asset)) + errors.push(`chain ${chainId} vault #${i} (${v?.address}): invalid asset ${v?.asset}`); + if (v?.totalAssets === undefined || v?.totalAssets === null) + errors.push(`chain ${chainId} vault ${v?.address}: missing totalAssets`); + if (v?.totalSupply === undefined || v?.totalSupply === null) + errors.push(`chain ${chainId} vault ${v?.address}: missing totalSupply`); + }); + return errors; +} + export async function cacheVaultData() { - // get vaultsData and tokens const newVaultsData: { [key: number]: VaultData[] } = {}; const newTokensData: { [key: number]: TokenByAddress } = {}; - const newStrategies: StrategiesByChain = {}; - await Promise.all( - SUPPORTED_NETWORKS.map(async (chain) => { - console.log(`Fetching Data for chain ${chain.id} (${new Date()})`); - const { vaultsData, tokens, strategies } = - await getTokenAndVaultsDataByChain({ - chain, - account: zeroAddress, - }); - - console.log( - `Completed fetching Data for chain ${chain.id} (${new Date()})` - ); + const successfulChains: number[] = []; + const failedChains: { id: number; error: string }[] = []; - newVaultsData[chain.id] = vaultsData; - newTokensData[chain.id] = tokens; - newStrategies[chain.id] = strategies; - - const vaults = Object.fromEntries( - Object.entries(newVaultsData).map(([chainId, vaults]) => [ - chainId, - vaults.map((v) => deepStringify(v)), - ]) - ); + const results = await Promise.allSettled( + SUPPORTED_NETWORKS.map(async (chain) => { + console.log(`Fetching Data for chain ${chain.id} (${new Date().toISOString()})`); + try { + const { vaultsData, tokens, strategies } = + await getTokenAndVaultsDataByChain({ + chain, + account: zeroAddress, + }); + + const validationErrors = validateChainVaults(chain.id, vaultsData); + if (validationErrors.length > 0) { + console.error( + `Chain ${chain.id} dropped due to validation errors:`, + validationErrors + ); + failedChains.push({ + id: chain.id, + error: `validation: ${validationErrors.length} errors`, + }); + return; + } + + newVaultsData[chain.id] = vaultsData; + newTokensData[chain.id] = tokens; + newStrategies[chain.id] = strategies; + successfulChains.push(chain.id); + console.log( + `Completed fetching Data for chain ${chain.id} (${new Date().toISOString()}) — ${vaultsData.length} vaults` + ); + } catch (e: any) { + const msg = e?.message || String(e); + console.error(`Chain ${chain.id} failed:`, msg); + failedChains.push({ id: chain.id, error: msg }); + } + }) + ); - const tokensData = Object.fromEntries( - Object.entries(newTokensData).map(([chainId, tokens]) => [ - chainId, - deepStringify(tokens), - ]) + // Unexpected throws inside .allSettled should still be logged (shouldn't happen since we try/catch inside, but belt-and-suspenders). + results.forEach((r, i) => { + if (r.status === "rejected") { + console.error( + `Unexpected top-level rejection for chain ${SUPPORTED_NETWORKS[i].id}:`, + r.reason ); + } + }); - const strategiesData = Object.fromEntries( - Object.entries(newStrategies).map(([chainId, strats]) => [ - chainId, - deepStringify(strats), - ]) - ); + if (successfulChains.length === 0) { + console.error("All chains failed — aborting without write."); + process.exit(1); + } - const vaultTVL = SUPPORTED_NETWORKS.map( - (chain) => newVaultsData[chain.id] - ) - .flat() - .reduce((a, b) => a + (Number(b?.tvl) || 0), 0); - const lockVaultTVL = 520000; // @dev hardcoded since we removed lock vaults - let stakingTVL = 0; - try { - stakingTVL = await axios - .get( - `https://pro-api.llama.fi/${process.env.DEFILLAMA_API_KEY}/api/protocol/vaultcraft` - ) - .then((res) => res.data.currentChainTvls["staking"]); - } catch (e) { - stakingTVL = 2590000; - } + // Always emit every supported chain, even when empty, so the client code's + // `vaultsData[chain.id]` access never returns undefined (shape-compat with old cache). + SUPPORTED_NETWORKS.forEach((chain) => { + if (!(chain.id in newVaultsData)) newVaultsData[chain.id] = []; + if (!(chain.id in newTokensData)) newTokensData[chain.id] = {}; + if (!(chain.id in newStrategies)) newStrategies[chain.id] = {}; + }); + + const vaults = Object.fromEntries( + Object.entries(newVaultsData).map(([chainId, vaultsArr]) => [ + chainId, + vaultsArr.map((v) => deepStringify(v)), + ]) + ); - const vaultData = { - vaultsData: vaults, - tokens: tokensData, - strategies: strategiesData, - vaultTVL, - lockVaultTVL, - stakingTVL, - }; + const tokensData = Object.fromEntries( + Object.entries(newTokensData).map(([chainId, tokens]) => [ + chainId, + deepStringify(tokens), + ]) + ); - const replacer = (_k: string, v: any) => - typeof v === "bigint" ? v.toString() : v; + const strategiesData = Object.fromEntries( + Object.entries(newStrategies).map(([chainId, strats]) => [ + chainId, + deepStringify(strats), + ]) + ); - // write to file - const OUTPUT_DIR_V2 = "cache"; + const vaultTVL = successfulChains + .map((id) => newVaultsData[id]) + .flat() + .reduce((a, b) => a + (Number(b?.tvl) || 0), 0); + + // TODO: lockVaultTVL/stakingTVL fallbacks below are hardcoded from a stale point in time. + // These should be replaced with on-chain reads or a real API — tracked separately. + const lockVaultTVL = 520000; // @dev hardcoded since we removed lock vaults + let stakingTVL = 0; + let stakingTVLFromFallback = false; + try { + stakingTVL = await axios + .get( + `https://pro-api.llama.fi/${process.env.DEFILLAMA_API_KEY}/api/protocol/vaultcraft` + ) + .then((res) => res.data.currentChainTvls["staking"]); + if (stakingTVL === undefined || stakingTVL === null || Number.isNaN(Number(stakingTVL))) { + throw new Error("DefiLlama returned non-numeric staking TVL"); + } + } catch (e: any) { + console.error( + `DefiLlama staking TVL fetch failed, using hardcoded fallback 2590000:`, + e?.message || String(e) + ); + stakingTVL = 2590000; + stakingTVLFromFallback = true; + } - if (!fs.existsSync(OUTPUT_DIR_V2)) - fs.mkdirSync(OUTPUT_DIR_V2, { recursive: true }); + const vaultData = { + generatedAt: new Date().toISOString(), + chains: successfulChains, + failedChains, + stakingTVLFromFallback, + vaultsData: vaults, + tokens: tokensData, + strategies: strategiesData, + vaultTVL, + lockVaultTVL, + stakingTVL, + }; + + const replacer = (_k: string, v: any) => + typeof v === "bigint" ? v.toString() : v; + + const OUTPUT_DIR_V2 = "cache"; + if (!fs.existsSync(OUTPUT_DIR_V2)) + fs.mkdirSync(OUTPUT_DIR_V2, { recursive: true }); + + fs.writeFileSync( + path.join(OUTPUT_DIR_V2, "vaultsData.json"), + JSON.stringify(vaultData, replacer, 2) + ); - fs.writeFileSync( - path.join(OUTPUT_DIR_V2, "vaultsData.json"), - JSON.stringify(vaultData, replacer, 2) - ); - }) + console.log( + `Cache written: ${successfulChains.length}/${SUPPORTED_NETWORKS.length} chains succeeded. Failed: ${failedChains.map((f) => f.id).join(", ") || "none"}.` ); + // Partial success is acceptable — the file is written with the chains that worked. + // The CI validation step decides whether to commit (it checks minimum chain count). } async function main() { diff --git a/lib/vault/prepareVaultData.ts b/lib/vault/prepareVaultData.ts index f8fa0b8c..a14ee5b0 100644 --- a/lib/vault/prepareVaultData.ts +++ b/lib/vault/prepareVaultData.ts @@ -371,6 +371,8 @@ async function getSafeVaultApy(vault: VaultData): Promise { (log) => log.args.base === vault.address && log.args.quote === vault.asset ); + if (filteredLogs.length === 0) return []; + let firstTimestamp = Number( ( await client.getBlock({