diff --git a/cluster/expected/observability/expected.json b/cluster/expected/observability/expected.json index dc56f0b7e6..d845a35fa3 100644 --- a/cluster/expected/observability/expected.json +++ b/cluster/expected/observability/expected.json @@ -137,6 +137,7 @@ "inputs": { "apiVersion": "v1", "data": { + "app-rewards.json": "{\n \"annotations\": {\n \"list\": [\n {\n \"builtIn\": 1,\n \"datasource\": {\n \"type\": \"grafana\",\n \"uid\": \"-- Grafana --\"\n },\n \"enable\": true,\n \"hide\": true,\n \"iconColor\": \"rgba(0, 211, 255, 1)\",\n \"name\": \"Annotations & Alerts\",\n \"type\": \"dashboard\"\n }\n ]\n },\n \"description\": \"Traffic based app rewards (CIP-104)\",\n \"editable\": true,\n \"fiscalYearStartMonth\": 0,\n \"graphTooltip\": 0,\n \"links\": [],\n \"panels\": [\n {\n \"collapsed\": false,\n \"gridPos\": {\n \"h\": 1,\n \"w\": 24,\n \"x\": 0,\n \"y\": 0\n },\n \"id\": 7,\n \"panels\": [],\n \"title\": \"Store lag\",\n \"type\": \"row\"\n },\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"description\": \"The time difference between the last ingested record time and the current wallclock time.\\n\\nNote that the last ingested record time metric only updates when the store ingests a new transaction so if there is no activity, the last ingested record time will not advance.\",\n \"fieldConfig\": {\n \"defaults\": {\n \"color\": {\n \"mode\": \"palette-classic\"\n },\n \"custom\": {\n \"axisBorderShow\": false,\n \"axisCenteredZero\": false,\n \"axisColorMode\": \"text\",\n \"axisLabel\": \"\",\n \"axisPlacement\": \"auto\",\n \"barAlignment\": 0,\n \"barWidthFactor\": 0.6,\n \"drawStyle\": \"line\",\n \"fillOpacity\": 0,\n \"gradientMode\": \"none\",\n \"hideFrom\": {\n \"legend\": false,\n \"tooltip\": false,\n \"viz\": false\n },\n \"insertNulls\": false,\n \"lineInterpolation\": \"linear\",\n \"lineWidth\": 1,\n \"pointSize\": 5,\n \"scaleDistribution\": {\n \"type\": \"linear\"\n },\n \"showPoints\": \"auto\",\n \"showValues\": false,\n \"spanNulls\": false,\n \"stacking\": {\n \"group\": \"A\",\n \"mode\": \"none\"\n },\n \"thresholdsStyle\": {\n \"mode\": \"off\"\n }\n },\n \"mappings\": [],\n \"thresholds\": {\n \"mode\": \"absolute\",\n \"steps\": [\n {\n \"color\": \"green\",\n \"value\": 0\n },\n {\n \"color\": \"red\",\n \"value\": 80\n }\n ]\n },\n \"unit\": \"s\"\n },\n \"overrides\": []\n },\n \"gridPos\": {\n \"h\": 8,\n \"w\": 12,\n \"x\": 0,\n \"y\": 1\n },\n \"id\": 11,\n \"options\": {\n \"legend\": {\n \"calcs\": [],\n \"displayMode\": \"list\",\n \"placement\": \"bottom\",\n \"showLegend\": true\n },\n \"tooltip\": {\n \"hideZeros\": false,\n \"mode\": \"multi\",\n \"sort\": \"desc\"\n }\n },\n \"pluginVersion\": \"12.4.0\",\n \"targets\": [\n {\n \"editorMode\": \"code\",\n \"expr\": \"(timestamp(splice_store_last_ingested_record_time_ms{namespace=~\\\"$namespace\\\",store_name=~\\\"DbScanRewardsReferenceStore\\\"}) - (splice_store_last_ingested_record_time_ms{namespace=~\\\"$namespace\\\",store_name=~\\\"DbScanRewardsReferenceStore\\\"} > 0) / 1e3)\",\n \"legendFormat\": \"{{namespace}}\",\n \"range\": true,\n \"refId\": \"A\"\n }\n ],\n \"title\": \"DbScanRewardsReferenceStore lag\",\n \"type\": \"timeseries\"\n },\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"description\": \"The time difference between the last ingested verdict record time and the current wall clock time.\",\n \"fieldConfig\": {\n \"defaults\": {\n \"color\": {\n \"mode\": \"palette-classic\"\n },\n \"custom\": {\n \"axisBorderShow\": false,\n \"axisCenteredZero\": false,\n \"axisColorMode\": \"text\",\n \"axisLabel\": \"\",\n \"axisPlacement\": \"auto\",\n \"barAlignment\": 0,\n \"barWidthFactor\": 0.6,\n \"drawStyle\": \"line\",\n \"fillOpacity\": 0,\n \"gradientMode\": \"none\",\n \"hideFrom\": {\n \"legend\": false,\n \"tooltip\": false,\n \"viz\": false\n },\n \"insertNulls\": false,\n \"lineInterpolation\": \"linear\",\n \"lineWidth\": 1,\n \"pointSize\": 5,\n \"scaleDistribution\": {\n \"type\": \"linear\"\n },\n \"showPoints\": \"auto\",\n \"showValues\": false,\n \"spanNulls\": false,\n \"stacking\": {\n \"group\": \"A\",\n \"mode\": \"none\"\n },\n \"thresholdsStyle\": {\n \"mode\": \"off\"\n }\n },\n \"mappings\": [],\n \"thresholds\": {\n \"mode\": \"absolute\",\n \"steps\": [\n {\n \"color\": \"green\",\n \"value\": 0\n },\n {\n \"color\": \"red\",\n \"value\": 80\n }\n ]\n },\n \"unit\": \"s\"\n },\n \"overrides\": []\n },\n \"gridPos\": {\n \"h\": 8,\n \"w\": 12,\n \"x\": 12,\n \"y\": 1\n },\n \"id\": 10,\n \"options\": {\n \"legend\": {\n \"calcs\": [],\n \"displayMode\": \"list\",\n \"placement\": \"bottom\",\n \"showLegend\": true\n },\n \"tooltip\": {\n \"hideZeros\": false,\n \"mode\": \"multi\",\n \"sort\": \"desc\"\n }\n },\n \"pluginVersion\": \"12.4.0\",\n \"targets\": [\n {\n \"disableTextWrap\": false,\n \"editorMode\": \"code\",\n \"expr\": \"(timestamp(splice_scan_verdict_ingestion_last_record_time_us{namespace=~\\\"$namespace\\\"}) - (splice_scan_verdict_ingestion_last_record_time_us{namespace=~\\\"$namespace\\\"} > 0) / 1e6)\",\n \"fullMetaSearch\": false,\n \"includeNullMetadata\": true,\n \"legendFormat\": \"{{namespace}}\",\n \"range\": true,\n \"refId\": \"A\",\n \"useBackend\": false\n }\n ],\n \"title\": \"Verdict ingestion lag\",\n \"type\": \"timeseries\"\n },\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"description\": \"The time difference between the last ingested record time and the current wallclock time.\\n\\nNote that the last ingested record time metric only updates when the store ingests a new transaction so if there is no activity, the last ingested record time will not advance. For a party performing reward collection, e.g., the validator operator party you expect at least one transaction every round so the lag should not go above 20min.\",\n \"fieldConfig\": {\n \"defaults\": {\n \"color\": {\n \"mode\": \"palette-classic\"\n },\n \"custom\": {\n \"axisBorderShow\": false,\n \"axisCenteredZero\": false,\n \"axisColorMode\": \"text\",\n \"axisLabel\": \"\",\n \"axisPlacement\": \"auto\",\n \"barAlignment\": 0,\n \"barWidthFactor\": 0.6,\n \"drawStyle\": \"line\",\n \"fillOpacity\": 0,\n \"gradientMode\": \"none\",\n \"hideFrom\": {\n \"legend\": false,\n \"tooltip\": false,\n \"viz\": false\n },\n \"insertNulls\": false,\n \"lineInterpolation\": \"linear\",\n \"lineWidth\": 1,\n \"pointSize\": 5,\n \"scaleDistribution\": {\n \"type\": \"linear\"\n },\n \"showPoints\": \"auto\",\n \"showValues\": false,\n \"spanNulls\": false,\n \"stacking\": {\n \"group\": \"A\",\n \"mode\": \"none\"\n },\n \"thresholdsStyle\": {\n \"mode\": \"off\"\n }\n },\n \"mappings\": [],\n \"thresholds\": {\n \"mode\": \"absolute\",\n \"steps\": [\n {\n \"color\": \"green\",\n \"value\": 0\n },\n {\n \"color\": \"red\",\n \"value\": 80\n }\n ]\n },\n \"unit\": \"s\"\n },\n \"overrides\": []\n },\n \"gridPos\": {\n \"h\": 8,\n \"w\": 12,\n \"x\": 0,\n \"y\": 9\n },\n \"id\": 12,\n \"options\": {\n \"legend\": {\n \"calcs\": [],\n \"displayMode\": \"list\",\n \"placement\": \"bottom\",\n \"showLegend\": true\n },\n \"tooltip\": {\n \"hideZeros\": false,\n \"mode\": \"multi\",\n \"sort\": \"desc\"\n }\n },\n \"pluginVersion\": \"12.4.0\",\n \"targets\": [\n {\n \"editorMode\": \"code\",\n \"expr\": \"(timestamp(splice_store_last_ingested_record_time_ms{namespace=~\\\"$namespace\\\",store_name=~\\\"DbSvDsoStore\\\"}) - (splice_store_last_ingested_record_time_ms{namespace=~\\\"$namespace\\\",store_name=~\\\"DbSvDsoStore\\\"} > 0) / 1e3)\",\n \"legendFormat\": \"{{namespace}}\",\n \"range\": true,\n \"refId\": \"A\"\n }\n ],\n \"title\": \"DbSvDsoStore lag\",\n \"type\": \"timeseries\"\n },\n {\n \"collapsed\": false,\n \"gridPos\": {\n \"h\": 1,\n \"w\": 24,\n \"x\": 0,\n \"y\": 17\n },\n \"id\": 13,\n \"panels\": [],\n \"title\": \"RewardComputationTrigger (calculate reward merkle tree)\",\n \"type\": \"row\"\n },\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"fieldConfig\": {\n \"defaults\": {\n \"color\": {\n \"mode\": \"palette-classic\"\n },\n \"custom\": {\n \"axisBorderShow\": false,\n \"axisCenteredZero\": false,\n \"axisColorMode\": \"text\",\n \"axisLabel\": \"\",\n \"axisPlacement\": \"auto\",\n \"barAlignment\": 0,\n \"barWidthFactor\": 0.6,\n \"drawStyle\": \"line\",\n \"fillOpacity\": 0,\n \"gradientMode\": \"none\",\n \"hideFrom\": {\n \"legend\": false,\n \"tooltip\": false,\n \"viz\": false\n },\n \"insertNulls\": false,\n \"lineInterpolation\": \"linear\",\n \"lineWidth\": 1,\n \"pointSize\": 5,\n \"scaleDistribution\": {\n \"type\": \"linear\"\n },\n \"showPoints\": \"auto\",\n \"showValues\": false,\n \"spanNulls\": false,\n \"stacking\": {\n \"group\": \"A\",\n \"mode\": \"none\"\n },\n \"thresholdsStyle\": {\n \"mode\": \"off\"\n }\n },\n \"mappings\": [],\n \"thresholds\": {\n \"mode\": \"absolute\",\n \"steps\": [\n {\n \"color\": \"green\",\n \"value\": 0\n },\n {\n \"color\": \"red\",\n \"value\": 80\n }\n ]\n }\n },\n \"overrides\": []\n },\n \"gridPos\": {\n \"h\": 8,\n \"w\": 12,\n \"x\": 0,\n \"y\": 18\n },\n \"id\": 14,\n \"options\": {\n \"legend\": {\n \"calcs\": [],\n \"displayMode\": \"list\",\n \"placement\": \"bottom\",\n \"showLegend\": true\n },\n \"tooltip\": {\n \"hideZeros\": false,\n \"mode\": \"single\",\n \"sort\": \"none\"\n }\n },\n \"pluginVersion\": \"12.4.0\",\n \"targets\": [\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"editorMode\": \"code\",\n \"expr\": \"rate(splice_trigger_completed_total{trigger_name=~\\\"RewardComputationTrigger\\\",outcome=~\\\"success|failure\\\", namespace=~\\\"$namespace\\\"}[$__rate_interval])\",\n \"legendFormat\": \"{{namespace}} {{outcome}}\",\n \"range\": true,\n \"refId\": \"A\"\n }\n ],\n \"title\": \"Trigger runs\",\n \"type\": \"timeseries\"\n },\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"description\": \"Number of activity records in the latest computed round\",\n \"fieldConfig\": {\n \"defaults\": {\n \"color\": {\n \"mode\": \"thresholds\"\n },\n \"mappings\": [],\n \"thresholds\": {\n \"mode\": \"absolute\",\n \"steps\": [\n {\n \"color\": \"green\",\n \"value\": 0\n },\n {\n \"color\": \"red\",\n \"value\": 10000\n }\n ]\n }\n },\n \"overrides\": []\n },\n \"gridPos\": {\n \"h\": 8,\n \"w\": 4,\n \"x\": 12,\n \"y\": 18\n },\n \"id\": 16,\n \"options\": {\n \"colorMode\": \"value\",\n \"graphMode\": \"area\",\n \"justifyMode\": \"auto\",\n \"orientation\": \"auto\",\n \"percentChangeColorMode\": \"standard\",\n \"reduceOptions\": {\n \"calcs\": [\n \"lastNotNull\"\n ],\n \"fields\": \"\",\n \"values\": false\n },\n \"showPercentChange\": false,\n \"textMode\": \"auto\",\n \"wideLayout\": true\n },\n \"pluginVersion\": \"12.4.0\",\n \"targets\": [\n {\n \"editorMode\": \"code\",\n \"expr\": \"quantile(0.5, splice_scan_reward_computation_activity_records_count{namespace=~\\\"$namespace\\\"})\",\n \"legendFormat\": \"{{namespace}}\",\n \"range\": true,\n \"refId\": \"A\"\n }\n ],\n \"title\": \"Activity records\",\n \"type\": \"stat\"\n },\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"description\": \"Number of parties with activity in the latest computed round\",\n \"fieldConfig\": {\n \"defaults\": {\n \"color\": {\n \"mode\": \"thresholds\"\n },\n \"mappings\": [],\n \"thresholds\": {\n \"mode\": \"absolute\",\n \"steps\": [\n {\n \"color\": \"green\",\n \"value\": 0\n },\n {\n \"color\": \"red\",\n \"value\": 1000\n }\n ]\n }\n },\n \"overrides\": []\n },\n \"gridPos\": {\n \"h\": 8,\n \"w\": 4,\n \"x\": 16,\n \"y\": 18\n },\n \"id\": 18,\n \"options\": {\n \"colorMode\": \"value\",\n \"graphMode\": \"area\",\n \"justifyMode\": \"auto\",\n \"orientation\": \"auto\",\n \"percentChangeColorMode\": \"standard\",\n \"reduceOptions\": {\n \"calcs\": [\n \"lastNotNull\"\n ],\n \"fields\": \"\",\n \"values\": false\n },\n \"showPercentChange\": false,\n \"textMode\": \"auto\",\n \"wideLayout\": true\n },\n \"pluginVersion\": \"12.4.0\",\n \"targets\": [\n {\n \"editorMode\": \"code\",\n \"expr\": \"quantile(0.5, splice_scan_reward_computation_active_parties_count{namespace=~\\\"$namespace\\\"})\",\n \"legendFormat\": \"__auto\",\n \"range\": true,\n \"refId\": \"A\"\n }\n ],\n \"title\": \"Active parties\",\n \"type\": \"stat\"\n },\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"description\": \"Number of parties with rewards in the latest computed round\",\n \"fieldConfig\": {\n \"defaults\": {\n \"color\": {\n \"mode\": \"thresholds\"\n },\n \"mappings\": [],\n \"thresholds\": {\n \"mode\": \"absolute\",\n \"steps\": [\n {\n \"color\": \"green\",\n \"value\": 0\n },\n {\n \"color\": \"red\",\n \"value\": 1000\n }\n ]\n }\n },\n \"overrides\": []\n },\n \"gridPos\": {\n \"h\": 8,\n \"w\": 4,\n \"x\": 20,\n \"y\": 18\n },\n \"id\": 17,\n \"options\": {\n \"colorMode\": \"value\",\n \"graphMode\": \"area\",\n \"justifyMode\": \"auto\",\n \"orientation\": \"auto\",\n \"percentChangeColorMode\": \"standard\",\n \"reduceOptions\": {\n \"calcs\": [\n \"lastNotNull\"\n ],\n \"fields\": \"\",\n \"values\": false\n },\n \"showPercentChange\": false,\n \"textMode\": \"auto\",\n \"wideLayout\": true\n },\n \"pluginVersion\": \"12.4.0\",\n \"targets\": [\n {\n \"editorMode\": \"code\",\n \"expr\": \"quantile(0.5, splice_scan_reward_computation_rewarded_parties_count{namespace=~\\\"$namespace\\\"})\",\n \"legendFormat\": \"__auto\",\n \"range\": true,\n \"refId\": \"A\"\n }\n ],\n \"title\": \"Rewarded parties\",\n \"type\": \"stat\"\n },\n {\n \"collapsed\": false,\n \"gridPos\": {\n \"h\": 1,\n \"w\": 24,\n \"x\": 0,\n \"y\": 26\n },\n \"id\": 3,\n \"panels\": [],\n \"title\": \"CalculateRewardsTrigger (voting on root hash)\",\n \"type\": \"row\"\n },\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"fieldConfig\": {\n \"defaults\": {\n \"color\": {\n \"mode\": \"palette-classic\"\n },\n \"custom\": {\n \"axisBorderShow\": false,\n \"axisCenteredZero\": false,\n \"axisColorMode\": \"text\",\n \"axisLabel\": \"\",\n \"axisPlacement\": \"auto\",\n \"barAlignment\": 0,\n \"barWidthFactor\": 0.6,\n \"drawStyle\": \"line\",\n \"fillOpacity\": 0,\n \"gradientMode\": \"none\",\n \"hideFrom\": {\n \"legend\": false,\n \"tooltip\": false,\n \"viz\": false\n },\n \"insertNulls\": false,\n \"lineInterpolation\": \"linear\",\n \"lineWidth\": 1,\n \"pointSize\": 5,\n \"scaleDistribution\": {\n \"type\": \"linear\"\n },\n \"showPoints\": \"auto\",\n \"showValues\": false,\n \"spanNulls\": false,\n \"stacking\": {\n \"group\": \"A\",\n \"mode\": \"none\"\n },\n \"thresholdsStyle\": {\n \"mode\": \"off\"\n }\n },\n \"mappings\": [],\n \"thresholds\": {\n \"mode\": \"absolute\",\n \"steps\": [\n {\n \"color\": \"green\",\n \"value\": 0\n },\n {\n \"color\": \"red\",\n \"value\": 80\n }\n ]\n }\n },\n \"overrides\": []\n },\n \"gridPos\": {\n \"h\": 8,\n \"w\": 12,\n \"x\": 0,\n \"y\": 27\n },\n \"id\": 1,\n \"options\": {\n \"legend\": {\n \"calcs\": [],\n \"displayMode\": \"list\",\n \"placement\": \"bottom\",\n \"showLegend\": true\n },\n \"tooltip\": {\n \"hideZeros\": false,\n \"mode\": \"single\",\n \"sort\": \"none\"\n }\n },\n \"pluginVersion\": \"12.4.0\",\n \"targets\": [\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"editorMode\": \"code\",\n \"expr\": \"rate(splice_trigger_completed_total{trigger_name=~\\\"ProcessRewards(DryRun)?Trigger\\\",outcome=~\\\"success|failure\\\", namespace=~\\\"$namespace\\\"}[$__rate_interval])\",\n \"legendFormat\": \"{{namespace}} {{outcome}}\",\n \"range\": true,\n \"refId\": \"A\"\n }\n ],\n \"title\": \"Trigger runs\",\n \"type\": \"timeseries\"\n },\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"description\": \"the time it took between the closing of a round, and this SV's confirmation for the CalculateRewardsV2 contract's processing\",\n \"fieldConfig\": {\n \"defaults\": {\n \"color\": {\n \"mode\": \"palette-classic\"\n },\n \"custom\": {\n \"axisBorderShow\": false,\n \"axisCenteredZero\": false,\n \"axisColorMode\": \"text\",\n \"axisLabel\": \"\",\n \"axisPlacement\": \"auto\",\n \"barAlignment\": 0,\n \"barWidthFactor\": 0.6,\n \"drawStyle\": \"line\",\n \"fillOpacity\": 0,\n \"gradientMode\": \"none\",\n \"hideFrom\": {\n \"legend\": false,\n \"tooltip\": false,\n \"viz\": false\n },\n \"insertNulls\": false,\n \"lineInterpolation\": \"linear\",\n \"lineWidth\": 1,\n \"pointSize\": 5,\n \"scaleDistribution\": {\n \"type\": \"linear\"\n },\n \"showPoints\": \"auto\",\n \"showValues\": false,\n \"spanNulls\": false,\n \"stacking\": {\n \"group\": \"A\",\n \"mode\": \"none\"\n },\n \"thresholdsStyle\": {\n \"mode\": \"off\"\n }\n },\n \"mappings\": [],\n \"thresholds\": {\n \"mode\": \"absolute\",\n \"steps\": [\n {\n \"color\": \"green\",\n \"value\": 0\n },\n {\n \"color\": \"red\",\n \"value\": 80\n }\n ]\n },\n \"unit\": \"s\"\n },\n \"overrides\": []\n },\n \"gridPos\": {\n \"h\": 8,\n \"w\": 12,\n \"x\": 12,\n \"y\": 27\n },\n \"id\": 4,\n \"options\": {\n \"legend\": {\n \"calcs\": [],\n \"displayMode\": \"list\",\n \"placement\": \"bottom\",\n \"showLegend\": true\n },\n \"tooltip\": {\n \"hideZeros\": false,\n \"mode\": \"single\",\n \"sort\": \"none\"\n }\n },\n \"pluginVersion\": \"12.4.0\",\n \"targets\": [\n {\n \"editorMode\": \"code\",\n \"exemplar\": false,\n \"expr\": \"histogram_avg(rate(splice_calculate_rewards_v2_processing_delay_duration_seconds{namespace=~\\\"$namespace\\\"}[$__rate_interval]))\",\n \"instant\": true,\n \"legendFormat\": \"{{namespace}} {{dryRun}}\",\n \"range\": true,\n \"refId\": \"A\"\n }\n ],\n \"title\": \"Latency\",\n \"type\": \"timeseries\"\n },\n {\n \"collapsed\": false,\n \"gridPos\": {\n \"h\": 1,\n \"w\": 24,\n \"x\": 0,\n \"y\": 35\n },\n \"id\": 5,\n \"panels\": [],\n \"title\": \"ProcessRewardsTrigger (creating minting allowances)\",\n \"type\": \"row\"\n },\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"fieldConfig\": {\n \"defaults\": {\n \"color\": {\n \"mode\": \"palette-classic\"\n },\n \"custom\": {\n \"axisBorderShow\": false,\n \"axisCenteredZero\": false,\n \"axisColorMode\": \"text\",\n \"axisLabel\": \"\",\n \"axisPlacement\": \"auto\",\n \"barAlignment\": 0,\n \"barWidthFactor\": 0.6,\n \"drawStyle\": \"line\",\n \"fillOpacity\": 0,\n \"gradientMode\": \"none\",\n \"hideFrom\": {\n \"legend\": false,\n \"tooltip\": false,\n \"viz\": false\n },\n \"insertNulls\": false,\n \"lineInterpolation\": \"linear\",\n \"lineWidth\": 1,\n \"pointSize\": 5,\n \"scaleDistribution\": {\n \"type\": \"linear\"\n },\n \"showPoints\": \"auto\",\n \"showValues\": false,\n \"spanNulls\": false,\n \"stacking\": {\n \"group\": \"A\",\n \"mode\": \"none\"\n },\n \"thresholdsStyle\": {\n \"mode\": \"off\"\n }\n },\n \"mappings\": [],\n \"thresholds\": {\n \"mode\": \"absolute\",\n \"steps\": [\n {\n \"color\": \"green\",\n \"value\": 0\n },\n {\n \"color\": \"red\",\n \"value\": 80\n }\n ]\n }\n },\n \"overrides\": []\n },\n \"gridPos\": {\n \"h\": 8,\n \"w\": 12,\n \"x\": 0,\n \"y\": 36\n },\n \"id\": 2,\n \"options\": {\n \"legend\": {\n \"calcs\": [],\n \"displayMode\": \"list\",\n \"placement\": \"bottom\",\n \"showLegend\": true\n },\n \"tooltip\": {\n \"hideZeros\": false,\n \"mode\": \"single\",\n \"sort\": \"none\"\n }\n },\n \"pluginVersion\": \"12.4.0\",\n \"targets\": [\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"editorMode\": \"code\",\n \"expr\": \"rate(splice_trigger_completed_total{trigger_name=~\\\"ProcessRewards(DryRun)?Trigger\\\",outcome=~\\\"success|failure\\\", namespace=~\\\"$namespace\\\"}[$__rate_interval])\",\n \"legendFormat\": \"{{namespace}} {{trigger_name}} {{outcome}}\",\n \"range\": true,\n \"refId\": \"A\"\n }\n ],\n \"title\": \"Trigger runs\",\n \"type\": \"timeseries\"\n },\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"description\": \"The time it took between the closing of a round, and this SV's processing of a ProcessRewardsV2 contract for that round\",\n \"fieldConfig\": {\n \"defaults\": {\n \"color\": {\n \"mode\": \"palette-classic\"\n },\n \"custom\": {\n \"axisBorderShow\": false,\n \"axisCenteredZero\": false,\n \"axisColorMode\": \"text\",\n \"axisLabel\": \"\",\n \"axisPlacement\": \"auto\",\n \"barAlignment\": 0,\n \"barWidthFactor\": 0.6,\n \"drawStyle\": \"line\",\n \"fillOpacity\": 0,\n \"gradientMode\": \"none\",\n \"hideFrom\": {\n \"legend\": false,\n \"tooltip\": false,\n \"viz\": false\n },\n \"insertNulls\": false,\n \"lineInterpolation\": \"linear\",\n \"lineWidth\": 1,\n \"pointSize\": 5,\n \"scaleDistribution\": {\n \"type\": \"linear\"\n },\n \"showPoints\": \"auto\",\n \"showValues\": false,\n \"spanNulls\": false,\n \"stacking\": {\n \"group\": \"A\",\n \"mode\": \"none\"\n },\n \"thresholdsStyle\": {\n \"mode\": \"off\"\n }\n },\n \"mappings\": [],\n \"thresholds\": {\n \"mode\": \"absolute\",\n \"steps\": [\n {\n \"color\": \"green\",\n \"value\": 0\n },\n {\n \"color\": \"red\",\n \"value\": 80\n }\n ]\n },\n \"unit\": \"s\"\n },\n \"overrides\": []\n },\n \"gridPos\": {\n \"h\": 8,\n \"w\": 12,\n \"x\": 12,\n \"y\": 36\n },\n \"id\": 6,\n \"options\": {\n \"legend\": {\n \"calcs\": [],\n \"displayMode\": \"list\",\n \"placement\": \"bottom\",\n \"showLegend\": true\n },\n \"tooltip\": {\n \"hideZeros\": false,\n \"mode\": \"single\",\n \"sort\": \"none\"\n }\n },\n \"pluginVersion\": \"12.4.0\",\n \"targets\": [\n {\n \"editorMode\": \"code\",\n \"expr\": \"histogram_avg(rate(splice_process_rewards_v2_processing_delay_duration_seconds{namespace=~\\\"$namespace\\\"}[$__rate_interval]))\",\n \"legendFormat\": \"{{namespace}} {{dryRun}}\",\n \"range\": true,\n \"refId\": \"A\"\n }\n ],\n \"title\": \"Latency\",\n \"type\": \"timeseries\"\n }\n ],\n \"preload\": false,\n \"schemaVersion\": 42,\n \"tags\": [],\n \"templating\": {\n \"list\": [\n {\n \"current\": {\n \"text\": \"sv-7\",\n \"value\": \"sv-7\"\n },\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"definition\": \"label_values(splice_calculate_rewards_v2_processing_delay_duration_seconds,namespace)\",\n \"includeAll\": true,\n \"name\": \"namespace\",\n \"options\": [],\n \"query\": {\n \"qryType\": 1,\n \"query\": \"label_values(splice_calculate_rewards_v2_processing_delay_duration_seconds,namespace)\",\n \"refId\": \"PrometheusVariableQueryEditor-VariableQuery\"\n },\n \"refresh\": 1,\n \"regex\": \"\",\n \"regexApplyTo\": \"value\",\n \"type\": \"query\"\n }\n ]\n },\n \"time\": {\n \"from\": \"now-12h\",\n \"to\": \"now\"\n },\n \"timepicker\": {},\n \"timezone\": \"\",\n \"title\": \"Traffic-based app rewards (CIP-104)\",\n \"uid\": \"cnx7gsn\",\n \"version\": 8,\n \"weekStart\": \"\"\n}\n", "automations.json": "{\n \"annotations\": {\n \"list\": [\n {\n \"builtIn\": 1,\n \"datasource\": {\n \"type\": \"grafana\",\n \"uid\": \"-- Grafana --\"\n },\n \"enable\": true,\n \"hide\": true,\n \"iconColor\": \"rgba(0, 211, 255, 1)\",\n \"name\": \"Annotations & Alerts\",\n \"type\": \"dashboard\"\n }\n ]\n },\n \"description\": \"Track the trigger automations being run by Splice Apps\",\n \"editable\": true,\n \"fiscalYearStartMonth\": 0,\n \"graphTooltip\": 0,\n \"links\": [],\n \"panels\": [\n {\n \"collapsed\": false,\n \"gridPos\": {\n \"h\": 1,\n \"w\": 24,\n \"x\": 0,\n \"y\": 0\n },\n \"id\": 30,\n \"panels\": [],\n \"title\": \"Automation Service Health\",\n \"type\": \"row\"\n },\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"fieldConfig\": {\n \"defaults\": {\n \"color\": {\n \"mode\": \"thresholds\"\n },\n \"mappings\": [],\n \"thresholds\": {\n \"mode\": \"absolute\",\n \"steps\": [\n {\n \"color\": \"green\",\n \"value\": 0\n },\n {\n \"color\": \"red\",\n \"value\": 1\n }\n ]\n }\n },\n \"overrides\": []\n },\n \"gridPos\": {\n \"h\": 9,\n \"w\": 5,\n \"x\": 0,\n \"y\": 1\n },\n \"id\": 34,\n \"options\": {\n \"colorMode\": \"value\",\n \"graphMode\": \"area\",\n \"justifyMode\": \"auto\",\n \"orientation\": \"auto\",\n \"percentChangeColorMode\": \"standard\",\n \"reduceOptions\": {\n \"calcs\": [\n \"lastNotNull\"\n ],\n \"fields\": \"\",\n \"values\": false\n },\n \"showPercentChange\": false,\n \"textMode\": \"auto\",\n \"wideLayout\": true\n },\n \"pluginVersion\": \"12.4.0\",\n \"targets\": [\n {\n \"editorMode\": \"code\",\n \"expr\": \"count by (namspace, job, automation_service, exported_service, party) (splice_automation_background_service_health{namespace=~\\\"$namespace\\\", node_type=~\\\"$app\\\"} == 1) or vector(0)\",\n \"legendFormat\": \"__auto\",\n \"range\": true,\n \"refId\": \"A\"\n }\n ],\n \"title\": \"Unhealthy services\",\n \"type\": \"stat\"\n },\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"description\": \"Current health of registered automation background services.\\n-1 = no reading yet (or error reading health), 0 = healthy, 1 = unhealthy.\",\n \"fieldConfig\": {\n \"defaults\": {\n \"color\": {\n \"mode\": \"thresholds\"\n },\n \"custom\": {\n \"align\": \"auto\",\n \"cellOptions\": {\n \"type\": \"color-background\"\n },\n \"footer\": {\n \"reducers\": []\n },\n \"inspect\": false\n },\n \"mappings\": [\n {\n \"options\": {\n \"0\": {\n \"color\": \"green\",\n \"index\": 1,\n \"text\": \"healthy\"\n },\n \"1\": {\n \"color\": \"red\",\n \"index\": 2,\n \"text\": \"unhealthy\"\n },\n \"-1\": {\n \"color\": \"text\",\n \"index\": 0,\n \"text\": \"unknown\"\n }\n },\n \"type\": \"value\"\n }\n ],\n \"thresholds\": {\n \"mode\": \"absolute\",\n \"steps\": [\n {\n \"color\": \"text\",\n \"value\": 0\n },\n {\n \"color\": \"green\",\n \"value\": 0\n },\n {\n \"color\": \"red\",\n \"value\": 1\n }\n ]\n }\n },\n \"overrides\": [\n {\n \"matcher\": {\n \"id\": \"byName\",\n \"options\": \"Time\"\n },\n \"properties\": [\n {\n \"id\": \"custom.hideFrom.viz\",\n \"value\": true\n }\n ]\n },\n {\n \"matcher\": {\n \"id\": \"byName\",\n \"options\": \"exported_service\"\n },\n \"properties\": [\n {\n \"id\": \"custom.width\",\n \"value\": 785\n }\n ]\n },\n {\n \"matcher\": {\n \"id\": \"byName\",\n \"options\": \"Node\"\n },\n \"properties\": [\n {\n \"id\": \"custom.width\",\n \"value\": 109\n }\n ]\n },\n {\n \"matcher\": {\n \"id\": \"byName\",\n \"options\": \"Namespace\"\n },\n \"properties\": [\n {\n \"id\": \"custom.width\",\n \"value\": 110\n }\n ]\n },\n {\n \"matcher\": {\n \"id\": \"byName\",\n \"options\": \"automation_service\"\n },\n \"properties\": [\n {\n \"id\": \"custom.width\",\n \"value\": 296\n }\n ]\n },\n {\n \"matcher\": {\n \"id\": \"byName\",\n \"options\": \"App\"\n },\n \"properties\": [\n {\n \"id\": \"custom.width\",\n \"value\": 94\n }\n ]\n }\n ]\n },\n \"gridPos\": {\n \"h\": 9,\n \"w\": 19,\n \"x\": 5,\n \"y\": 1\n },\n \"id\": 31,\n \"options\": {\n \"cellHeight\": \"sm\",\n \"showHeader\": true,\n \"sortBy\": [\n {\n \"desc\": true,\n \"displayName\": \"Status\"\n }\n ]\n },\n \"pluginVersion\": \"12.4.0\",\n \"targets\": [\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"editorMode\": \"code\",\n \"exemplar\": false,\n \"expr\": \"splice_automation_background_service_health{namespace=~\\\"$namespace\\\", node_type=~\\\"$app\\\"}\",\n \"format\": \"table\",\n \"instant\": true,\n \"legendFormat\": \"__auto\",\n \"range\": false,\n \"refId\": \"A\"\n }\n ],\n \"title\": \"Background service health\",\n \"transformations\": [\n {\n \"id\": \"organize\",\n \"options\": {\n \"excludeByName\": {\n \"Time\": true,\n \"__name__\": true,\n \"container\": true,\n \"endpoint\": true,\n \"instance\": true,\n \"job\": true,\n \"migration\": true,\n \"otel_scope_name\": true,\n \"pod\": true,\n \"prometheus\": true,\n \"service\": true\n },\n \"includeByName\": {},\n \"indexByName\": {},\n \"renameByName\": {\n \"Value\": \"Status\",\n \"instance\": \"Instance\",\n \"namespace\": \"Namespace\",\n \"node_name\": \"Node\",\n \"node_type\": \"App\",\n \"service\": \"Service\"\n }\n }\n }\n ],\n \"type\": \"table\"\n },\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"description\": \"Health of automation background services over time (-1 unknown, 0 healthy, 1 unhealthy).\",\n \"fieldConfig\": {\n \"defaults\": {\n \"color\": {\n \"mode\": \"palette-classic\"\n },\n \"custom\": {\n \"axisBorderShow\": false,\n \"axisCenteredZero\": false,\n \"axisColorMode\": \"text\",\n \"axisLabel\": \"\",\n \"axisPlacement\": \"auto\",\n \"barAlignment\": 0,\n \"barWidthFactor\": 0.6,\n \"drawStyle\": \"line\",\n \"fillOpacity\": 10,\n \"gradientMode\": \"none\",\n \"hideFrom\": {\n \"legend\": false,\n \"tooltip\": false,\n \"viz\": false\n },\n \"insertNulls\": false,\n \"lineInterpolation\": \"stepAfter\",\n \"lineWidth\": 1,\n \"pointSize\": 5,\n \"scaleDistribution\": {\n \"type\": \"linear\"\n },\n \"showPoints\": \"auto\",\n \"showValues\": false,\n \"spanNulls\": false,\n \"stacking\": {\n \"group\": \"A\",\n \"mode\": \"none\"\n },\n \"thresholdsStyle\": {\n \"mode\": \"off\"\n }\n },\n \"mappings\": [\n {\n \"options\": {\n \"0\": {\n \"index\": 1,\n \"text\": \"healthy\"\n },\n \"1\": {\n \"index\": 2,\n \"text\": \"unhealthy\"\n },\n \"-1\": {\n \"index\": 0,\n \"text\": \"unknown\"\n }\n },\n \"type\": \"value\"\n }\n ],\n \"max\": 1,\n \"min\": -1,\n \"thresholds\": {\n \"mode\": \"absolute\",\n \"steps\": [\n {\n \"color\": \"green\",\n \"value\": 0\n },\n {\n \"color\": \"red\",\n \"value\": 1\n }\n ]\n }\n },\n \"overrides\": []\n },\n \"gridPos\": {\n \"h\": 10,\n \"w\": 24,\n \"x\": 0,\n \"y\": 10\n },\n \"id\": 32,\n \"options\": {\n \"legend\": {\n \"calcs\": [\n \"lastNotNull\"\n ],\n \"displayMode\": \"table\",\n \"placement\": \"right\",\n \"showLegend\": true,\n \"sortBy\": \"Last *\",\n \"sortDesc\": true\n },\n \"tooltip\": {\n \"hideZeros\": false,\n \"mode\": \"multi\",\n \"sort\": \"desc\"\n }\n },\n \"pluginVersion\": \"12.4.0\",\n \"targets\": [\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"editorMode\": \"code\",\n \"expr\": \"splice_automation_background_service_health{namespace=~\\\"$namespace\\\", node_type=~\\\"$app\\\"}\",\n \"instant\": false,\n \"legendFormat\": \"{{namespace}} {{node_type}} {{automation_service}} {{exported_service}}\",\n \"range\": true,\n \"refId\": \"A\"\n }\n ],\n \"title\": \"Background service health over time\",\n \"type\": \"timeseries\"\n },\n {\n \"collapsed\": false,\n \"gridPos\": {\n \"h\": 1,\n \"w\": 24,\n \"x\": 0,\n \"y\": 20\n },\n \"id\": 33,\n \"panels\": [],\n \"title\": \"Runs\",\n \"type\": \"row\"\n },\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"fieldConfig\": {\n \"defaults\": {\n \"color\": {\n \"mode\": \"palette-classic\"\n },\n \"custom\": {\n \"axisBorderShow\": false,\n \"axisCenteredZero\": false,\n \"axisColorMode\": \"text\",\n \"axisLabel\": \"\",\n \"axisPlacement\": \"auto\",\n \"barAlignment\": 0,\n \"barWidthFactor\": 0.6,\n \"drawStyle\": \"line\",\n \"fillOpacity\": 0,\n \"gradientMode\": \"none\",\n \"hideFrom\": {\n \"legend\": false,\n \"tooltip\": false,\n \"viz\": false\n },\n \"insertNulls\": false,\n \"lineInterpolation\": \"linear\",\n \"lineWidth\": 1,\n \"pointSize\": 5,\n \"scaleDistribution\": {\n \"type\": \"linear\"\n },\n \"showPoints\": \"auto\",\n \"showValues\": false,\n \"spanNulls\": false,\n \"stacking\": {\n \"group\": \"A\",\n \"mode\": \"none\"\n },\n \"thresholdsStyle\": {\n \"mode\": \"off\"\n }\n },\n \"mappings\": [],\n \"thresholds\": {\n \"mode\": \"absolute\",\n \"steps\": [\n {\n \"color\": \"green\",\n \"value\": 0\n },\n {\n \"color\": \"red\",\n \"value\": 80\n }\n ]\n },\n \"unit\": \"runs / second\"\n },\n \"overrides\": [\n {\n \"matcher\": {\n \"id\": \"byRegexp\",\n \"options\": \"/.*failure/\"\n },\n \"properties\": [\n {\n \"id\": \"custom.fillOpacity\",\n \"value\": 50\n }\n ]\n }\n ]\n },\n \"gridPos\": {\n \"h\": 10,\n \"w\": 24,\n \"x\": 0,\n \"y\": 21\n },\n \"id\": 18,\n \"options\": {\n \"legend\": {\n \"calcs\": [\n \"lastNotNull\"\n ],\n \"displayMode\": \"table\",\n \"placement\": \"right\",\n \"showLegend\": true,\n \"sortBy\": \"Last *\",\n \"sortDesc\": true\n },\n \"tooltip\": {\n \"hideZeros\": false,\n \"mode\": \"multi\",\n \"sort\": \"desc\"\n }\n },\n \"pluginVersion\": \"12.4.0\",\n \"targets\": [\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"disableTextWrap\": false,\n \"editorMode\": \"code\",\n \"expr\": \"rate(splice_trigger_completed_total{namespace=~\\\"$namespace\\\", node_type=~\\\"$app\\\", trigger_name=~\\\"$trigger_name\\\", outcome=~\\\"success|failure\\\"}[$__rate_interval])\",\n \"fullMetaSearch\": false,\n \"includeNullMetadata\": false,\n \"instant\": false,\n \"legendFormat\": \"{{node_name}} {{trigger_name}} {{outcome}} \",\n \"range\": true,\n \"refId\": \"A\",\n \"useBackend\": false\n }\n ],\n \"title\": \"Runs\",\n \"type\": \"timeseries\"\n },\n {\n \"collapsed\": false,\n \"gridPos\": {\n \"h\": 1,\n \"w\": 24,\n \"x\": 0,\n \"y\": 31\n },\n \"id\": 3,\n \"panels\": [],\n \"repeat\": \"trigger_type\",\n \"title\": \"Triggers $trigger_type\",\n \"type\": \"row\"\n },\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"description\": \"The rate of successful iterations completed\",\n \"fieldConfig\": {\n \"defaults\": {\n \"color\": {\n \"mode\": \"thresholds\"\n },\n \"mappings\": [],\n \"noValue\": \"0\",\n \"thresholds\": {\n \"mode\": \"absolute\",\n \"steps\": [\n {\n \"color\": \"green\",\n \"value\": 0\n }\n ]\n }\n },\n \"overrides\": []\n },\n \"gridPos\": {\n \"h\": 7,\n \"w\": 4,\n \"x\": 0,\n \"y\": 32\n },\n \"id\": 5,\n \"options\": {\n \"colorMode\": \"none\",\n \"graphMode\": \"area\",\n \"justifyMode\": \"center\",\n \"orientation\": \"auto\",\n \"percentChangeColorMode\": \"standard\",\n \"reduceOptions\": {\n \"calcs\": [\n \"lastNotNull\"\n ],\n \"fields\": \"\",\n \"values\": false\n },\n \"showPercentChange\": false,\n \"textMode\": \"auto\",\n \"wideLayout\": true\n },\n \"pluginVersion\": \"12.4.0\",\n \"targets\": [\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"editorMode\": \"code\",\n \"exemplar\": false,\n \"expr\": \"sum(histogram_sum(rate(splice_trigger_latency_duration_seconds{namespace=~\\\"$namespace\\\", node_type=~\\\"$app\\\", trigger_name=~\\\"$trigger_name\\\", trigger_type=\\\"$trigger_type\\\"}[$__rate_interval])))\",\n \"instant\": true,\n \"range\": false,\n \"refId\": \"A\"\n }\n ],\n \"title\": \"Traffic\",\n \"type\": \"stat\"\n },\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"description\": \"The rate at which the trigger(s) are failing with retryable errors\",\n \"fieldConfig\": {\n \"defaults\": {\n \"color\": {\n \"mode\": \"thresholds\"\n },\n \"mappings\": [],\n \"thresholds\": {\n \"mode\": \"absolute\",\n \"steps\": [\n {\n \"color\": \"green\",\n \"value\": 0\n }\n ]\n }\n },\n \"overrides\": []\n },\n \"gridPos\": {\n \"h\": 7,\n \"w\": 4,\n \"x\": 4,\n \"y\": 32\n },\n \"id\": 7,\n \"options\": {\n \"colorMode\": \"value\",\n \"graphMode\": \"area\",\n \"justifyMode\": \"auto\",\n \"orientation\": \"auto\",\n \"percentChangeColorMode\": \"standard\",\n \"reduceOptions\": {\n \"calcs\": [\n \"lastNotNull\"\n ],\n \"fields\": \"\",\n \"values\": false\n },\n \"showPercentChange\": false,\n \"textMode\": \"auto\",\n \"wideLayout\": true\n },\n \"pluginVersion\": \"12.4.0\",\n \"targets\": [\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"editorMode\": \"code\",\n \"expr\": \"sum(rate(splice_retries_failures{namespace=~\\\"$namespace\\\", node_type=~\\\"$app\\\", trigger_name=~\\\"$trigger_name\\\", trigger_type=\\\"$trigger_type\\\", error_kind=\\\"transient\\\"}[$__rate_interval]))\",\n \"instant\": false,\n \"range\": true,\n \"refId\": \"A\"\n }\n ],\n \"title\": \"Retries\",\n \"type\": \"stat\"\n },\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"description\": \"The rate at which trigger(s) are failing with fatal/non-retryable errors\",\n \"fieldConfig\": {\n \"defaults\": {\n \"color\": {\n \"mode\": \"thresholds\"\n },\n \"mappings\": [],\n \"thresholds\": {\n \"mode\": \"absolute\",\n \"steps\": [\n {\n \"color\": \"green\",\n \"value\": 0\n }\n ]\n }\n },\n \"overrides\": []\n },\n \"gridPos\": {\n \"h\": 7,\n \"w\": 4,\n \"x\": 8,\n \"y\": 32\n },\n \"id\": 8,\n \"options\": {\n \"colorMode\": \"value\",\n \"graphMode\": \"area\",\n \"justifyMode\": \"auto\",\n \"orientation\": \"auto\",\n \"percentChangeColorMode\": \"standard\",\n \"reduceOptions\": {\n \"calcs\": [\n \"lastNotNull\"\n ],\n \"fields\": \"\",\n \"values\": false\n },\n \"showPercentChange\": false,\n \"textMode\": \"auto\",\n \"wideLayout\": true\n },\n \"pluginVersion\": \"12.4.0\",\n \"targets\": [\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"editorMode\": \"code\",\n \"expr\": \"sum(rate(splice_retries_failures{namespace=~\\\"$namespace\\\", node_type=~\\\"$app\\\", trigger_name=~\\\"$trigger_name\\\", trigger_type=\\\"$trigger_type\\\", error_kind=\\\"fatal\\\"}[$__rate_interval]))\",\n \"instant\": false,\n \"range\": true,\n \"refId\": \"A\"\n }\n ],\n \"title\": \"Errors\",\n \"type\": \"stat\"\n },\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"description\": \"The time it took to perform an iteration\",\n \"fieldConfig\": {\n \"defaults\": {\n \"color\": {\n \"mode\": \"palette-classic\"\n },\n \"custom\": {\n \"axisBorderShow\": false,\n \"axisCenteredZero\": false,\n \"axisColorMode\": \"text\",\n \"axisLabel\": \"\",\n \"axisPlacement\": \"auto\",\n \"barAlignment\": 0,\n \"barWidthFactor\": 0.6,\n \"drawStyle\": \"line\",\n \"fillOpacity\": 0,\n \"gradientMode\": \"none\",\n \"hideFrom\": {\n \"legend\": false,\n \"tooltip\": false,\n \"viz\": false\n },\n \"insertNulls\": false,\n \"lineInterpolation\": \"linear\",\n \"lineWidth\": 1,\n \"pointSize\": 5,\n \"scaleDistribution\": {\n \"type\": \"linear\"\n },\n \"showPoints\": \"auto\",\n \"showValues\": false,\n \"spanNulls\": false,\n \"stacking\": {\n \"group\": \"A\",\n \"mode\": \"none\"\n },\n \"thresholdsStyle\": {\n \"mode\": \"off\"\n }\n },\n \"mappings\": [],\n \"thresholds\": {\n \"mode\": \"absolute\",\n \"steps\": [\n {\n \"color\": \"green\",\n \"value\": 0\n },\n {\n \"color\": \"red\",\n \"value\": 80\n }\n ]\n },\n \"unit\": \"s\"\n },\n \"overrides\": []\n },\n \"gridPos\": {\n \"h\": 7,\n \"w\": 12,\n \"x\": 12,\n \"y\": 32\n },\n \"id\": 6,\n \"options\": {\n \"legend\": {\n \"calcs\": [],\n \"displayMode\": \"list\",\n \"placement\": \"bottom\",\n \"showLegend\": true\n },\n \"tooltip\": {\n \"hideZeros\": false,\n \"mode\": \"single\",\n \"sort\": \"none\"\n }\n },\n \"pluginVersion\": \"12.4.0\",\n \"targets\": [\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"editorMode\": \"code\",\n \"expr\": \"histogram_quantile(0.95, rate(splice_trigger_latency_duration_seconds{namespace=~\\\"$namespace\\\", node_type=~\\\"$app\\\", trigger_name=~\\\"$trigger_name\\\", trigger_type=\\\"$trigger_type\\\"}[$__rate_interval]))\",\n \"instant\": false,\n \"legendFormat\": \"{{namespace}} {{node_type}} {{service}} {{trigger_name}}\",\n \"range\": true,\n \"refId\": \"A\"\n }\n ],\n \"title\": \"Latency\",\n \"type\": \"timeseries\"\n },\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"fieldConfig\": {\n \"defaults\": {\n \"color\": {\n \"mode\": \"continuous-RdYlGr\"\n },\n \"custom\": {\n \"align\": \"auto\",\n \"cellOptions\": {\n \"type\": \"auto\"\n },\n \"footer\": {\n \"reducers\": []\n },\n \"inspect\": false\n },\n \"mappings\": [],\n \"thresholds\": {\n \"mode\": \"absolute\",\n \"steps\": [\n {\n \"color\": \"green\",\n \"value\": 0\n },\n {\n \"color\": \"red\",\n \"value\": 80\n },\n {\n \"color\": \"#EAB839\",\n \"value\": 90\n },\n {\n \"color\": \"#6ED0E0\",\n \"value\": 100\n },\n {\n \"color\": \"#EF843C\",\n \"value\": 110\n },\n {\n \"color\": \"#E24D42\",\n \"value\": 120\n }\n ]\n }\n },\n \"overrides\": []\n },\n \"gridPos\": {\n \"h\": 8,\n \"w\": 8,\n \"x\": 0,\n \"y\": 39\n },\n \"id\": 10,\n \"options\": {\n \"cellHeight\": \"sm\",\n \"frameIndex\": 0,\n \"showHeader\": true,\n \"sortBy\": []\n },\n \"pluginVersion\": \"12.4.0\",\n \"targets\": [\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"editorMode\": \"code\",\n \"exemplar\": false,\n \"expr\": \"sum by(trigger_name) (histogram_count(rate(splice_trigger_latency_duration_seconds{namespace=~\\\"$namespace\\\", node_type=~\\\"$app\\\", trigger_type=\\\"$trigger_type\\\"}[$__rate_interval])))\",\n \"format\": \"time_series\",\n \"instant\": false,\n \"range\": true,\n \"refId\": \"A\"\n }\n ],\n \"title\": \"Top 5 Active Triggers\",\n \"transformations\": [\n {\n \"id\": \"reduce\",\n \"options\": {\n \"includeTimeField\": false,\n \"labelsToFields\": true,\n \"mode\": \"seriesToRows\",\n \"reducers\": [\n \"sum\"\n ]\n }\n },\n {\n \"id\": \"sortBy\",\n \"options\": {\n \"fields\": {},\n \"sort\": [\n {\n \"desc\": true,\n \"field\": \"Total\"\n }\n ]\n }\n },\n {\n \"id\": \"organize\",\n \"options\": {\n \"excludeByName\": {\n \"Field\": true\n },\n \"indexByName\": {},\n \"renameByName\": {\n \"Total\": \"Traffic\",\n \"trigger_name\": \"Trigger\"\n }\n }\n },\n {\n \"id\": \"limit\",\n \"options\": {\n \"limitField\": 5\n }\n }\n ],\n \"type\": \"table\"\n },\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"description\": \"How busy triggers are within a certain time window\",\n \"fieldConfig\": {\n \"defaults\": {\n \"color\": {\n \"mode\": \"palette-classic\"\n },\n \"custom\": {\n \"axisBorderShow\": false,\n \"axisCenteredZero\": false,\n \"axisColorMode\": \"text\",\n \"axisLabel\": \"\",\n \"axisPlacement\": \"auto\",\n \"barAlignment\": 0,\n \"barWidthFactor\": 0.6,\n \"drawStyle\": \"line\",\n \"fillOpacity\": 0,\n \"gradientMode\": \"none\",\n \"hideFrom\": {\n \"legend\": false,\n \"tooltip\": false,\n \"viz\": false\n },\n \"insertNulls\": false,\n \"lineInterpolation\": \"linear\",\n \"lineWidth\": 1,\n \"pointSize\": 5,\n \"scaleDistribution\": {\n \"type\": \"linear\"\n },\n \"showPoints\": \"auto\",\n \"showValues\": false,\n \"spanNulls\": false,\n \"stacking\": {\n \"group\": \"A\",\n \"mode\": \"none\"\n },\n \"thresholdsStyle\": {\n \"mode\": \"off\"\n }\n },\n \"mappings\": [],\n \"thresholds\": {\n \"mode\": \"absolute\",\n \"steps\": [\n {\n \"color\": \"green\",\n \"value\": 0\n }\n ]\n },\n \"unit\": \"percentunit\"\n },\n \"overrides\": []\n },\n \"gridPos\": {\n \"h\": 8,\n \"w\": 16,\n \"x\": 8,\n \"y\": 39\n },\n \"id\": 9,\n \"options\": {\n \"legend\": {\n \"calcs\": [\n \"lastNotNull\"\n ],\n \"displayMode\": \"table\",\n \"placement\": \"right\",\n \"showLegend\": true,\n \"sortBy\": \"Last *\",\n \"sortDesc\": true\n },\n \"tooltip\": {\n \"hideZeros\": false,\n \"mode\": \"multi\",\n \"sort\": \"desc\"\n }\n },\n \"pluginVersion\": \"12.4.0\",\n \"targets\": [\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"editorMode\": \"code\",\n \"exemplar\": false,\n \"expr\": \"sum by(trigger_name, instance, node_type, trigger_type) (histogram_sum(rate(splice_trigger_latency_duration_seconds{namespace=~\\\"$namespace\\\", node_type=~\\\"$app\\\", trigger_name=~\\\"$trigger_name\\\", trigger_type=\\\"$trigger_type\\\"}[$__rate_interval])))\",\n \"instant\": false,\n \"legendFormat\": \"{{node_type}} {{instance}} {{trigger_name}}\",\n \"range\": true,\n \"refId\": \"A\"\n }\n ],\n \"title\": \"Saturation\",\n \"type\": \"timeseries\"\n },\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"description\": \"How busy triggers are within a certain time window\",\n \"fieldConfig\": {\n \"defaults\": {\n \"color\": {\n \"mode\": \"palette-classic\"\n },\n \"custom\": {\n \"axisBorderShow\": false,\n \"axisCenteredZero\": false,\n \"axisColorMode\": \"text\",\n \"axisLabel\": \"\",\n \"axisPlacement\": \"auto\",\n \"barAlignment\": 0,\n \"barWidthFactor\": 0.6,\n \"drawStyle\": \"line\",\n \"fillOpacity\": 0,\n \"gradientMode\": \"none\",\n \"hideFrom\": {\n \"legend\": false,\n \"tooltip\": false,\n \"viz\": false\n },\n \"insertNulls\": false,\n \"lineInterpolation\": \"linear\",\n \"lineWidth\": 1,\n \"pointSize\": 5,\n \"scaleDistribution\": {\n \"type\": \"linear\"\n },\n \"showPoints\": \"auto\",\n \"showValues\": false,\n \"spanNulls\": false,\n \"stacking\": {\n \"group\": \"A\",\n \"mode\": \"none\"\n },\n \"thresholdsStyle\": {\n \"mode\": \"off\"\n }\n },\n \"mappings\": [],\n \"thresholds\": {\n \"mode\": \"absolute\",\n \"steps\": [\n {\n \"color\": \"green\",\n \"value\": 0\n }\n ]\n },\n \"unit\": \"ops\"\n },\n \"overrides\": []\n },\n \"gridPos\": {\n \"h\": 8,\n \"w\": 24,\n \"x\": 0,\n \"y\": 47\n },\n \"id\": 23,\n \"options\": {\n \"legend\": {\n \"calcs\": [\n \"lastNotNull\"\n ],\n \"displayMode\": \"table\",\n \"placement\": \"right\",\n \"showLegend\": true,\n \"sortBy\": \"Last *\",\n \"sortDesc\": true\n },\n \"tooltip\": {\n \"hideZeros\": false,\n \"mode\": \"multi\",\n \"sort\": \"desc\"\n }\n },\n \"pluginVersion\": \"12.4.0\",\n \"targets\": [\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"editorMode\": \"code\",\n \"exemplar\": false,\n \"expr\": \"sum by(trigger_name, instance, node_type, trigger_type, outcome) (rate(splice_trigger_completed_total{namespace=~\\\"$namespace\\\", node_type=~\\\"$app\\\", trigger_name=~\\\"$trigger_name\\\", trigger_type=\\\"$trigger_type\\\"}[$__rate_interval]))\",\n \"instant\": false,\n \"legendFormat\": \"{{node_type}} {{instance}} {{trigger_name}} {{outcome}}\",\n \"range\": true,\n \"refId\": \"A\"\n }\n ],\n \"title\": \"Completed\",\n \"type\": \"timeseries\"\n },\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"fieldConfig\": {\n \"defaults\": {\n \"color\": {\n \"mode\": \"palette-classic\"\n },\n \"custom\": {\n \"axisBorderShow\": false,\n \"axisCenteredZero\": false,\n \"axisColorMode\": \"text\",\n \"axisLabel\": \"\",\n \"axisPlacement\": \"auto\",\n \"barAlignment\": 0,\n \"barWidthFactor\": 0.6,\n \"drawStyle\": \"line\",\n \"fillOpacity\": 0,\n \"gradientMode\": \"none\",\n \"hideFrom\": {\n \"legend\": false,\n \"tooltip\": false,\n \"viz\": false\n },\n \"insertNulls\": false,\n \"lineInterpolation\": \"linear\",\n \"lineWidth\": 1,\n \"pointSize\": 5,\n \"scaleDistribution\": {\n \"type\": \"linear\"\n },\n \"showPoints\": \"auto\",\n \"showValues\": false,\n \"spanNulls\": false,\n \"stacking\": {\n \"group\": \"A\",\n \"mode\": \"none\"\n },\n \"thresholdsStyle\": {\n \"mode\": \"off\"\n }\n },\n \"mappings\": [],\n \"thresholds\": {\n \"mode\": \"absolute\",\n \"steps\": [\n {\n \"color\": \"green\",\n \"value\": 0\n },\n {\n \"color\": \"red\",\n \"value\": 80\n }\n ]\n }\n },\n \"overrides\": []\n },\n \"gridPos\": {\n \"h\": 8,\n \"w\": 24,\n \"x\": 0,\n \"y\": 55\n },\n \"id\": 22,\n \"options\": {\n \"legend\": {\n \"calcs\": [],\n \"displayMode\": \"list\",\n \"placement\": \"bottom\",\n \"showLegend\": true\n },\n \"tooltip\": {\n \"hideZeros\": false,\n \"mode\": \"single\",\n \"sort\": \"none\"\n }\n },\n \"pluginVersion\": \"12.4.0\",\n \"targets\": [\n {\n \"disableTextWrap\": false,\n \"editorMode\": \"code\",\n \"expr\": \"sum(rate(splice_trigger_attempted_total{isDsoDelegateTrigger=~\\\"$isDsoDelegateTrigger\\\", trigger_name=~\\\"$trigger_name\\\", namespace=~\\\"$namespace\\\", node_type=~\\\"$app\\\"}[$__rate_interval]))\",\n \"fullMetaSearch\": false,\n \"includeNullMetadata\": true,\n \"legendFormat\": \"total attempts\",\n \"range\": true,\n \"refId\": \"A\",\n \"useBackend\": false\n },\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"disableTextWrap\": false,\n \"editorMode\": \"code\",\n \"expr\": \"sum(rate(splice_trigger_attempted_total{statusCode!~\\\"OK\\\", isDsoDelegateTrigger=~\\\"$isDsoDelegateTrigger\\\", namespace=~\\\"$namespace\\\", node_type=~\\\"$app\\\", trigger_name=~\\\"$trigger_name\\\", contentionFailure=\\\"true\\\"}[$__rate_interval]))\",\n \"fullMetaSearch\": false,\n \"includeNullMetadata\": true,\n \"legendFormat\": \"total contentions\",\n \"range\": true,\n \"refId\": \"B\",\n \"useBackend\": false\n }\n ],\n \"title\": \"Trigger attempts\",\n \"type\": \"timeseries\"\n },\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"fieldConfig\": {\n \"defaults\": {\n \"color\": {\n \"mode\": \"palette-classic\"\n },\n \"custom\": {\n \"axisBorderShow\": false,\n \"axisCenteredZero\": false,\n \"axisColorMode\": \"text\",\n \"axisLabel\": \"\",\n \"axisPlacement\": \"auto\",\n \"barAlignment\": 0,\n \"barWidthFactor\": 0.6,\n \"drawStyle\": \"line\",\n \"fillOpacity\": 0,\n \"gradientMode\": \"none\",\n \"hideFrom\": {\n \"legend\": false,\n \"tooltip\": false,\n \"viz\": false\n },\n \"insertNulls\": false,\n \"lineInterpolation\": \"linear\",\n \"lineWidth\": 1,\n \"pointSize\": 5,\n \"scaleDistribution\": {\n \"type\": \"linear\"\n },\n \"showPoints\": \"auto\",\n \"showValues\": false,\n \"spanNulls\": false,\n \"stacking\": {\n \"group\": \"A\",\n \"mode\": \"none\"\n },\n \"thresholdsStyle\": {\n \"mode\": \"off\"\n }\n },\n \"mappings\": [],\n \"thresholds\": {\n \"mode\": \"absolute\",\n \"steps\": [\n {\n \"color\": \"green\",\n \"value\": 0\n },\n {\n \"color\": \"red\",\n \"value\": 80\n }\n ]\n }\n },\n \"overrides\": []\n },\n \"gridPos\": {\n \"h\": 8,\n \"w\": 24,\n \"x\": 0,\n \"y\": 63\n },\n \"id\": 20,\n \"options\": {\n \"legend\": {\n \"calcs\": [\n \"last\"\n ],\n \"displayMode\": \"list\",\n \"placement\": \"bottom\",\n \"showLegend\": true\n },\n \"tooltip\": {\n \"hideZeros\": false,\n \"mode\": \"single\",\n \"sort\": \"none\"\n }\n },\n \"pluginVersion\": \"12.4.0\",\n \"targets\": [\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"disableTextWrap\": false,\n \"editorMode\": \"code\",\n \"expr\": \"sum by(errorCodeId, namespace, node_type, trigger_name) (rate(splice_trigger_attempted_total{namespace=~\\\"$namespace\\\", node_type=~\\\"$app\\\", trigger_name=~\\\"$trigger_name\\\", isDsoDelegateTrigger=~\\\"$isDsoDelegateTrigger\\\", contentionFailure=\\\"true\\\"}[$__rate_interval]))\",\n \"fullMetaSearch\": false,\n \"includeNullMetadata\": true,\n \"instant\": false,\n \"legendFormat\": \"__auto\",\n \"range\": true,\n \"refId\": \"A\",\n \"useBackend\": false\n }\n ],\n \"title\": \"Failed trigger attempts with contention errors\",\n \"type\": \"timeseries\"\n },\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"fieldConfig\": {\n \"defaults\": {\n \"color\": {\n \"mode\": \"palette-classic\"\n },\n \"custom\": {\n \"axisBorderShow\": false,\n \"axisCenteredZero\": false,\n \"axisColorMode\": \"text\",\n \"axisLabel\": \"\",\n \"axisPlacement\": \"auto\",\n \"barAlignment\": 0,\n \"barWidthFactor\": 0.6,\n \"drawStyle\": \"line\",\n \"fillOpacity\": 0,\n \"gradientMode\": \"none\",\n \"hideFrom\": {\n \"legend\": false,\n \"tooltip\": false,\n \"viz\": false\n },\n \"insertNulls\": false,\n \"lineInterpolation\": \"linear\",\n \"lineWidth\": 1,\n \"pointSize\": 5,\n \"scaleDistribution\": {\n \"type\": \"linear\"\n },\n \"showPoints\": \"auto\",\n \"showValues\": false,\n \"spanNulls\": false,\n \"stacking\": {\n \"group\": \"A\",\n \"mode\": \"none\"\n },\n \"thresholdsStyle\": {\n \"mode\": \"off\"\n }\n },\n \"mappings\": [],\n \"thresholds\": {\n \"mode\": \"absolute\",\n \"steps\": [\n {\n \"color\": \"green\",\n \"value\": 0\n },\n {\n \"color\": \"red\",\n \"value\": 80\n }\n ]\n }\n },\n \"overrides\": []\n },\n \"gridPos\": {\n \"h\": 8,\n \"w\": 24,\n \"x\": 0,\n \"y\": 71\n },\n \"id\": 21,\n \"options\": {\n \"legend\": {\n \"calcs\": [\n \"last\"\n ],\n \"displayMode\": \"list\",\n \"placement\": \"bottom\",\n \"showLegend\": true\n },\n \"tooltip\": {\n \"hideZeros\": false,\n \"mode\": \"single\",\n \"sort\": \"none\"\n }\n },\n \"pluginVersion\": \"12.4.0\",\n \"targets\": [\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"disableTextWrap\": false,\n \"editorMode\": \"code\",\n \"expr\": \"sum by (errorCodeId, namespace, node_type, trigger_name) (rate(splice_trigger_attempted_total{namespace=~\\\"$namespace\\\",node_type=~\\\"$app\\\",trigger_name=~\\\"$trigger_name\\\",isDsoDelegateTrigger=~\\\"$isDsoDelegateTrigger\\\", statusCode!~\\\"OK\\\", contentionFailure=\\\"false\\\"}[$__rate_interval]))\",\n \"fullMetaSearch\": false,\n \"includeNullMetadata\": true,\n \"instant\": false,\n \"legendFormat\": \"__auto\",\n \"range\": true,\n \"refId\": \"B\",\n \"useBackend\": false\n }\n ],\n \"title\": \"Failed trigger attempts with non-contention errors\",\n \"type\": \"timeseries\"\n }\n ],\n \"preload\": false,\n \"refresh\": \"\",\n \"schemaVersion\": 42,\n \"tags\": [],\n \"templating\": {\n \"list\": [\n {\n \"current\": {\n \"text\": \"sv-1\",\n \"value\": \"sv-1\"\n },\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"definition\": \"label_values(splice_trigger_completed_total,namespace)\",\n \"includeAll\": true,\n \"label\": \"Namespace\",\n \"name\": \"namespace\",\n \"options\": [],\n \"query\": {\n \"qryType\": 1,\n \"query\": \"label_values(splice_trigger_completed_total,namespace)\",\n \"refId\": \"PrometheusVariableQueryEditor-VariableQuery\"\n },\n \"refresh\": 1,\n \"regex\": \"\",\n \"regexApplyTo\": \"value\",\n \"type\": \"query\"\n },\n {\n \"current\": {\n \"text\": \"scan\",\n \"value\": \"scan\"\n },\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"definition\": \"label_values(splice_trigger_completed_total{namespace=~\\\"$namespace\\\"},node_type)\",\n \"includeAll\": true,\n \"label\": \"App\",\n \"name\": \"app\",\n \"options\": [],\n \"query\": {\n \"qryType\": 1,\n \"query\": \"label_values(splice_trigger_completed_total{namespace=~\\\"$namespace\\\"},node_type)\",\n \"refId\": \"PrometheusVariableQueryEditor-VariableQuery\"\n },\n \"refresh\": 1,\n \"regex\": \"\",\n \"regexApplyTo\": \"value\",\n \"type\": \"query\"\n },\n {\n \"current\": {\n \"text\": \"All\",\n \"value\": \"$__all\"\n },\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"definition\": \"label_values(splice_trigger_completed_total{namespace=~\\\"$namespace\\\", node_type=~\\\"$app\\\"},trigger_name)\",\n \"includeAll\": true,\n \"label\": \"Trigger Name\",\n \"name\": \"trigger_name\",\n \"options\": [],\n \"query\": {\n \"qryType\": 1,\n \"query\": \"label_values(splice_trigger_completed_total{namespace=~\\\"$namespace\\\", node_type=~\\\"$app\\\"},trigger_name)\",\n \"refId\": \"PrometheusVariableQueryEditor-VariableQuery\"\n },\n \"refresh\": 1,\n \"regex\": \"\",\n \"regexApplyTo\": \"value\",\n \"type\": \"query\"\n },\n {\n \"current\": {\n \"text\": [\n \"taskbased\",\n \"polling\"\n ],\n \"value\": [\n \"taskbased\",\n \"polling\"\n ]\n },\n \"includeAll\": false,\n \"multi\": true,\n \"name\": \"trigger_type\",\n \"options\": [\n {\n \"selected\": true,\n \"text\": \"taskbased\",\n \"value\": \"taskbased\"\n },\n {\n \"selected\": true,\n \"text\": \"polling\",\n \"value\": \"polling\"\n }\n ],\n \"query\": \"taskbased, polling\",\n \"type\": \"custom\",\n \"valuesFormat\": \"csv\"\n },\n {\n \"current\": {\n \"text\": \"All\",\n \"value\": \"$__all\"\n },\n \"definition\": \"label_values(isDsoDelegateTrigger)\",\n \"includeAll\": true,\n \"label\": \"isDsoDelegateTrigger\",\n \"name\": \"isDsoDelegateTrigger\",\n \"options\": [],\n \"query\": {\n \"qryType\": 1,\n \"query\": \"label_values(isDsoDelegateTrigger)\",\n \"refId\": \"PrometheusVariableQueryEditor-VariableQuery\"\n },\n \"refresh\": 1,\n \"regex\": \"\",\n \"regexApplyTo\": \"value\",\n \"type\": \"query\"\n }\n ]\n },\n \"time\": {\n \"from\": \"now-3h\",\n \"to\": \"now\"\n },\n \"timepicker\": {},\n \"timezone\": \"\",\n \"title\": \"Automations\",\n \"uid\": \"a3e1385f-6f03-46d9-908c-34aca0f507a6\",\n \"version\": 3,\n \"weekStart\": \"\"\n}\n", "catchup.json": "{\n \"annotations\": {\n \"list\": [\n {\n \"builtIn\": 1,\n \"datasource\": {\n \"type\": \"grafana\",\n \"uid\": \"-- Grafana --\"\n },\n \"enable\": true,\n \"hide\": true,\n \"iconColor\": \"rgba(0, 211, 255, 1)\",\n \"name\": \"Annotations & Alerts\",\n \"type\": \"dashboard\"\n }\n ]\n },\n \"editable\": true,\n \"fiscalYearStartMonth\": 0,\n \"graphTooltip\": 0,\n \"links\": [],\n \"panels\": [\n {\n \"collapsed\": false,\n \"gridPos\": {\n \"h\": 1,\n \"w\": 24,\n \"x\": 0,\n \"y\": 0\n },\n \"id\": 16,\n \"panels\": [],\n \"title\": \"Client Delay\",\n \"type\": \"row\"\n },\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"${DS}\"\n },\n \"description\": \"Delay on event processing of a sequencer, compared to the collective\",\n \"fieldConfig\": {\n \"defaults\": {\n \"color\": {\n \"mode\": \"palette-classic\"\n },\n \"custom\": {\n \"axisBorderShow\": false,\n \"axisCenteredZero\": false,\n \"axisColorMode\": \"text\",\n \"axisLabel\": \"\",\n \"axisPlacement\": \"auto\",\n \"barAlignment\": 0,\n \"barWidthFactor\": 0.6,\n \"drawStyle\": \"line\",\n \"fillOpacity\": 0,\n \"gradientMode\": \"none\",\n \"hideFrom\": {\n \"legend\": false,\n \"tooltip\": false,\n \"viz\": false\n },\n \"insertNulls\": false,\n \"lineInterpolation\": \"linear\",\n \"lineWidth\": 1,\n \"pointSize\": 5,\n \"scaleDistribution\": {\n \"log\": 2,\n \"type\": \"log\"\n },\n \"showPoints\": \"auto\",\n \"showValues\": false,\n \"spanNulls\": false,\n \"stacking\": {\n \"group\": \"A\",\n \"mode\": \"none\"\n },\n \"thresholdsStyle\": {\n \"mode\": \"off\"\n }\n },\n \"mappings\": [],\n \"thresholds\": {\n \"mode\": \"absolute\",\n \"steps\": [\n {\n \"color\": \"green\",\n \"value\": 0\n },\n {\n \"color\": \"red\",\n \"value\": 80\n }\n ]\n },\n \"unit\": \"ms\"\n },\n \"overrides\": []\n },\n \"gridPos\": {\n \"h\": 11,\n \"w\": 24,\n \"x\": 0,\n \"y\": 1\n },\n \"id\": 1,\n \"options\": {\n \"legend\": {\n \"calcs\": [\"lastNotNull\"],\n \"displayMode\": \"table\",\n \"placement\": \"right\",\n \"showLegend\": true,\n \"sortBy\": \"Last *\",\n \"sortDesc\": true\n },\n \"tooltip\": {\n \"hideZeros\": false,\n \"mode\": \"multi\",\n \"sort\": \"desc\"\n }\n },\n \"pluginVersion\": \"12.4.0\",\n \"targets\": [\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"${DS}\"\n },\n \"disableTextWrap\": false,\n \"editorMode\": \"code\",\n \"expr\": \"min by (namespace, job) (daml_sequencer_block_delay{namespace=~\\\"$namespace\\\", component=\\\"sequencer\\\",job=~\\\"global-domain-$migration-sequencer\\\"})\",\n \"fullMetaSearch\": false,\n \"includeNullMetadata\": true,\n \"instant\": false,\n \"legendFormat\": \"{{namespace}} migration $migration\",\n \"range\": true,\n \"refId\": \"A\",\n \"useBackend\": false\n }\n ],\n \"title\": \"Sequencer Client Delay\",\n \"type\": \"timeseries\"\n },\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"${DS}\"\n },\n \"description\": \"Catchup speed of the sequencer in the last 5min\",\n \"fieldConfig\": {\n \"defaults\": {\n \"color\": {\n \"mode\": \"palette-classic\"\n },\n \"custom\": {\n \"axisBorderShow\": false,\n \"axisCenteredZero\": false,\n \"axisColorMode\": \"text\",\n \"axisLabel\": \"\",\n \"axisPlacement\": \"auto\",\n \"barAlignment\": 0,\n \"barWidthFactor\": 0.6,\n \"drawStyle\": \"line\",\n \"fillOpacity\": 0,\n \"gradientMode\": \"none\",\n \"hideFrom\": {\n \"legend\": false,\n \"tooltip\": false,\n \"viz\": false\n },\n \"insertNulls\": false,\n \"lineInterpolation\": \"linear\",\n \"lineWidth\": 1,\n \"pointSize\": 5,\n \"scaleDistribution\": {\n \"log\": 2,\n \"type\": \"log\"\n },\n \"showPoints\": \"auto\",\n \"showValues\": false,\n \"spanNulls\": false,\n \"stacking\": {\n \"group\": \"A\",\n \"mode\": \"none\"\n },\n \"thresholdsStyle\": {\n \"mode\": \"off\"\n }\n },\n \"mappings\": [],\n \"thresholds\": {\n \"mode\": \"absolute\",\n \"steps\": [\n {\n \"color\": \"green\",\n \"value\": 0\n },\n {\n \"color\": \"red\",\n \"value\": 80\n }\n ]\n },\n \"unit\": \"ms\"\n },\n \"overrides\": []\n },\n \"gridPos\": {\n \"h\": 11,\n \"w\": 24,\n \"x\": 0,\n \"y\": 12\n },\n \"id\": 18,\n \"options\": {\n \"legend\": {\n \"calcs\": [\"lastNotNull\"],\n \"displayMode\": \"table\",\n \"placement\": \"right\",\n \"showLegend\": true,\n \"sortBy\": \"Last *\",\n \"sortDesc\": true\n },\n \"tooltip\": {\n \"hideZeros\": false,\n \"mode\": \"multi\",\n \"sort\": \"desc\"\n }\n },\n \"pluginVersion\": \"12.4.0\",\n \"targets\": [\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"${DS}\"\n },\n \"disableTextWrap\": false,\n \"editorMode\": \"code\",\n \"exemplar\": false,\n \"expr\": \"max by (namespace, job) (delta(daml_sequencer_block_delay{namespace=~\\\"$namespace\\\", component=\\\"sequencer\\\",job=~\\\"global-domain-$migration-sequencer\\\"}[5m])) / 5\",\n \"fullMetaSearch\": false,\n \"includeNullMetadata\": true,\n \"instant\": false,\n \"legendFormat\": \"processing time {{namespace}} migration $migration / min\",\n \"range\": true,\n \"refId\": \"B\",\n \"useBackend\": false\n }\n ],\n \"title\": \"Sequencer Catchup Speed\",\n \"type\": \"timeseries\"\n },\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"${DS}\"\n },\n \"description\": \"Delay on event processing of the participant, compared to the sequencers it is connected to.\",\n \"fieldConfig\": {\n \"defaults\": {\n \"color\": {\n \"mode\": \"palette-classic\"\n },\n \"custom\": {\n \"axisBorderShow\": false,\n \"axisCenteredZero\": false,\n \"axisColorMode\": \"text\",\n \"axisLabel\": \"\",\n \"axisPlacement\": \"auto\",\n \"barAlignment\": 0,\n \"barWidthFactor\": 0.6,\n \"drawStyle\": \"line\",\n \"fillOpacity\": 0,\n \"gradientMode\": \"none\",\n \"hideFrom\": {\n \"legend\": false,\n \"tooltip\": false,\n \"viz\": false\n },\n \"insertNulls\": false,\n \"lineInterpolation\": \"linear\",\n \"lineWidth\": 1,\n \"pointSize\": 5,\n \"scaleDistribution\": {\n \"log\": 2,\n \"type\": \"log\"\n },\n \"showPoints\": \"auto\",\n \"showValues\": false,\n \"spanNulls\": false,\n \"stacking\": {\n \"group\": \"A\",\n \"mode\": \"none\"\n },\n \"thresholdsStyle\": {\n \"mode\": \"off\"\n }\n },\n \"mappings\": [],\n \"thresholds\": {\n \"mode\": \"absolute\",\n \"steps\": [\n {\n \"color\": \"green\",\n \"value\": 0\n },\n {\n \"color\": \"red\",\n \"value\": 80\n }\n ]\n },\n \"unit\": \"s\"\n },\n \"overrides\": []\n },\n \"gridPos\": {\n \"h\": 8,\n \"w\": 12,\n \"x\": 0,\n \"y\": 23\n },\n \"id\": 3,\n \"options\": {\n \"legend\": {\n \"calcs\": [\"lastNotNull\"],\n \"displayMode\": \"table\",\n \"placement\": \"right\",\n \"showLegend\": true,\n \"sortBy\": \"Last *\",\n \"sortDesc\": true\n },\n \"tooltip\": {\n \"hideZeros\": false,\n \"mode\": \"multi\",\n \"sort\": \"desc\"\n }\n },\n \"pluginVersion\": \"12.4.0\",\n \"targets\": [\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"${DS}\"\n },\n \"disableTextWrap\": false,\n \"editorMode\": \"code\",\n \"expr\": \"max by (namespace) (timestamp(daml_sequencer_client_handler_last_sequencing_time_micros{namespace=~\\\"$namespace\\\",component=\\\"participant\\\"}) - ((daml_sequencer_client_handler_last_sequencing_time_micros{namespace=~\\\"$namespace\\\",component=\\\"participant\\\"} > 0) / 1e6))\",\n \"fullMetaSearch\": false,\n \"includeNullMetadata\": true,\n \"instant\": false,\n \"legendFormat\": \"{{namespace}} {{node}}\",\n \"range\": true,\n \"refId\": \"A\",\n \"useBackend\": false\n }\n ],\n \"title\": \"Participant Client Delay\",\n \"type\": \"timeseries\"\n },\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"${DS}\"\n },\n \"description\": \"Delay on event processing of a mediator, compared to its corresponding sequencer\",\n \"fieldConfig\": {\n \"defaults\": {\n \"color\": {\n \"mode\": \"palette-classic\"\n },\n \"custom\": {\n \"axisBorderShow\": false,\n \"axisCenteredZero\": false,\n \"axisColorMode\": \"text\",\n \"axisLabel\": \"\",\n \"axisPlacement\": \"auto\",\n \"barAlignment\": 0,\n \"barWidthFactor\": 0.6,\n \"drawStyle\": \"line\",\n \"fillOpacity\": 0,\n \"gradientMode\": \"none\",\n \"hideFrom\": {\n \"legend\": false,\n \"tooltip\": false,\n \"viz\": false\n },\n \"insertNulls\": false,\n \"lineInterpolation\": \"linear\",\n \"lineWidth\": 1,\n \"pointSize\": 5,\n \"scaleDistribution\": {\n \"log\": 2,\n \"type\": \"log\"\n },\n \"showPoints\": \"auto\",\n \"showValues\": false,\n \"spanNulls\": false,\n \"stacking\": {\n \"group\": \"A\",\n \"mode\": \"none\"\n },\n \"thresholdsStyle\": {\n \"mode\": \"off\"\n }\n },\n \"mappings\": [],\n \"thresholds\": {\n \"mode\": \"absolute\",\n \"steps\": [\n {\n \"color\": \"green\",\n \"value\": 0\n },\n {\n \"color\": \"red\",\n \"value\": 80\n }\n ]\n },\n \"unit\": \"s\"\n },\n \"overrides\": []\n },\n \"gridPos\": {\n \"h\": 8,\n \"w\": 12,\n \"x\": 12,\n \"y\": 23\n },\n \"id\": 2,\n \"options\": {\n \"legend\": {\n \"calcs\": [\"lastNotNull\"],\n \"displayMode\": \"table\",\n \"placement\": \"right\",\n \"showLegend\": true,\n \"sortBy\": \"Last *\",\n \"sortDesc\": true\n },\n \"tooltip\": {\n \"hideZeros\": false,\n \"mode\": \"multi\",\n \"sort\": \"desc\"\n }\n },\n \"pluginVersion\": \"12.4.0\",\n \"targets\": [\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"${DS}\"\n },\n \"disableTextWrap\": false,\n \"editorMode\": \"code\",\n \"expr\": \"max by (namespace, job) (timestamp(daml_sequencer_client_handler_last_sequencing_time_micros{namespace=~\\\"$namespace\\\",component=\\\"mediator\\\",job=~\\\"global-domain-$migration-mediator\\\"}) - ((daml_sequencer_client_handler_last_sequencing_time_micros{namespace=~\\\"$namespace\\\",component=\\\"mediator\\\",job=~\\\"global-domain-$migration-mediator\\\"} > 0) / 1e6)) \",\n \"fullMetaSearch\": false,\n \"includeNullMetadata\": true,\n \"instant\": false,\n \"legendFormat\": \"{{namespace}}\",\n \"range\": true,\n \"refId\": \"A\",\n \"useBackend\": false\n }\n ],\n \"title\": \"Mediator Client Delay\",\n \"type\": \"timeseries\"\n },\n {\n \"collapsed\": false,\n \"gridPos\": {\n \"h\": 1,\n \"w\": 24,\n \"x\": 0,\n \"y\": 31\n },\n \"id\": 10,\n \"panels\": [],\n \"title\": \"CometBFT\",\n \"type\": \"row\"\n },\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"$DS\"\n },\n \"fieldConfig\": {\n \"defaults\": {\n \"color\": {\n \"mode\": \"palette-classic\"\n },\n \"custom\": {\n \"axisBorderShow\": false,\n \"axisCenteredZero\": false,\n \"axisColorMode\": \"text\",\n \"axisLabel\": \"\",\n \"axisPlacement\": \"auto\",\n \"barAlignment\": 0,\n \"barWidthFactor\": 0.6,\n \"drawStyle\": \"line\",\n \"fillOpacity\": 0,\n \"gradientMode\": \"none\",\n \"hideFrom\": {\n \"legend\": false,\n \"tooltip\": false,\n \"viz\": false\n },\n \"insertNulls\": false,\n \"lineInterpolation\": \"linear\",\n \"lineWidth\": 1,\n \"pointSize\": 5,\n \"scaleDistribution\": {\n \"type\": \"linear\"\n },\n \"showPoints\": \"auto\",\n \"showValues\": false,\n \"spanNulls\": false,\n \"stacking\": {\n \"group\": \"A\",\n \"mode\": \"none\"\n },\n \"thresholdsStyle\": {\n \"mode\": \"off\"\n }\n },\n \"decimals\": 2,\n \"mappings\": [],\n \"thresholds\": {\n \"mode\": \"absolute\",\n \"steps\": [\n {\n \"color\": \"green\",\n \"value\": 0\n },\n {\n \"color\": \"red\",\n \"value\": 80\n }\n ]\n },\n \"unit\": \"cps\"\n },\n \"overrides\": []\n },\n \"gridPos\": {\n \"h\": 7,\n \"w\": 24,\n \"x\": 0,\n \"y\": 32\n },\n \"id\": 11,\n \"options\": {\n \"legend\": {\n \"calcs\": [],\n \"displayMode\": \"list\",\n \"placement\": \"bottom\",\n \"showLegend\": false\n },\n \"tooltip\": {\n \"hideZeros\": false,\n \"mode\": \"multi\",\n \"sort\": \"desc\"\n }\n },\n \"pluginVersion\": \"12.4.0\",\n \"targets\": [\n {\n \"datasource\": {\n \"uid\": \"$DS\"\n },\n \"disableTextWrap\": false,\n \"editorMode\": \"code\",\n \"expr\": \"sum by(namespace, job) (rate(daml_sequencer_block_events_total{namespace=~\\\"$namespace\\\",job=~\\\"global-domain-$migration-sequencer\\\"}[$__rate_interval]))\",\n \"format\": \"time_series\",\n \"fullMetaSearch\": false,\n \"includeNullMetadata\": true,\n \"instant\": false,\n \"interval\": \"\",\n \"intervalFactor\": 1,\n \"legendFormat\": \"{{namespace}} {{job}}\",\n \"range\": true,\n \"refId\": \"A\",\n \"useBackend\": false\n }\n ],\n \"title\": \"Sequencer processing event Rate\",\n \"type\": \"timeseries\"\n },\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"$DS\"\n },\n \"fieldConfig\": {\n \"defaults\": {\n \"color\": {\n \"mode\": \"palette-classic\"\n },\n \"custom\": {\n \"axisBorderShow\": false,\n \"axisCenteredZero\": false,\n \"axisColorMode\": \"text\",\n \"axisLabel\": \"\",\n \"axisPlacement\": \"auto\",\n \"barAlignment\": 0,\n \"barWidthFactor\": 0.6,\n \"drawStyle\": \"line\",\n \"fillOpacity\": 0,\n \"gradientMode\": \"none\",\n \"hideFrom\": {\n \"legend\": false,\n \"tooltip\": false,\n \"viz\": false\n },\n \"insertNulls\": false,\n \"lineInterpolation\": \"linear\",\n \"lineWidth\": 1,\n \"pointSize\": 5,\n \"scaleDistribution\": {\n \"type\": \"linear\"\n },\n \"showPoints\": \"auto\",\n \"showValues\": false,\n \"spanNulls\": false,\n \"stacking\": {\n \"group\": \"A\",\n \"mode\": \"none\"\n },\n \"thresholdsStyle\": {\n \"mode\": \"off\"\n }\n },\n \"decimals\": 2,\n \"mappings\": [],\n \"thresholds\": {\n \"mode\": \"absolute\",\n \"steps\": [\n {\n \"color\": \"green\",\n \"value\": 0\n },\n {\n \"color\": \"red\",\n \"value\": 80\n }\n ]\n },\n \"unit\": \"cps\"\n },\n \"overrides\": []\n },\n \"gridPos\": {\n \"h\": 7,\n \"w\": 24,\n \"x\": 0,\n \"y\": 39\n },\n \"id\": 7,\n \"options\": {\n \"legend\": {\n \"calcs\": [],\n \"displayMode\": \"list\",\n \"placement\": \"bottom\",\n \"showLegend\": false\n },\n \"tooltip\": {\n \"hideZeros\": false,\n \"mode\": \"multi\",\n \"sort\": \"desc\"\n }\n },\n \"pluginVersion\": \"12.4.0\",\n \"targets\": [\n {\n \"datasource\": {\n \"uid\": \"$DS\"\n },\n \"disableTextWrap\": false,\n \"editorMode\": \"code\",\n \"expr\": \"rate((daml_sequencer_block_height{namespace=~\\\"$namespace\\\", job=~\\\"global-domain-$migration-sequencer\\\"} > 0)[$__rate_interval:])\",\n \"format\": \"time_series\",\n \"fullMetaSearch\": false,\n \"includeNullMetadata\": true,\n \"instant\": false,\n \"interval\": \"\",\n \"intervalFactor\": 1,\n \"legendFormat\": \"{{namespace}} {{job}}\",\n \"range\": true,\n \"refId\": \"A\",\n \"useBackend\": false\n }\n ],\n \"title\": \"Sequencer processing Block Rate\",\n \"type\": \"timeseries\"\n },\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"$DS\"\n },\n \"fieldConfig\": {\n \"defaults\": {\n \"color\": {\n \"mode\": \"palette-classic\"\n },\n \"custom\": {\n \"axisBorderShow\": false,\n \"axisCenteredZero\": false,\n \"axisColorMode\": \"text\",\n \"axisLabel\": \"\",\n \"axisPlacement\": \"auto\",\n \"barAlignment\": 0,\n \"barWidthFactor\": 0.6,\n \"drawStyle\": \"line\",\n \"fillOpacity\": 0,\n \"gradientMode\": \"none\",\n \"hideFrom\": {\n \"legend\": false,\n \"tooltip\": false,\n \"viz\": false\n },\n \"insertNulls\": false,\n \"lineInterpolation\": \"linear\",\n \"lineWidth\": 1,\n \"pointSize\": 5,\n \"scaleDistribution\": {\n \"type\": \"linear\"\n },\n \"showPoints\": \"auto\",\n \"showValues\": false,\n \"spanNulls\": false,\n \"stacking\": {\n \"group\": \"A\",\n \"mode\": \"none\"\n },\n \"thresholdsStyle\": {\n \"mode\": \"off\"\n }\n },\n \"decimals\": 2,\n \"mappings\": [],\n \"thresholds\": {\n \"mode\": \"absolute\",\n \"steps\": [\n {\n \"color\": \"green\",\n \"value\": 0\n },\n {\n \"color\": \"red\",\n \"value\": 80\n }\n ]\n },\n \"unit\": \"cps\"\n },\n \"overrides\": []\n },\n \"gridPos\": {\n \"h\": 6,\n \"w\": 24,\n \"x\": 0,\n \"y\": 46\n },\n \"id\": 6,\n \"options\": {\n \"legend\": {\n \"calcs\": [],\n \"displayMode\": \"list\",\n \"placement\": \"bottom\",\n \"showLegend\": false\n },\n \"tooltip\": {\n \"hideZeros\": false,\n \"mode\": \"single\",\n \"sort\": \"none\"\n }\n },\n \"pluginVersion\": \"12.4.0\",\n \"targets\": [\n {\n \"datasource\": {\n \"uid\": \"$DS\"\n },\n \"disableTextWrap\": false,\n \"editorMode\": \"code\",\n \"expr\": \"rate(cometbft_consensus_height{namespace=~\\\"$namespace\\\", job=~\\\"global-domain-$migration-cometbft-cometbft-rpc\\\"}[$__rate_interval])\",\n \"format\": \"time_series\",\n \"fullMetaSearch\": false,\n \"includeNullMetadata\": true,\n \"instant\": false,\n \"interval\": \"\",\n \"intervalFactor\": 1,\n \"legendFormat\": \"{{namespace}}\",\n \"range\": true,\n \"refId\": \"A\",\n \"useBackend\": false\n }\n ],\n \"title\": \"Cometbft producing Block Rate\",\n \"type\": \"timeseries\"\n },\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"fieldConfig\": {\n \"defaults\": {\n \"color\": {\n \"mode\": \"palette-classic\"\n },\n \"custom\": {\n \"axisBorderShow\": false,\n \"axisCenteredZero\": false,\n \"axisColorMode\": \"text\",\n \"axisLabel\": \"\",\n \"axisPlacement\": \"auto\",\n \"barAlignment\": 0,\n \"barWidthFactor\": 0.6,\n \"drawStyle\": \"line\",\n \"fillOpacity\": 0,\n \"gradientMode\": \"none\",\n \"hideFrom\": {\n \"legend\": false,\n \"tooltip\": false,\n \"viz\": false\n },\n \"insertNulls\": false,\n \"lineInterpolation\": \"linear\",\n \"lineWidth\": 1,\n \"pointSize\": 5,\n \"scaleDistribution\": {\n \"type\": \"linear\"\n },\n \"showPoints\": \"auto\",\n \"showValues\": false,\n \"spanNulls\": false,\n \"stacking\": {\n \"group\": \"A\",\n \"mode\": \"none\"\n },\n \"thresholdsStyle\": {\n \"mode\": \"off\"\n }\n },\n \"mappings\": [],\n \"thresholds\": {\n \"mode\": \"absolute\",\n \"steps\": [\n {\n \"color\": \"green\",\n \"value\": 0\n },\n {\n \"color\": \"red\",\n \"value\": 80\n }\n ]\n }\n },\n \"overrides\": []\n },\n \"gridPos\": {\n \"h\": 8,\n \"w\": 12,\n \"x\": 0,\n \"y\": 52\n },\n \"id\": 8,\n \"options\": {\n \"legend\": {\n \"calcs\": [],\n \"displayMode\": \"list\",\n \"placement\": \"bottom\",\n \"showLegend\": true\n },\n \"tooltip\": {\n \"hideZeros\": false,\n \"mode\": \"single\",\n \"sort\": \"none\"\n }\n },\n \"pluginVersion\": \"12.4.0\",\n \"targets\": [\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"disableTextWrap\": false,\n \"editorMode\": \"code\",\n \"expr\": \"daml_sequencer_block_height{namespace=~\\\"$namespace\\\", job=~\\\"global-domain-$migration-sequencer\\\"}\",\n \"fullMetaSearch\": false,\n \"includeNullMetadata\": true,\n \"instant\": false,\n \"legendFormat\": \"{{job}}\",\n \"range\": true,\n \"refId\": \"A\",\n \"useBackend\": false\n }\n ],\n \"title\": \"Sequencer Cometbft height \",\n \"type\": \"timeseries\"\n },\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"${DS}\"\n },\n \"fieldConfig\": {\n \"defaults\": {\n \"color\": {\n \"mode\": \"palette-classic\"\n },\n \"custom\": {\n \"axisBorderShow\": false,\n \"axisCenteredZero\": false,\n \"axisColorMode\": \"text\",\n \"axisLabel\": \"\",\n \"axisPlacement\": \"auto\",\n \"barAlignment\": 0,\n \"barWidthFactor\": 0.6,\n \"drawStyle\": \"line\",\n \"fillOpacity\": 0,\n \"gradientMode\": \"none\",\n \"hideFrom\": {\n \"legend\": false,\n \"tooltip\": false,\n \"viz\": false\n },\n \"insertNulls\": false,\n \"lineInterpolation\": \"linear\",\n \"lineWidth\": 1,\n \"pointSize\": 5,\n \"scaleDistribution\": {\n \"type\": \"linear\"\n },\n \"showPoints\": \"auto\",\n \"showValues\": false,\n \"spanNulls\": false,\n \"stacking\": {\n \"group\": \"A\",\n \"mode\": \"none\"\n },\n \"thresholdsStyle\": {\n \"mode\": \"off\"\n }\n },\n \"mappings\": [],\n \"thresholds\": {\n \"mode\": \"absolute\",\n \"steps\": [\n {\n \"color\": \"green\",\n \"value\": 0\n },\n {\n \"color\": \"red\",\n \"value\": 80\n }\n ]\n }\n },\n \"overrides\": []\n },\n \"gridPos\": {\n \"h\": 8,\n \"w\": 12,\n \"x\": 12,\n \"y\": 52\n },\n \"id\": 4,\n \"options\": {\n \"legend\": {\n \"calcs\": [],\n \"displayMode\": \"list\",\n \"placement\": \"bottom\",\n \"showLegend\": true\n },\n \"tooltip\": {\n \"hideZeros\": false,\n \"mode\": \"single\",\n \"sort\": \"none\"\n }\n },\n \"pluginVersion\": \"12.4.0\",\n \"targets\": [\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"${DS}\"\n },\n \"disableTextWrap\": false,\n \"editorMode\": \"code\",\n \"expr\": \"cometbft_consensus_latest_block_height{namespace=~\\\"$namespace\\\", job=~\\\"global-domain-$migration-cometbft-cometbft-rpc\\\"}\",\n \"fullMetaSearch\": false,\n \"includeNullMetadata\": true,\n \"instant\": false,\n \"legendFormat\": \"{{namespace}}-{{chain_id}}\",\n \"range\": true,\n \"refId\": \"A\",\n \"useBackend\": false\n }\n ],\n \"title\": \"CometBFT Block Height\",\n \"type\": \"timeseries\"\n },\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"fieldConfig\": {\n \"defaults\": {\n \"color\": {\n \"mode\": \"palette-classic\"\n },\n \"custom\": {\n \"axisBorderShow\": false,\n \"axisCenteredZero\": false,\n \"axisColorMode\": \"text\",\n \"axisLabel\": \"\",\n \"axisPlacement\": \"auto\",\n \"barAlignment\": 0,\n \"barWidthFactor\": 0.6,\n \"drawStyle\": \"line\",\n \"fillOpacity\": 0,\n \"gradientMode\": \"none\",\n \"hideFrom\": {\n \"legend\": false,\n \"tooltip\": false,\n \"viz\": false\n },\n \"insertNulls\": false,\n \"lineInterpolation\": \"linear\",\n \"lineWidth\": 1,\n \"pointSize\": 5,\n \"scaleDistribution\": {\n \"type\": \"linear\"\n },\n \"showPoints\": \"auto\",\n \"showValues\": false,\n \"spanNulls\": false,\n \"stacking\": {\n \"group\": \"A\",\n \"mode\": \"none\"\n },\n \"thresholdsStyle\": {\n \"mode\": \"off\"\n }\n },\n \"mappings\": [],\n \"thresholds\": {\n \"mode\": \"absolute\",\n \"steps\": [\n {\n \"color\": \"green\",\n \"value\": 0\n },\n {\n \"color\": \"red\",\n \"value\": 80\n }\n ]\n }\n },\n \"overrides\": []\n },\n \"gridPos\": {\n \"h\": 8,\n \"w\": 12,\n \"x\": 0,\n \"y\": 60\n },\n \"id\": 5,\n \"options\": {\n \"legend\": {\n \"calcs\": [],\n \"displayMode\": \"list\",\n \"placement\": \"bottom\",\n \"showLegend\": true\n },\n \"tooltip\": {\n \"hideZeros\": false,\n \"mode\": \"single\",\n \"sort\": \"none\"\n }\n },\n \"pluginVersion\": \"12.4.0\",\n \"targets\": [\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"disableTextWrap\": false,\n \"editorMode\": \"code\",\n \"expr\": \"cometbft_consensus_block_syncing{namespace=~\\\"$namespace\\\", job=~\\\"global-domain-$migration-cometbft-cometbft-rpc\\\"}\",\n \"fullMetaSearch\": false,\n \"includeNullMetadata\": true,\n \"instant\": false,\n \"legendFormat\": \"{{namespace}}\",\n \"range\": true,\n \"refId\": \"A\",\n \"useBackend\": false\n }\n ],\n \"title\": \"CometBFT Block Syncing\",\n \"type\": \"timeseries\"\n },\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"${DS}\"\n },\n \"fieldConfig\": {\n \"defaults\": {\n \"color\": {\n \"mode\": \"thresholds\"\n },\n \"mappings\": [],\n \"thresholds\": {\n \"mode\": \"absolute\",\n \"steps\": [\n {\n \"color\": \"green\",\n \"value\": 0\n },\n {\n \"color\": \"red\",\n \"value\": 80\n }\n ]\n }\n },\n \"overrides\": []\n },\n \"gridPos\": {\n \"h\": 8,\n \"w\": 12,\n \"x\": 12,\n \"y\": 60\n },\n \"id\": 15,\n \"options\": {\n \"minVizHeight\": 75,\n \"minVizWidth\": 75,\n \"orientation\": \"auto\",\n \"reduceOptions\": {\n \"calcs\": [\"lastNotNull\"],\n \"fields\": \"\",\n \"values\": false\n },\n \"showThresholdLabels\": false,\n \"showThresholdMarkers\": true,\n \"sizing\": \"auto\"\n },\n \"pluginVersion\": \"12.4.0\",\n \"targets\": [\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"${DS}\"\n },\n \"disableTextWrap\": false,\n \"editorMode\": \"builder\",\n \"exemplar\": false,\n \"expr\": \"max by(namespace, migration) (label_replace(cometbft_consensus_latest_block_height{namespace=~\\\"$namespace\\\", job=~\\\"global-domain-$migration-cometbft-cometbft-rpc\\\"}, \\\"migration\\\", \\\"$1\\\", \\\"job\\\", \\\"global-domain-(\\\\\\\\d)-.*\\\")) - max by(namespace, migration) (label_replace(daml_sequencer_block_height{namespace=~\\\"$namespace\\\", job=~\\\"global-domain-$migration-sequencer\\\"}, \\\"migration\\\", \\\"$1\\\", \\\"job\\\", \\\"global-domain-(\\\\\\\\d)-.*\\\"))\",\n \"fullMetaSearch\": false,\n \"includeNullMetadata\": true,\n \"instant\": true,\n \"legendFormat\": \"{{namespace}}, migration={{migration}}\",\n \"range\": false,\n \"refId\": \"A\",\n \"useBackend\": false\n }\n ],\n \"title\": \"CometBFT blocks to process\",\n \"type\": \"gauge\"\n },\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"${DS}\"\n },\n \"fieldConfig\": {\n \"defaults\": {\n \"color\": {\n \"mode\": \"palette-classic\"\n },\n \"custom\": {\n \"axisBorderShow\": false,\n \"axisCenteredZero\": false,\n \"axisColorMode\": \"text\",\n \"axisLabel\": \"\",\n \"axisPlacement\": \"auto\",\n \"barAlignment\": 0,\n \"barWidthFactor\": 0.6,\n \"drawStyle\": \"line\",\n \"fillOpacity\": 0,\n \"gradientMode\": \"none\",\n \"hideFrom\": {\n \"legend\": false,\n \"tooltip\": false,\n \"viz\": false\n },\n \"insertNulls\": false,\n \"lineInterpolation\": \"linear\",\n \"lineWidth\": 1,\n \"pointSize\": 5,\n \"scaleDistribution\": {\n \"type\": \"linear\"\n },\n \"showPoints\": \"auto\",\n \"showValues\": false,\n \"spanNulls\": false,\n \"stacking\": {\n \"group\": \"A\",\n \"mode\": \"none\"\n },\n \"thresholdsStyle\": {\n \"mode\": \"off\"\n }\n },\n \"mappings\": [],\n \"thresholds\": {\n \"mode\": \"absolute\",\n \"steps\": [\n {\n \"color\": \"green\",\n \"value\": 0\n },\n {\n \"color\": \"red\",\n \"value\": 80\n }\n ]\n }\n },\n \"overrides\": []\n },\n \"gridPos\": {\n \"h\": 9,\n \"w\": 24,\n \"x\": 0,\n \"y\": 68\n },\n \"id\": 17,\n \"options\": {\n \"legend\": {\n \"calcs\": [],\n \"displayMode\": \"list\",\n \"placement\": \"bottom\",\n \"showLegend\": true\n },\n \"tooltip\": {\n \"hideZeros\": false,\n \"mode\": \"multi\",\n \"sort\": \"desc\"\n }\n },\n \"pluginVersion\": \"12.4.0\",\n \"targets\": [\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"${DS}\"\n },\n \"disableTextWrap\": false,\n \"editorMode\": \"builder\",\n \"exemplar\": false,\n \"expr\": \"max by(namespace, migration) (label_replace(cometbft_consensus_latest_block_height{namespace=~\\\"$namespace\\\", job=~\\\"global-domain-$migration-cometbft-cometbft-rpc\\\"}, \\\"migration\\\", \\\"$1\\\", \\\"job\\\", \\\"global-domain-(\\\\\\\\d)-.*\\\")) - max by(namespace, migration) (label_replace(daml_sequencer_block_height{namespace=~\\\"$namespace\\\", job=~\\\"global-domain-$migration-sequencer\\\"}, \\\"migration\\\", \\\"$1\\\", \\\"job\\\", \\\"global-domain-(\\\\\\\\d)-.*\\\"))\",\n \"fullMetaSearch\": false,\n \"includeNullMetadata\": true,\n \"instant\": false,\n \"legendFormat\": \"{{namespace}}, migration={{migration}}\",\n \"range\": true,\n \"refId\": \"A\",\n \"useBackend\": false\n }\n ],\n \"title\": \"CometBFT blocks to process\",\n \"type\": \"timeseries\"\n }\n ],\n \"preload\": false,\n \"refresh\": \"30s\",\n \"schemaVersion\": 42,\n \"tags\": [],\n \"templating\": {\n \"list\": [\n {\n \"current\": {\n \"text\": \"Prometheus\",\n \"value\": \"prometheus\"\n },\n \"includeAll\": false,\n \"name\": \"DS\",\n \"options\": [],\n \"query\": \"prometheus\",\n \"refresh\": 1,\n \"regex\": \"\",\n \"type\": \"datasource\"\n },\n {\n \"current\": {\n \"text\": [\"sv-1\"],\n \"value\": [\"sv-1\"]\n },\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"definition\": \"label_values(daml_sequencer_client_handler_delay,namespace)\",\n \"includeAll\": true,\n \"label\": \"Namespace\",\n \"multi\": true,\n \"name\": \"namespace\",\n \"options\": [],\n \"query\": {\n \"query\": \"label_values(daml_sequencer_client_handler_delay,namespace)\",\n \"refId\": \"PrometheusVariableQueryEditor-VariableQuery\"\n },\n \"refresh\": 1,\n \"regex\": \"\",\n \"regexApplyTo\": \"value\",\n \"type\": \"query\"\n },\n {\n \"current\": {\n \"text\": \"All\",\n \"value\": [\n \"$__all\"\n ]\n },\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"definition\": \"label_values(daml_sequencer_client_handler_delay{namespace=~\\\"$namespace\\\"},job)\",\n \"includeAll\": true,\n \"multi\": true,\n \"name\": \"migration\",\n \"options\": [],\n \"query\": {\n \"qryType\": 1,\n \"query\": \"label_values(daml_sequencer_client_handler_delay{namespace=~\\\"$namespace\\\"},job)\",\n \"refId\": \"PrometheusVariableQueryEditor-VariableQuery\"\n },\n \"refresh\": 2,\n \"regex\": \"/global-domain-(?\\\\d+)-sequencer/g\",\n \"regexApplyTo\": \"value\",\n \"sort\": 1,\n \"type\": \"query\"\n }\n ]\n },\n \"time\": {\n \"from\": \"now-1h\",\n \"to\": \"now\"\n },\n \"timepicker\": {},\n \"timezone\": \"\",\n \"title\": \"Global Domain Catchup\",\n \"uid\": \"ca9df344-c699-4efe-83c2-5fb2639d96d9\",\n \"version\": 2,\n \"weekStart\": \"\"\n}\n", "cometbft-network-status.json": "{\n \"annotations\": {\n \"list\": [\n {\n \"builtIn\": 1,\n \"datasource\": {\n \"type\": \"datasource\",\n \"uid\": \"grafana\"\n },\n \"enable\": true,\n \"hide\": true,\n \"iconColor\": \"rgba(0, 211, 255, 1)\",\n \"name\": \"Annotations & Alerts\",\n \"target\": {\n \"limit\": 100,\n \"matchAny\": false,\n \"tags\": [],\n \"type\": \"dashboard\"\n },\n \"type\": \"dashboard\"\n }\n ]\n },\n \"description\": \"Health status of the CometBFT P2P Network as measured by incoming / outgoing \",\n \"editable\": true,\n \"fiscalYearStartMonth\": 0,\n \"graphTooltip\": 0,\n \"id\": 1283,\n \"links\": [],\n \"liveNow\": false,\n \"panels\": [\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"$DS\"\n },\n \"description\": \"Total peer-wise rate of bytes sent or received over all channels over the P2P network\",\n \"fieldConfig\": {\n \"defaults\": {\n \"color\": {\n \"mode\": \"thresholds\"\n },\n \"fieldMinMax\": false,\n \"mappings\": [],\n \"min\": 0,\n \"thresholds\": {\n \"mode\": \"percentage\",\n \"steps\": [\n {\n \"color\": \"red\",\n \"value\": null\n },\n {\n \"color\": \"#EAB839\",\n \"value\": 1\n },\n {\n \"color\": \"green\",\n \"value\": 10\n }\n ]\n },\n \"unit\": \"KBs\"\n },\n \"overrides\": []\n },\n \"gridPos\": {\n \"h\": 13,\n \"w\": 24,\n \"x\": 0,\n \"y\": 0\n },\n \"id\": 66,\n \"options\": {\n \"minVizHeight\": 75,\n \"minVizWidth\": 75,\n \"orientation\": \"auto\",\n \"reduceOptions\": {\n \"calcs\": [\n \"last\"\n ],\n \"fields\": \"\",\n \"values\": false\n },\n \"showThresholdLabels\": false,\n \"showThresholdMarkers\": true,\n \"sizing\": \"auto\"\n },\n \"pluginVersion\": \"11.1.0\",\n \"targets\": [\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"editorMode\": \"code\",\n \"expr\": \"sum by (peer_id) (rate(cometbft_p2p_peer_receive_bytes_total{namespace=\\\"$namespace\\\", chain_id=\\\"$chain_id\\\", job=\\\"$instance\\\"}[$__rate_interval]) / 1000) + sum by (peer_id) (rate(cometbft_p2p_peer_send_bytes_total{namespace=\\\"$namespace\\\", chain_id=\\\"$chain_id\\\", job=\\\"$instance\\\"}[$__rate_interval]) / 1000)\",\n \"instant\": false,\n \"legendFormat\": \"__auto\",\n \"range\": true,\n \"refId\": \"A\"\n }\n ],\n \"title\": \"Total Network Throughput (current)\",\n \"type\": \"gauge\"\n },\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"$DS\"\n },\n \"description\": \"Peer-wise rate of bytes received over all channels over the P2P network\",\n \"fieldConfig\": {\n \"defaults\": {\n \"color\": {\n \"mode\": \"palette-classic\"\n },\n \"custom\": {\n \"axisBorderShow\": false,\n \"axisCenteredZero\": false,\n \"axisColorMode\": \"text\",\n \"axisLabel\": \"\",\n \"axisPlacement\": \"auto\",\n \"barAlignment\": 0,\n \"drawStyle\": \"line\",\n \"fillOpacity\": 0,\n \"gradientMode\": \"none\",\n \"hideFrom\": {\n \"legend\": false,\n \"tooltip\": false,\n \"viz\": false\n },\n \"insertNulls\": false,\n \"lineInterpolation\": \"linear\",\n \"lineWidth\": 1,\n \"pointSize\": 5,\n \"scaleDistribution\": {\n \"type\": \"linear\"\n },\n \"showPoints\": \"never\",\n \"spanNulls\": false,\n \"stacking\": {\n \"group\": \"A\",\n \"mode\": \"none\"\n },\n \"thresholdsStyle\": {\n \"mode\": \"dashed\"\n }\n },\n \"fieldMinMax\": false,\n \"links\": [],\n \"mappings\": [],\n \"thresholds\": {\n \"mode\": \"absolute\",\n \"steps\": [\n {\n \"color\": \"green\",\n \"value\": null\n },\n {\n \"color\": \"red\",\n \"value\": 2000\n }\n ]\n },\n \"unit\": \"binBps\"\n },\n \"overrides\": []\n },\n \"gridPos\": {\n \"h\": 12,\n \"w\": 24,\n \"x\": 0,\n \"y\": 13\n },\n \"id\": 59,\n \"options\": {\n \"legend\": {\n \"calcs\": [],\n \"displayMode\": \"table\",\n \"placement\": \"right\",\n \"showLegend\": true\n },\n \"tooltip\": {\n \"mode\": \"multi\",\n \"sort\": \"asc\"\n }\n },\n \"pluginVersion\": \"10.1.5\",\n \"targets\": [\n {\n \"datasource\": {\n \"uid\": \"$DS\"\n },\n \"editorMode\": \"code\",\n \"expr\": \"sum by (peer_id) (rate(cometbft_p2p_peer_receive_bytes_total{namespace=\\\"$namespace\\\", chain_id=\\\"$chain_id\\\", job=\\\"$instance\\\"}[$__rate_interval]))\",\n \"format\": \"time_series\",\n \"intervalFactor\": 1,\n \"legendFormat\": \"{{peer_id}}\",\n \"range\": true,\n \"refId\": \"A\"\n }\n ],\n \"title\": \"Incoming Network Throughput\",\n \"type\": \"timeseries\"\n },\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"$DS\"\n },\n \"description\": \"Peer-wise rate of bytes sent over all channels over the P2P network\",\n \"fieldConfig\": {\n \"defaults\": {\n \"color\": {\n \"mode\": \"palette-classic\"\n },\n \"custom\": {\n \"axisBorderShow\": false,\n \"axisCenteredZero\": false,\n \"axisColorMode\": \"text\",\n \"axisLabel\": \"\",\n \"axisPlacement\": \"auto\",\n \"barAlignment\": 0,\n \"drawStyle\": \"line\",\n \"fillOpacity\": 0,\n \"gradientMode\": \"none\",\n \"hideFrom\": {\n \"legend\": false,\n \"tooltip\": false,\n \"viz\": false\n },\n \"insertNulls\": false,\n \"lineInterpolation\": \"linear\",\n \"lineWidth\": 1,\n \"pointSize\": 5,\n \"scaleDistribution\": {\n \"type\": \"linear\"\n },\n \"showPoints\": \"never\",\n \"spanNulls\": false,\n \"stacking\": {\n \"group\": \"A\",\n \"mode\": \"none\"\n },\n \"thresholdsStyle\": {\n \"mode\": \"dashed\"\n }\n },\n \"fieldMinMax\": false,\n \"links\": [],\n \"mappings\": [],\n \"thresholds\": {\n \"mode\": \"absolute\",\n \"steps\": [\n {\n \"color\": \"green\",\n \"value\": null\n },\n {\n \"color\": \"red\",\n \"value\": 2000\n }\n ]\n },\n \"unit\": \"binBps\"\n },\n \"overrides\": []\n },\n \"gridPos\": {\n \"h\": 12,\n \"w\": 24,\n \"x\": 0,\n \"y\": 25\n },\n \"id\": 65,\n \"options\": {\n \"legend\": {\n \"calcs\": [],\n \"displayMode\": \"table\",\n \"placement\": \"right\",\n \"showLegend\": true\n },\n \"tooltip\": {\n \"mode\": \"multi\",\n \"sort\": \"asc\"\n }\n },\n \"pluginVersion\": \"10.1.5\",\n \"targets\": [\n {\n \"datasource\": {\n \"uid\": \"$DS\"\n },\n \"editorMode\": \"code\",\n \"expr\": \"sum by (peer_id) (rate(cometbft_p2p_peer_send_bytes_total{namespace=\\\"$namespace\\\", chain_id=\\\"$chain_id\\\", job=\\\"$instance\\\"}[$__rate_interval]))\",\n \"format\": \"time_series\",\n \"intervalFactor\": 1,\n \"legendFormat\": \"{{peer_id}}\",\n \"range\": true,\n \"refId\": \"A\"\n }\n ],\n \"title\": \"Outgoing Network Throughput\",\n \"type\": \"timeseries\"\n }\n ],\n \"refresh\": \"\",\n \"schemaVersion\": 39,\n \"tags\": [\n \"Blockchain\"\n ],\n \"templating\": {\n \"list\": [\n {\n \"current\": {\n \"selected\": false,\n \"text\": \"default\",\n \"value\": \"default\"\n },\n \"hide\": 0,\n \"includeAll\": false,\n \"label\": \"Datasource\",\n \"multi\": false,\n \"name\": \"DS\",\n \"options\": [],\n \"query\": \"prometheus\",\n \"queryValue\": \"\",\n \"refresh\": 1,\n \"regex\": \"\",\n \"skipUrlSync\": false,\n \"type\": \"datasource\"\n },\n {\n \"current\": {\n \"selected\": false,\n \"text\": \"sv-1\",\n \"value\": \"sv-1\"\n },\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"$DS\"\n },\n \"definition\": \"label_values(cometbft_p2p_message_receive_bytes_total,namespace)\",\n \"hide\": 0,\n \"includeAll\": false,\n \"label\": \"namespace\",\n \"multi\": false,\n \"name\": \"namespace\",\n \"options\": [],\n \"query\": {\n \"qryType\": 1,\n \"query\": \"label_values(cometbft_p2p_message_receive_bytes_total,namespace)\",\n \"refId\": \"PrometheusVariableQueryEditor-VariableQuery\"\n },\n \"refresh\": 2,\n \"regex\": \"\",\n \"skipUrlSync\": false,\n \"sort\": 1,\n \"tagValuesQuery\": \"\",\n \"tagsQuery\": \"\",\n \"type\": \"query\",\n \"useTags\": false\n },\n {\n \"current\": {\n \"selected\": true,\n \"text\": \"cidaily-0-0.2.0-snapshot.20240725.6534.0.v112d3fc6\",\n \"value\": \"cidaily-0-0.2.0-snapshot.20240725.6534.0.v112d3fc6\"\n },\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"$DS\"\n },\n \"definition\": \"label_values(cometbft_p2p_message_receive_bytes_total,chain_id)\",\n \"hide\": 0,\n \"includeAll\": false,\n \"label\": \"Chain ID\",\n \"multi\": false,\n \"name\": \"chain_id\",\n \"options\": [],\n \"query\": {\n \"qryType\": 1,\n \"query\": \"label_values(cometbft_p2p_message_receive_bytes_total,chain_id)\",\n \"refId\": \"PrometheusVariableQueryEditor-VariableQuery\"\n },\n \"refresh\": 1,\n \"regex\": \"\",\n \"skipUrlSync\": false,\n \"sort\": 0,\n \"tagValuesQuery\": \"\",\n \"tagsQuery\": \"\",\n \"type\": \"query\",\n \"useTags\": false\n },\n {\n \"allValue\": \"\",\n \"current\": {\n \"selected\": false,\n \"text\": \"global-domain-0-cometbft-cometbft-rpc\",\n \"value\": \"global-domain-0-cometbft-cometbft-rpc\"\n },\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"$DS\"\n },\n \"definition\": \"label_values(cometbft_p2p_message_receive_bytes_total{chain_id=\\\"$chain_id\\\"},job)\",\n \"hide\": 0,\n \"includeAll\": false,\n \"label\": \"Instance\",\n \"multi\": false,\n \"name\": \"instance\",\n \"options\": [],\n \"query\": {\n \"qryType\": 1,\n \"query\": \"label_values(cometbft_p2p_message_receive_bytes_total{chain_id=\\\"$chain_id\\\"},job)\",\n \"refId\": \"PrometheusVariableQueryEditor-VariableQuery\"\n },\n \"refresh\": 1,\n \"regex\": \"\",\n \"skipUrlSync\": false,\n \"sort\": 5,\n \"tagValuesQuery\": \"\",\n \"tagsQuery\": \"\",\n \"type\": \"query\",\n \"useTags\": false\n }\n ]\n },\n \"time\": {\n \"from\": \"now-12h\",\n \"to\": \"now\"\n },\n \"timepicker\": {\n \"refresh_intervals\": [\n \"5s\",\n \"10s\",\n \"30s\",\n \"1m\",\n \"5m\",\n \"15m\",\n \"30m\",\n \"1h\",\n \"2h\",\n \"1d\"\n ],\n \"time_options\": [\n \"5m\",\n \"15m\",\n \"1h\",\n \"6h\",\n \"12h\",\n \"24h\",\n \"2d\",\n \"7d\",\n \"30d\"\n ]\n },\n \"timezone\": \"\",\n \"title\": \"CometBFT Network Status\",\n \"uid\": \"ddsuu1wnxwetcd\",\n \"version\": 7,\n \"weekStart\": \"\"\n}\n", diff --git a/cluster/pulumi/observability/grafana-dashboards/canton-network/app-rewards.json b/cluster/pulumi/observability/grafana-dashboards/canton-network/app-rewards.json new file mode 100644 index 0000000000..e45e507297 --- /dev/null +++ b/cluster/pulumi/observability/grafana-dashboards/canton-network/app-rewards.json @@ -0,0 +1,1092 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Traffic based app rewards (CIP-104)", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 7, + "panels": [], + "title": "Store lag", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "The time difference between the last ingested record time and the current wallclock time.\n\nNote that the last ingested record time metric only updates when the store ingests a new transaction so if there is no activity, the last ingested record time will not advance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 11, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.4.0", + "targets": [ + { + "editorMode": "code", + "expr": "(timestamp(splice_store_last_ingested_record_time_ms{namespace=~\"$namespace\",store_name=~\"DbScanRewardsReferenceStore\"}) - (splice_store_last_ingested_record_time_ms{namespace=~\"$namespace\",store_name=~\"DbScanRewardsReferenceStore\"} > 0) / 1e3)", + "legendFormat": "{{namespace}}", + "range": true, + "refId": "A" + } + ], + "title": "DbScanRewardsReferenceStore lag", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "The time difference between the last ingested verdict record time and the current wall clock time.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 10, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.4.0", + "targets": [ + { + "disableTextWrap": false, + "editorMode": "code", + "expr": "(timestamp(splice_scan_verdict_ingestion_last_record_time_us{namespace=~\"$namespace\"}) - (splice_scan_verdict_ingestion_last_record_time_us{namespace=~\"$namespace\"} > 0) / 1e6)", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "{{namespace}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Verdict ingestion lag", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "The time difference between the last ingested record time and the current wallclock time.\n\nNote that the last ingested record time metric only updates when the store ingests a new transaction so if there is no activity, the last ingested record time will not advance. For a party performing reward collection, e.g., the validator operator party you expect at least one transaction every round so the lag should not go above 20min.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 9 + }, + "id": 12, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.4.0", + "targets": [ + { + "editorMode": "code", + "expr": "(timestamp(splice_store_last_ingested_record_time_ms{namespace=~\"$namespace\",store_name=~\"DbSvDsoStore\"}) - (splice_store_last_ingested_record_time_ms{namespace=~\"$namespace\",store_name=~\"DbSvDsoStore\"} > 0) / 1e3)", + "legendFormat": "{{namespace}}", + "range": true, + "refId": "A" + } + ], + "title": "DbSvDsoStore lag", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 17 + }, + "id": 13, + "panels": [], + "title": "RewardComputationTrigger (calculate reward merkle tree)", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 18 + }, + "id": 14, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "rate(splice_trigger_completed_total{trigger_name=~\"RewardComputationTrigger\",outcome=~\"success|failure\", namespace=~\"$namespace\"}[$__rate_interval])", + "legendFormat": "{{namespace}} {{outcome}}", + "range": true, + "refId": "A" + } + ], + "title": "Trigger runs", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Number of activity records in the latest computed round", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 10000 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 4, + "x": 12, + "y": 18 + }, + "id": 16, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.4.0", + "targets": [ + { + "editorMode": "code", + "expr": "quantile(0.5, splice_scan_reward_computation_activity_records_count{namespace=~\"$namespace\"})", + "legendFormat": "{{namespace}}", + "range": true, + "refId": "A" + } + ], + "title": "Activity records", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Number of parties with activity in the latest computed round", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 1000 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 4, + "x": 16, + "y": 18 + }, + "id": 18, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.4.0", + "targets": [ + { + "editorMode": "code", + "expr": "quantile(0.5, splice_scan_reward_computation_active_parties_count{namespace=~\"$namespace\"})", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Active parties", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Number of parties with rewards in the latest computed round", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 1000 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 4, + "x": 20, + "y": 18 + }, + "id": 17, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.4.0", + "targets": [ + { + "editorMode": "code", + "expr": "quantile(0.5, splice_scan_reward_computation_rewarded_parties_count{namespace=~\"$namespace\"})", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Rewarded parties", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 26 + }, + "id": 3, + "panels": [], + "title": "CalculateRewardsTrigger (voting on root hash)", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 27 + }, + "id": 1, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "rate(splice_trigger_completed_total{trigger_name=~\"ProcessRewards(DryRun)?Trigger\",outcome=~\"success|failure\", namespace=~\"$namespace\"}[$__rate_interval])", + "legendFormat": "{{namespace}} {{outcome}}", + "range": true, + "refId": "A" + } + ], + "title": "Trigger runs", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "the time it took between the closing of a round, and this SV's confirmation for the CalculateRewardsV2 contract's processing", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 27 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.4.0", + "targets": [ + { + "editorMode": "code", + "exemplar": false, + "expr": "histogram_avg(rate(splice_calculate_rewards_v2_processing_delay_duration_seconds{namespace=~\"$namespace\"}[$__rate_interval]))", + "instant": true, + "legendFormat": "{{namespace}} {{dryRun}}", + "range": true, + "refId": "A" + } + ], + "title": "Latency", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 35 + }, + "id": 5, + "panels": [], + "title": "ProcessRewardsTrigger (creating minting allowances)", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 36 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "rate(splice_trigger_completed_total{trigger_name=~\"ProcessRewards(DryRun)?Trigger\",outcome=~\"success|failure\", namespace=~\"$namespace\"}[$__rate_interval])", + "legendFormat": "{{namespace}} {{trigger_name}} {{outcome}}", + "range": true, + "refId": "A" + } + ], + "title": "Trigger runs", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "The time it took between the closing of a round, and this SV's processing of a ProcessRewardsV2 contract for that round", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 36 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.4.0", + "targets": [ + { + "editorMode": "code", + "expr": "histogram_avg(rate(splice_process_rewards_v2_processing_delay_duration_seconds{namespace=~\"$namespace\"}[$__rate_interval]))", + "legendFormat": "{{namespace}} {{dryRun}}", + "range": true, + "refId": "A" + } + ], + "title": "Latency", + "type": "timeseries" + } + ], + "preload": false, + "schemaVersion": 42, + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "sv-7", + "value": "sv-7" + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(splice_calculate_rewards_v2_processing_delay_duration_seconds,namespace)", + "includeAll": true, + "name": "namespace", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(splice_calculate_rewards_v2_processing_delay_duration_seconds,namespace)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "regexApplyTo": "value", + "type": "query" + } + ] + }, + "time": { + "from": "now-12h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Traffic-based app rewards (CIP-104)", + "uid": "cnx7gsn", + "version": 8, + "weekStart": "" +}