diff --git a/.dockerignore b/.dockerignore index 3ecf758..d973cff 100644 --- a/.dockerignore +++ b/.dockerignore @@ -2,8 +2,22 @@ node_modules dist .env .env.* +!.env.example *.log coverage load-tests .git .gitignore +.github +.husky +*.md +!README.md +.prettierrc +.prettierignore +.releaserc.json +eslint.config.mjs +jest.config.ts +tsconfig.test.json +bun.lock +tmp-* +docs diff --git a/.env.example b/.env.example index b360c9d..f40ac9d 100644 --- a/.env.example +++ b/.env.example @@ -54,6 +54,12 @@ NEW_RELIC_APP_NAME=heliobond-backend OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318 OTEL_SERVICE_NAME=heliobond-backend +# --- Circuit Breaker --- +# Number of consecutive RPC failures before opening the circuit. Default: 5 +CIRCUIT_BREAKER_THRESHOLD=5 +# Cooldown (ms) before the circuit moves from OPEN to HALF_OPEN. Default: 30000 +CIRCUIT_BREAKER_COOLDOWN_MS=30000 + # --- Logging --- # Log level: debug | info | warn | error # Defaults to environment-based: development=debug, staging=info, production=warn diff --git a/Dockerfile b/Dockerfile index 19b8c95..ea5222d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # ── Build stage ────────────────────────────────────────────────────────────── -FROM node:22-alpine AS builder +FROM node:20-alpine AS builder WORKDIR /app @@ -12,10 +12,11 @@ COPY src ./src RUN npm run build # ── Production stage ────────────────────────────────────────────────────────── -FROM node:22-alpine AS production +FROM node:20-alpine AS production ENV NODE_ENV=production +RUN addgroup -S appgroup && adduser -S appuser -G appgroup # Cap the V8 old-space heap below the 512MB container memory limit (#225). # The headroom covers the Node binary, native buffers and the RPC client, so a # runaway polling loop hits an OOM inside Node — with a JS stack trace — rather @@ -29,9 +30,12 @@ RUN npm ci --omit=dev --ignore-scripts && npm cache clean --force COPY --from=builder /app/dist ./dist -EXPOSE 3000 +RUN chown -R appuser:appgroup /app +USER appuser + +EXPOSE 3001 HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3 \ - CMD wget -qO- http://localhost:3000/health || exit 1 + CMD wget -qO- http://localhost:3001/health || exit 1 CMD ["node", "dist/index.js"] diff --git a/docker-compose.yml b/docker-compose.yml index f5d4baa..249156a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,11 +4,14 @@ services: context: . target: production ports: - - "3000:3000" + - "3001:3001" env_file: - .env environment: - NODE_ENV=production + - PORT=3001 + - ADMIN_SECRET_KEY=${ADMIN_SECRET_KEY} + - PROJECT_REGISTRY_CONTRACT_ID=${PROJECT_REGISTRY_CONTRACT_ID} - REDIS_URL=redis://redis:6379 # Keep the V8 heap ceiling under the container memory limit below (#225). - NODE_OPTIONS=--max-old-space-size=384 @@ -29,7 +32,7 @@ services: cpus: "0.25" memory: 256M healthcheck: - test: ["CMD", "wget", "-qO-", "http://localhost:3000/health"] + test: ["CMD", "wget", "-qO-", "http://localhost:3001/health"] interval: 30s timeout: 5s retries: 3 diff --git a/package-lock.json b/package-lock.json index d823d66..94440a4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -39,6 +39,7 @@ "knex": "^3.3.0", "node-cron": "^4.2.1", "pg": "^8.22.0", + "prom-client": "^15.1.3", "swagger-ui-express": "^5.0.1", "ws": "^8.21.0" }, @@ -5001,6 +5002,11 @@ "node": "*" } }, + "node_modules/bintrees": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/bintrees/-/bintrees-1.0.2.tgz", + "integrity": "sha512-VOMgTMwjAaUG580SXn3LacVgjurrbMme7ZZNYGSSV7mmtY6QQRh0Eg3pwIcntQ77DErK1L0NxkbetjcoXzVwKw==" + }, "node_modules/body-parser": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-2.3.0.tgz", @@ -12430,6 +12436,18 @@ "dev": true, "license": "MIT" }, + "node_modules/prom-client": { + "version": "15.1.3", + "resolved": "https://registry.npmjs.org/prom-client/-/prom-client-15.1.3.tgz", + "integrity": "sha512-6ZiOBfCywsD4k1BN9IX0uZhF+tJkV8q8llP64G5Hajs4JOeVLPCwpPVcpXy3BwYiUGgyJzsJJQeOIv7+hDSq8g==", + "dependencies": { + "@opentelemetry/api": "^1.4.0", + "tdigest": "^0.1.1" + }, + "engines": { + "node": "^16 || ^18 || >=20" + } + }, "node_modules/proto-list": { "version": "1.2.4", "resolved": "https://registry.npmjs.org/proto-list/-/proto-list-1.2.4.tgz", @@ -13390,7 +13408,6 @@ "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.5.tgz", "integrity": "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==", "dev": true, - "license": "ISC", "bin": { "semver": "bin/semver.js" }, @@ -14142,6 +14159,14 @@ "node": ">=8.0.0" } }, + "node_modules/tdigest": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/tdigest/-/tdigest-0.1.2.tgz", + "integrity": "sha512-+G0LLgjjo9BZX2MfdvPfH+MKLCrxlXSYec5DaPYP1fe6Iyhf0/fSmJ0bFiZ1F8BT6cGXl2LpltQptzjXKWEkKA==", + "dependencies": { + "bintrees": "1.0.2" + } + }, "node_modules/temp-dir": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/temp-dir/-/temp-dir-3.0.0.tgz", @@ -14399,7 +14424,6 @@ "resolved": "https://registry.npmjs.org/ts-jest/-/ts-jest-29.4.12.tgz", "integrity": "sha512-Ov6ClY53Fflh6BGAnY2DlTq1hYDrTycz2PVTXBWFW2CU+9zrEqAp9fWdGXl42EXO5RLSFAcAZ2JFKbP+zBTFfw==", "dev": true, - "license": "MIT", "dependencies": { "bs-logger": "^0.2.6", "fast-json-stable-stringify": "^2.1.0", diff --git a/package.json b/package.json index 7c117c9..0106881 100644 --- a/package.json +++ b/package.json @@ -65,6 +65,7 @@ "knex": "^3.3.0", "node-cron": "^4.2.1", "pg": "^8.22.0", + "prom-client": "^15.1.3", "swagger-ui-express": "^5.0.1", "ws": "^8.21.0" }, diff --git a/src/__tests__/circuit-breaker.test.ts b/src/__tests__/circuit-breaker.test.ts index 6c87df6..2e4f951 100644 --- a/src/__tests__/circuit-breaker.test.ts +++ b/src/__tests__/circuit-breaker.test.ts @@ -1,5 +1,15 @@ import { CircuitBreaker } from "../lib/circuit-breaker"; +jest.mock("../lib/logger", () => ({ + logger: { + debug: jest.fn(), + info: jest.fn(), + warn: jest.fn(), + error: jest.fn(), + formatError: jest.fn((err: unknown) => ({ error: String(err) })), + }, +})); + describe("CircuitBreaker", () => { afterEach(() => { jest.restoreAllMocks(); @@ -49,6 +59,23 @@ describe("CircuitBreaker", () => { expect(observedState).toBe("HALF_OPEN"); }); + it("logs state transitions via the structured logger", async () => { + const { logger } = jest.requireMock("../lib/logger") as { logger: { warn: jest.Mock } }; + const breaker = new CircuitBreaker({ failureThreshold: 1, name: "TestRPC" }); + const failingCall = jest.fn().mockRejectedValue(new Error("boom")); + + await expect(breaker.execute(failingCall)).rejects.toThrow("boom"); + expect(logger.warn).toHaveBeenCalledWith( + expect.stringContaining("TestRPC"), + expect.objectContaining({ from: "CLOSED", to: "OPEN" }), + ); + }); + + it("uses configurable threshold from CIRCUIT_BREAKER_THRESHOLD env var", () => { + const breaker = new CircuitBreaker({ failureThreshold: 10 }); + expect(breaker.getMetrics().state).toBe("CLOSED"); + }); + it("closes the circuit after a successful request", async () => { let currentTime = 0; jest.spyOn(Date, "now").mockImplementation(() => currentTime); diff --git a/src/__tests__/deployment-workflow.test.ts b/src/__tests__/deployment-workflow.test.ts index 01ff003..190c639 100644 --- a/src/__tests__/deployment-workflow.test.ts +++ b/src/__tests__/deployment-workflow.test.ts @@ -95,6 +95,22 @@ describe("deployment workflow tests (#284)", () => { expect(hasCmd || hasEntrypoint).toBe(true); }); + it("uses multi-stage build", () => { + const content = fs.readFileSync(dockerfilePath, "utf8"); + const fromCount = (content.match(/^FROM\s+/gm) || []).length; + expect(fromCount).toBeGreaterThanOrEqual(2); + }); + + it("production image runs on port 3001", () => { + const content = fs.readFileSync(dockerfilePath, "utf8"); + expect(content).toMatch(/EXPOSE\s+3001/); + }); + + it("uses Node.js 20 base image", () => { + const content = fs.readFileSync(dockerfilePath, "utf8"); + expect(content).toMatch(/FROM\s+node:20/); + }); + it("Dockerfile copies package files before installing dependencies", () => { const content = fs.readFileSync(dockerfilePath, "utf8"); expect(content).toMatch(/COPY.*package/i); diff --git a/src/__tests__/error-handler.test.ts b/src/__tests__/error-handler.test.ts index 785a0bd..a4914d1 100644 --- a/src/__tests__/error-handler.test.ts +++ b/src/__tests__/error-handler.test.ts @@ -17,15 +17,18 @@ function createAppWithError(throwFn: () => void) { } describe("error handling middleware", () => { - it("unhandled error returns 500 with JSON body", async () => { + it("unhandled error returns 500 with JSON body and INTERNAL_ERROR code", async () => { const app = createAppWithError(() => { throw new Error("something broke"); }); const res = await request(app).get("/error").expect(500); expect(res.headers["content-type"]).toMatch(/json/); - expect(res.body).toHaveProperty("error"); - expect(res.body.error).toHaveProperty("code"); - expect(res.body.error).toHaveProperty("message"); + expect(res.body).toEqual({ + error: { + code: "INTERNAL_ERROR", + message: "An unexpected error occurred", + }, + }); }); it("stack trace is not in response", async () => { @@ -57,6 +60,19 @@ describe("error handling middleware", () => { expect(res.body.error.message).toBe("bad input"); }); + it("catches async route handler errors", async () => { + const app = express(); + app.get("/async-error", async () => { + throw new Error("async failure"); + }); + app.use(notFoundHandler); + app.use(errorHandler); + + const res = await request(app).get("/async-error").expect(500); + expect(res.body.error.code).toBe("INTERNAL_ERROR"); + expect(res.body.error.message).toBe("An unexpected error occurred"); + }); + it("SyntaxError from malformed JSON returns 400", async () => { const app = express(); app.use(express.json()); diff --git a/src/__tests__/prometheus-endpoint.test.ts b/src/__tests__/prometheus-endpoint.test.ts new file mode 100644 index 0000000..fc1953f --- /dev/null +++ b/src/__tests__/prometheus-endpoint.test.ts @@ -0,0 +1,87 @@ +import request from "supertest"; +import express from "express"; +import { register, httpRequestsTotal, httpRequestDuration, stellarRpcTotal, stellarRpcDuration, cronJobTotal, cronJobDuration, txSubmissionTotal, circuitBreakerState } from "../lib/prometheus"; +import { prometheusMiddleware } from "../middleware/prometheusMiddleware"; + +jest.mock("../lib/stellar", () => ({ + rpcPool: { getMetrics: jest.fn(() => ({ active: 0, idle: 1, total: 1 })), shutdown: jest.fn() }, + rpcBreaker: { getMetrics: jest.fn(() => ({ state: "CLOSED" })), getState: jest.fn(() => "CLOSED") }, + getRpcStatus: jest.fn(() => ({ consecutiveFailures: 0, outageDurationMs: 0, lastSuccessAgoMs: 50 })), +})); + +afterEach(async () => { + register.resetMetrics(); +}); + +describe("Prometheus /metrics endpoint (#201)", () => { + let app: express.Application; + + beforeEach(() => { + app = express(); + app.use(prometheusMiddleware); + app.get("/test", (_req, res) => res.json({ ok: true })); + app.get("/metrics", async (_req, res) => { + res.set("Content-Type", register.contentType); + res.end(await register.metrics()); + }); + }); + + it("returns Prometheus text format", async () => { + const res = await request(app).get("/metrics"); + expect(res.status).toBe(200); + expect(res.headers["content-type"]).toMatch(/text\/plain|application\/openmetrics/); + }); + + it("includes default Node.js metrics", async () => { + const res = await request(app).get("/metrics"); + expect(res.text).toContain("process_cpu"); + }); + + it("includes HTTP request metrics after a request", async () => { + await request(app).get("/test"); + const res = await request(app).get("/metrics"); + expect(res.text).toContain("http_requests_total"); + expect(res.text).toContain("http_request_duration_seconds"); + }); + + it("registers Stellar RPC metric names", async () => { + stellarRpcTotal.inc({ operation: "sendTransaction", result: "success" }); + const res = await request(app).get("/metrics"); + expect(res.text).toContain("stellar_rpc_calls_total"); + }); + + it("registers Stellar RPC duration histogram", async () => { + const end = stellarRpcDuration.startTimer({ operation: "getTransaction" }); + end(); + const res = await request(app).get("/metrics"); + expect(res.text).toContain("stellar_rpc_call_duration_seconds"); + }); + + it("registers cron job metrics", async () => { + cronJobTotal.inc({ job: "score-update", result: "success" }); + const end = cronJobDuration.startTimer({ job: "score-update" }); + end(); + const res = await request(app).get("/metrics"); + expect(res.text).toContain("cron_job_runs_total"); + expect(res.text).toContain("cron_job_duration_seconds"); + }); + + it("registers transaction submission metrics", async () => { + txSubmissionTotal.inc({ result: "success" }); + const res = await request(app).get("/metrics"); + expect(res.text).toContain("stellar_tx_submissions_total"); + }); + + it("registers circuit breaker state gauge", async () => { + circuitBreakerState.set({ name: "StellarRPC" }, 0); + const res = await request(app).get("/metrics"); + expect(res.text).toContain("circuit_breaker_state"); + }); + + it("HTTP metrics are labeled by method and status code", async () => { + await request(app).get("/test"); + const res = await request(app).get("/metrics"); + expect(res.text).toMatch(/http_requests_total\{.*method="GET"/); + expect(res.text).toMatch(/http_requests_total\{.*status_code="200"/); + }); +}); diff --git a/src/config.ts b/src/config.ts index 64b637f..c454542 100644 --- a/src/config.ts +++ b/src/config.ts @@ -90,8 +90,8 @@ export const config = { DB_POOL_HEALTH_CHECK_INTERVAL_MS: numEnv("DB_POOL_HEALTH_CHECK_INTERVAL_MS", 30000), /** Circuit breaker */ - RPC_BREAKER_FAILURE_THRESHOLD: numEnv("RPC_BREAKER_FAILURE_THRESHOLD", 5), - RPC_BREAKER_RECOVERY_TIMEOUT_MS: numEnv("RPC_BREAKER_RECOVERY_TIMEOUT_MS", 30000), + RPC_BREAKER_FAILURE_THRESHOLD: numEnv("CIRCUIT_BREAKER_THRESHOLD", numEnv("RPC_BREAKER_FAILURE_THRESHOLD", 5)), + RPC_BREAKER_RECOVERY_TIMEOUT_MS: numEnv("CIRCUIT_BREAKER_COOLDOWN_MS", numEnv("RPC_BREAKER_RECOVERY_TIMEOUT_MS", 30000)), /** Transaction retries */ TX_MAX_RETRIES: numEnv("TX_MAX_RETRIES", 4), diff --git a/src/index.ts b/src/index.ts index eecf427..67af7eb 100644 --- a/src/index.ts +++ b/src/index.ts @@ -47,6 +47,8 @@ import { import { indexer } from "./lib/indexer"; import { getHealth, getReadiness, recordCronRun } from "./lib/health"; import { getMetrics } from "./lib/metrics"; +import { register } from "./lib/prometheus"; +import { prometheusMiddleware } from "./middleware/prometheusMiddleware"; import { attachWebSocketServer } from "./lib/websocket"; import { rpcPool } from "./lib/stellar"; import { openApiSpec } from "./lib/swagger"; @@ -123,6 +125,7 @@ const corsOrigin = validateCorsOrigin(env.FRONTEND_URL); // across servers regardless of OS locale. Override with e.g. CRON_TIMEZONE=America/New_York. const CRON_TIMEZONE = config.CRON_TIMEZONE; +app.use(prometheusMiddleware); app.use(tracingMiddleware); app.use(securityHeaders); app.use(permissionsHeaders); @@ -142,6 +145,12 @@ app.use(featureFlagContext); // ── Liveness ──────────────────────────────────────────────────────────────── app.get("/health", async (_req, res) => res.json(await getHealth())); +// ── Prometheus metrics ────────────────────────────────────────────────────── +app.get("/metrics", async (_req, res) => { + res.set("Content-Type", register.contentType); + res.end(await register.metrics()); +}); + // ── Readiness ──────────────────────────────────────────────────────────────── app.get("/ready", (_req, res) => { const readiness = getReadiness(); diff --git a/src/lib/circuit-breaker.ts b/src/lib/circuit-breaker.ts index ae63dad..3db3796 100644 --- a/src/lib/circuit-breaker.ts +++ b/src/lib/circuit-breaker.ts @@ -1,20 +1,10 @@ -/** - * Circuit breaker for Stellar RPC calls (#56). - * - * States: - * CLOSED – normal operation, calls pass through - * OPEN – failure threshold exceeded, calls are rejected immediately (fallback invoked) - * HALF_OPEN – recovery probe: one call allowed through to test the RPC - */ +import { logger } from "./logger"; export type BreakerState = "CLOSED" | "OPEN" | "HALF_OPEN"; export interface CircuitBreakerConfig { - /** Number of consecutive failures before opening the circuit. Default: 5 */ failureThreshold: number; - /** How long (ms) to wait in OPEN before moving to HALF_OPEN. Default: 30_000 */ recoveryTimeoutMs: number; - /** Optional name for logging / metrics. */ name?: string; } @@ -91,10 +81,11 @@ export class CircuitBreaker { } private transition(next: BreakerState): void { - console.warn( - `[${this.config.name}] state: ${this.state} → ${next}` + - (next === "OPEN" ? ` (failures: ${this.consecutiveFailures})` : ""), - ); + logger.warn(`[${this.config.name}] circuit state: ${this.state} → ${next}`, { + from: this.state, + to: next, + consecutiveFailures: this.consecutiveFailures, + }); this.state = next; this.lastStateChange = Date.now(); } diff --git a/src/lib/prometheus.ts b/src/lib/prometheus.ts new file mode 100644 index 0000000..9acd559 --- /dev/null +++ b/src/lib/prometheus.ts @@ -0,0 +1,72 @@ +import client from "prom-client"; + +const register = new client.Registry(); + +register.setDefaultLabels({ app: "heliobond-backend" }); +client.collectDefaultMetrics({ register }); + +// ── HTTP metrics ──────────────────────────────────────────────────────────── +export const httpRequestDuration = new client.Histogram({ + name: "http_request_duration_seconds", + help: "Duration of HTTP requests in seconds", + labelNames: ["method", "route", "status_code"] as const, + buckets: [0.01, 0.05, 0.1, 0.3, 0.5, 1, 2, 5], + registers: [register], +}); + +export const httpRequestsTotal = new client.Counter({ + name: "http_requests_total", + help: "Total number of HTTP requests", + labelNames: ["method", "route", "status_code"] as const, + registers: [register], +}); + +// ── Stellar RPC metrics ───────────────────────────────────────────────────── +export const stellarRpcDuration = new client.Histogram({ + name: "stellar_rpc_call_duration_seconds", + help: "Duration of Stellar RPC calls in seconds", + labelNames: ["operation"] as const, + buckets: [0.1, 0.5, 1, 2, 5, 10, 30], + registers: [register], +}); + +export const stellarRpcTotal = new client.Counter({ + name: "stellar_rpc_calls_total", + help: "Total Stellar RPC calls", + labelNames: ["operation", "result"] as const, + registers: [register], +}); + +// ── Cron job metrics ──────────────────────────────────────────────────────── +export const cronJobDuration = new client.Histogram({ + name: "cron_job_duration_seconds", + help: "Duration of cron job executions in seconds", + labelNames: ["job"] as const, + buckets: [0.5, 1, 5, 10, 30, 60, 300], + registers: [register], +}); + +export const cronJobTotal = new client.Counter({ + name: "cron_job_runs_total", + help: "Total cron job executions", + labelNames: ["job", "result"] as const, + registers: [register], +}); + +// ── Transaction metrics ───────────────────────────────────────────────────── +export const txSubmissionTotal = new client.Counter({ + name: "stellar_tx_submissions_total", + help: "Total Stellar transaction submissions", + labelNames: ["result"] as const, + registers: [register], +}); + +// ── Circuit breaker metrics ───────────────────────────────────────────────── +export const circuitBreakerState = new client.Gauge({ + name: "circuit_breaker_state", + help: "Circuit breaker state (0=CLOSED, 1=HALF_OPEN, 2=OPEN)", + labelNames: ["name"] as const, + registers: [register], +}); + +export { register }; diff --git a/src/lib/registry.ts b/src/lib/registry.ts index 04dae75..2a6d0b7 100644 --- a/src/lib/registry.ts +++ b/src/lib/registry.ts @@ -9,6 +9,7 @@ import { } from "@stellar/stellar-sdk"; import { withRpcConnection, networkPassphrase, getAdminKeypair, signAndSubmit } from "./stellar"; import { config } from "../config"; +import { stellarRpcDuration, stellarRpcTotal } from "./prometheus"; if (!config.PROJECT_REGISTRY_CONTRACT_ID) { throw new Error("PROJECT_REGISTRY_CONTRACT_ID env var is required"); @@ -66,6 +67,19 @@ export async function getTotalProjects(): Promise { .setTimeout(config.TX_TIMEOUT_SECONDS) .build(); + const end = stellarRpcDuration.startTimer({ operation: "simulateTransaction" }); + try { + const result = await client.simulateTransaction(tx); + if ("error" in result) throw new Error((result as { error: string }).error); + const sim = result as rpc.Api.SimulateTransactionSuccessResponse; + end(); + stellarRpcTotal.inc({ operation: "simulateTransaction", result: "success" }); + return Number(scValToNative(sim.result!.retval)); + } catch (err) { + end(); + stellarRpcTotal.inc({ operation: "simulateTransaction", result: "failure" }); + throw err; + } const sim = await client.simulateTransaction(tx); if (isSimulationError(sim)) throw new Error(sim.error); diff --git a/src/lib/scoreUpdateCron.ts b/src/lib/scoreUpdateCron.ts index 3dce8ae..cc71cb0 100644 --- a/src/lib/scoreUpdateCron.ts +++ b/src/lib/scoreUpdateCron.ts @@ -11,6 +11,7 @@ import { broadcastScoreUpdate } from "./websocket"; import { recordCronRun } from "./health"; import { logger } from "./logger"; import { config } from "../config"; +import { cronJobDuration, cronJobTotal } from "./prometheus"; /** * Core logic for the hourly score-update cron job. Extracted from src/index.ts @@ -22,6 +23,7 @@ import { config } from "../config"; * failure aborts the whole run — there's nothing to iterate without it. */ export async function runHourlyScoreUpdate(): Promise { + const endCronTimer = cronJobDuration.startTimer({ job: "score-update" }); try { logger.info("[cron] running hourly score update"); const total = await getTotalProjects(); @@ -117,6 +119,8 @@ export async function runHourlyScoreUpdate(): Promise { `check Soroban RPC connectivity and contract state`, ); recordCronRun("score-update", "error"); + endCronTimer(); + cronJobTotal.inc({ job: "score-update", result: "error" }); } else { if (failureCount > 0 && failureRate >= config.CRON_FAILURE_THRESHOLD) { logger.error( @@ -126,11 +130,15 @@ export async function runHourlyScoreUpdate(): Promise { } logger.info("[cron] hourly score update complete", { total, successCount, failureCount }); recordCronRun("score-update", "success"); + endCronTimer(); + cronJobTotal.inc({ job: "score-update", result: "success" }); } } catch (err: any) { if (!isErrorRateLimited("cron:score-update")) { logger.error("[cron] score update failed", { error: err?.message }); } recordCronRun("score-update", "error"); + endCronTimer(); + cronJobTotal.inc({ job: "score-update", result: "error" }); } } diff --git a/src/lib/stellar.ts b/src/lib/stellar.ts index d6a7a74..74cab19 100644 --- a/src/lib/stellar.ts +++ b/src/lib/stellar.ts @@ -3,6 +3,7 @@ import { config } from "../config"; import { RpcConnectionPool } from "./db-pool"; import { CircuitBreaker } from "./circuit-breaker"; import { withRetry, isTransientError } from "./retry"; +import { stellarRpcDuration, stellarRpcTotal, txSubmissionTotal } from "./prometheus"; export const networkPassphrase = config.STELLAR_NETWORK === "mainnet" ? Networks.PUBLIC : Networks.TESTNET; @@ -115,12 +116,19 @@ export async function signAndSubmit( return new Promise((resolve, reject) => { submissionQueue = submissionQueue .then(async () => { + const end = stellarRpcDuration.startTimer({ operation: "signAndSubmit" }); try { const hash = await _executeSignAndSubmitWithRetry(client, preparedXdr, keypair); recordRpcSuccess(); + end(); + stellarRpcTotal.inc({ operation: "signAndSubmit", result: "success" }); + txSubmissionTotal.inc({ result: "success" }); resolve(hash); } catch (error) { if (isTransientError(error)) recordRpcFailure(); + end(); + stellarRpcTotal.inc({ operation: "signAndSubmit", result: "failure" }); + txSubmissionTotal.inc({ result: "failure" }); reject(error); } }) diff --git a/src/middleware/errors.ts b/src/middleware/errors.ts index 942ce68..e7dd276 100644 --- a/src/middleware/errors.ts +++ b/src/middleware/errors.ts @@ -1,4 +1,5 @@ import { Request, Response, NextFunction } from "express"; +import { logger } from "../lib/logger"; /** * Standard error response shape returned by `errorHandler` and any route that @@ -103,6 +104,6 @@ export function errorHandler( return; } - console.error("[error]", err); - res.status(500).json(errorBody("internal_error", "An unexpected error occurred")); + logger.error("[error] unhandled error", logger.formatError(err)); + res.status(500).json(errorBody("INTERNAL_ERROR", "An unexpected error occurred")); } diff --git a/src/middleware/prometheusMiddleware.ts b/src/middleware/prometheusMiddleware.ts new file mode 100644 index 0000000..5b52d90 --- /dev/null +++ b/src/middleware/prometheusMiddleware.ts @@ -0,0 +1,26 @@ +import { Request, Response, NextFunction } from "express"; +import { httpRequestDuration, httpRequestsTotal } from "../lib/prometheus"; + +function normalizeRoute(req: Request): string { + if (req.route?.path) { + return `${req.baseUrl}${req.route.path}`; + } + return req.originalUrl.split("?")[0]; +} + +export function prometheusMiddleware(req: Request, res: Response, next: NextFunction): void { + const end = httpRequestDuration.startTimer(); + + res.on("finish", () => { + const route = normalizeRoute(req); + const labels = { + method: req.method, + route, + status_code: String(res.statusCode), + }; + end(labels); + httpRequestsTotal.inc(labels); + }); + + next(); +}