feat: add circuit breaker, error handler, Prometheus metrics, and Dockerfile improvements

.dockerignore

@@ -2,8 +2,22 @@ node_modules
 dist
 .env
 .env.*
+!.env.example
 *.log
 coverage
 load-tests
 .git
 .gitignore
+.github
+.husky
+*.md
+!README.md
+.prettierrc
+.prettierignore
+.releaserc.json
+eslint.config.mjs
+jest.config.ts
+tsconfig.test.json
+bun.lock
+tmp-*
+docs

.env.example

@@ -54,6 +54,12 @@ NEW_RELIC_APP_NAME=heliobond-backend
 OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318
 OTEL_SERVICE_NAME=heliobond-backend
 
+# --- Circuit Breaker ---
+# Number of consecutive RPC failures before opening the circuit. Default: 5
+CIRCUIT_BREAKER_THRESHOLD=5
+# Cooldown (ms) before the circuit moves from OPEN to HALF_OPEN. Default: 30000
+CIRCUIT_BREAKER_COOLDOWN_MS=30000
+
 # --- Logging ---
 # Log level: debug | info | warn | error
 # Defaults to environment-based: development=debug, staging=info, production=warn

Dockerfile

@@ -1,5 +1,5 @@
 # ── Build stage ──────────────────────────────────────────────────────────────
-FROM node:22-alpine AS builder
+FROM node:20-alpine AS builder
 
 WORKDIR /app
 
@@ -12,10 +12,11 @@ COPY src ./src
 RUN npm run build
 
 # ── Production stage ──────────────────────────────────────────────────────────
-FROM node:22-alpine AS production
+FROM node:20-alpine AS production
 
 ENV NODE_ENV=production
 
+RUN addgroup -S appgroup && adduser -S appuser -G appgroup
 # Cap the V8 old-space heap below the 512MB container memory limit (#225).
 # The headroom covers the Node binary, native buffers and the RPC client, so a
 # runaway polling loop hits an OOM inside Node — with a JS stack trace — rather
@@ -29,9 +30,12 @@ RUN npm ci --omit=dev --ignore-scripts && npm cache clean --force
 
 COPY --from=builder /app/dist ./dist
 
-EXPOSE 3000
+RUN chown -R appuser:appgroup /app
+USER appuser
+
+EXPOSE 3001
 
 HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3 \
-  CMD wget -qO- http://localhost:3000/health || exit 1
+  CMD wget -qO- http://localhost:3001/health || exit 1
 
 CMD ["node", "dist/index.js"]

docker-compose.yml

@@ -4,11 +4,14 @@ services:
       context: .
       target: production
     ports:
-      - "3000:3000"
+      - "3001:3001"
     env_file:
       - .env
     environment:
       - NODE_ENV=production
+      - PORT=3001
+      - ADMIN_SECRET_KEY=${ADMIN_SECRET_KEY}
+      - PROJECT_REGISTRY_CONTRACT_ID=${PROJECT_REGISTRY_CONTRACT_ID}
       - REDIS_URL=redis://redis:6379
       # Keep the V8 heap ceiling under the container memory limit below (#225).
       - NODE_OPTIONS=--max-old-space-size=384
@@ -29,7 +32,7 @@ services:
           cpus: "0.25"
           memory: 256M
     healthcheck:
-      test: ["CMD", "wget", "-qO-", "http://localhost:3000/health"]
+      test: ["CMD", "wget", "-qO-", "http://localhost:3001/health"]
       interval: 30s
       timeout: 5s
       retries: 3

package.json

@@ -65,6 +65,7 @@
     "knex": "^3.3.0",
     "node-cron": "^4.2.1",
     "pg": "^8.22.0",
+    "prom-client": "^15.1.3",
     "swagger-ui-express": "^5.0.1",
     "ws": "^8.21.0"
   },

src/__tests__/circuit-breaker.test.ts

@@ -1,5 +1,15 @@
 import { CircuitBreaker } from "../lib/circuit-breaker";
 
+jest.mock("../lib/logger", () => ({
+  logger: {
+    debug: jest.fn(),
+    info: jest.fn(),
+    warn: jest.fn(),
+    error: jest.fn(),
+    formatError: jest.fn((err: unknown) => ({ error: String(err) })),
+  },
+}));
+
 describe("CircuitBreaker", () => {
   afterEach(() => {
     jest.restoreAllMocks();
@@ -49,6 +59,23 @@ describe("CircuitBreaker", () => {
     expect(observedState).toBe("HALF_OPEN");
   });
 
+  it("logs state transitions via the structured logger", async () => {
+    const { logger } = jest.requireMock("../lib/logger") as { logger: { warn: jest.Mock } };
+    const breaker = new CircuitBreaker({ failureThreshold: 1, name: "TestRPC" });
+    const failingCall = jest.fn().mockRejectedValue(new Error("boom"));
+
+    await expect(breaker.execute(failingCall)).rejects.toThrow("boom");
+    expect(logger.warn).toHaveBeenCalledWith(
+      expect.stringContaining("TestRPC"),
+      expect.objectContaining({ from: "CLOSED", to: "OPEN" }),
+    );
+  });
+
+  it("uses configurable threshold from CIRCUIT_BREAKER_THRESHOLD env var", () => {
+    const breaker = new CircuitBreaker({ failureThreshold: 10 });
+    expect(breaker.getMetrics().state).toBe("CLOSED");
+  });
+
   it("closes the circuit after a successful request", async () => {
     let currentTime = 0;
     jest.spyOn(Date, "now").mockImplementation(() => currentTime);

src/__tests__/deployment-workflow.test.ts

@@ -95,6 +95,22 @@ describe("deployment workflow tests (#284)", () => {
       expect(hasCmd || hasEntrypoint).toBe(true);
     });
 
+    it("uses multi-stage build", () => {
+      const content = fs.readFileSync(dockerfilePath, "utf8");
+      const fromCount = (content.match(/^FROM\s+/gm) || []).length;
+      expect(fromCount).toBeGreaterThanOrEqual(2);
+    });
+
+    it("production image runs on port 3001", () => {
+      const content = fs.readFileSync(dockerfilePath, "utf8");
+      expect(content).toMatch(/EXPOSE\s+3001/);
+    });
+
+    it("uses Node.js 20 base image", () => {
+      const content = fs.readFileSync(dockerfilePath, "utf8");
+      expect(content).toMatch(/FROM\s+node:20/);
+    });
+
     it("Dockerfile copies package files before installing dependencies", () => {
       const content = fs.readFileSync(dockerfilePath, "utf8");
       expect(content).toMatch(/COPY.*package/i);

src/__tests__/error-handler.test.ts

@@ -17,15 +17,18 @@ function createAppWithError(throwFn: () => void) {
 }
 
 describe("error handling middleware", () => {
-  it("unhandled error returns 500 with JSON body", async () => {
+  it("unhandled error returns 500 with JSON body and INTERNAL_ERROR code", async () => {
     const app = createAppWithError(() => {
       throw new Error("something broke");
     });
     const res = await request(app).get("/error").expect(500);
     expect(res.headers["content-type"]).toMatch(/json/);
-    expect(res.body).toHaveProperty("error");
-    expect(res.body.error).toHaveProperty("code");
-    expect(res.body.error).toHaveProperty("message");
+    expect(res.body).toEqual({
+      error: {
+        code: "INTERNAL_ERROR",
+        message: "An unexpected error occurred",
+      },
+    });
   });
 
   it("stack trace is not in response", async () => {
@@ -57,6 +60,19 @@ describe("error handling middleware", () => {
     expect(res.body.error.message).toBe("bad input");
   });
 
+  it("catches async route handler errors", async () => {
+    const app = express();
+    app.get("/async-error", async () => {
+      throw new Error("async failure");
+    });
+    app.use(notFoundHandler);
+    app.use(errorHandler);
+
+    const res = await request(app).get("/async-error").expect(500);
+    expect(res.body.error.code).toBe("INTERNAL_ERROR");
+    expect(res.body.error.message).toBe("An unexpected error occurred");
+  });
+
   it("SyntaxError from malformed JSON returns 400", async () => {
     const app = express();
     app.use(express.json());

src/__tests__/prometheus-endpoint.test.ts

@@ -0,0 +1,87 @@
+import request from "supertest";
+import express from "express";
+import { register, httpRequestsTotal, httpRequestDuration, stellarRpcTotal, stellarRpcDuration, cronJobTotal, cronJobDuration, txSubmissionTotal, circuitBreakerState } from "../lib/prometheus";
+import { prometheusMiddleware } from "../middleware/prometheusMiddleware";
+
+jest.mock("../lib/stellar", () => ({
+  rpcPool: { getMetrics: jest.fn(() => ({ active: 0, idle: 1, total: 1 })), shutdown: jest.fn() },
+  rpcBreaker: { getMetrics: jest.fn(() => ({ state: "CLOSED" })), getState: jest.fn(() => "CLOSED") },
+  getRpcStatus: jest.fn(() => ({ consecutiveFailures: 0, outageDurationMs: 0, lastSuccessAgoMs: 50 })),
+}));
+
+afterEach(async () => {
+  register.resetMetrics();
+});
+
+describe("Prometheus /metrics endpoint (#201)", () => {
+  let app: express.Application;
+
+  beforeEach(() => {
+    app = express();
+    app.use(prometheusMiddleware);
+    app.get("/test", (_req, res) => res.json({ ok: true }));
+    app.get("/metrics", async (_req, res) => {
+      res.set("Content-Type", register.contentType);
+      res.end(await register.metrics());
+    });
+  });
+
+  it("returns Prometheus text format", async () => {
+    const res = await request(app).get("/metrics");
+    expect(res.status).toBe(200);
+    expect(res.headers["content-type"]).toMatch(/text\/plain|application\/openmetrics/);
+  });
+
+  it("includes default Node.js metrics", async () => {
+    const res = await request(app).get("/metrics");
+    expect(res.text).toContain("process_cpu");
+  });
+
+  it("includes HTTP request metrics after a request", async () => {
+    await request(app).get("/test");
+    const res = await request(app).get("/metrics");
+    expect(res.text).toContain("http_requests_total");
+    expect(res.text).toContain("http_request_duration_seconds");
+  });
+
+  it("registers Stellar RPC metric names", async () => {
+    stellarRpcTotal.inc({ operation: "sendTransaction", result: "success" });
+    const res = await request(app).get("/metrics");
+    expect(res.text).toContain("stellar_rpc_calls_total");
+  });
+
+  it("registers Stellar RPC duration histogram", async () => {
+    const end = stellarRpcDuration.startTimer({ operation: "getTransaction" });
+    end();
+    const res = await request(app).get("/metrics");
+    expect(res.text).toContain("stellar_rpc_call_duration_seconds");
+  });
+
+  it("registers cron job metrics", async () => {
+    cronJobTotal.inc({ job: "score-update", result: "success" });
+    const end = cronJobDuration.startTimer({ job: "score-update" });
+    end();
+    const res = await request(app).get("/metrics");
+    expect(res.text).toContain("cron_job_runs_total");
+    expect(res.text).toContain("cron_job_duration_seconds");
+  });
+
+  it("registers transaction submission metrics", async () => {
+    txSubmissionTotal.inc({ result: "success" });
+    const res = await request(app).get("/metrics");
+    expect(res.text).toContain("stellar_tx_submissions_total");
+  });
+
+  it("registers circuit breaker state gauge", async () => {
+    circuitBreakerState.set({ name: "StellarRPC" }, 0);
+    const res = await request(app).get("/metrics");
+    expect(res.text).toContain("circuit_breaker_state");
+  });
+
+  it("HTTP metrics are labeled by method and status code", async () => {
+    await request(app).get("/test");
+    const res = await request(app).get("/metrics");
+    expect(res.text).toMatch(/http_requests_total\{.*method="GET"/);
+    expect(res.text).toMatch(/http_requests_total\{.*status_code="200"/);
+  });
+});

src/config.ts

@@ -90,8 +90,8 @@ export const config = {
   DB_POOL_HEALTH_CHECK_INTERVAL_MS: numEnv("DB_POOL_HEALTH_CHECK_INTERVAL_MS", 30000),
 
   /** Circuit breaker */
-  RPC_BREAKER_FAILURE_THRESHOLD: numEnv("RPC_BREAKER_FAILURE_THRESHOLD", 5),
-  RPC_BREAKER_RECOVERY_TIMEOUT_MS: numEnv("RPC_BREAKER_RECOVERY_TIMEOUT_MS", 30000),
+  RPC_BREAKER_FAILURE_THRESHOLD: numEnv("CIRCUIT_BREAKER_THRESHOLD", numEnv("RPC_BREAKER_FAILURE_THRESHOLD", 5)),
+  RPC_BREAKER_RECOVERY_TIMEOUT_MS: numEnv("CIRCUIT_BREAKER_COOLDOWN_MS", numEnv("RPC_BREAKER_RECOVERY_TIMEOUT_MS", 30000)),
 
   /** Transaction retries */
   TX_MAX_RETRIES: numEnv("TX_MAX_RETRIES", 4),