Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 15 additions & 13 deletions web/components/service/details/service-details-overview.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,8 @@ export type ServiceMetricsResponse = {
windowEnd: string;
stepSeconds: number;
totalRequests: number;
totalIngressBytes: number | null;
totalEgressBytes: number | null;
statusCodes: string[];
buckets: Array<{
timestamp: string;
Expand Down Expand Up @@ -313,8 +315,8 @@ export function ServiceMetricsPanel({
data={chartRows}
margin={{
top: 8,
right: 4,
left: getYAxisMargin(chartMode),
right: fixedMode ? 48 : 4,
left: fixedMode ? 0 : getYAxisMargin(chartMode),
bottom: 0,
}}
>
Expand Down Expand Up @@ -1013,10 +1015,12 @@ function buildServiceMetricSummaryItems(
hasMetricData: boolean,
rangeLabel = "24h",
): ServiceMetricSummaryItem[] {
const summaryRangeLabel = stats?.range ?? rangeLabel;

if (mode === "requests") {
return [
{
label: `requests in ${rangeLabel}`,
label: `requests in ${summaryRangeLabel}`,
value:
hasMetricData && stats
? formatCompactNumber(stats.totalRequests)
Expand Down Expand Up @@ -1061,18 +1065,16 @@ function buildServiceMetricSummaryItems(
if (mode === "traffic") {
return [
{
label: "ingress",
value: formatNullableMetric(
getLatestValue(rows, "ingressBytesPerSecond"),
(value) => `${formatBytes(value)}/s`,
),
label: `ingress in ${summaryRangeLabel}`,
value: hasMetricData
? formatNullableMetric(stats?.totalIngressBytes ?? null, formatBytes)
: "-",
},
{
label: "egress",
value: formatNullableMetric(
getLatestValue(rows, "egressBytesPerSecond"),
(value) => `${formatBytes(value)}/s`,
),
label: `egress in ${summaryRangeLabel}`,
value: hasMetricData
? formatNullableMetric(stats?.totalEgressBytes ?? null, formatBytes)
: "-",
},
];
}
Expand Down
46 changes: 46 additions & 0 deletions web/lib/victoria-metrics.ts
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,8 @@ export type ServiceMetrics = {
windowEnd: string;
stepSeconds: number;
totalRequests: number;
totalIngressBytes: number | null;
totalEgressBytes: number | null;
statusCodes: string[];
buckets: ServiceMetricsBucket[];
};
Expand Down Expand Up @@ -310,6 +312,8 @@ export function createEmptyServiceMetrics(
windowEnd: window.end.toISOString(),
stepSeconds: window.stepSeconds,
totalRequests: 0,
totalIngressBytes: null,
totalEgressBytes: null,
statusCodes: [],
buckets: [],
};
Expand All @@ -328,6 +332,7 @@ export async function queryServiceMetrics(options: {
const serviceMatcher = buildTraefikServiceMatcher(options.serviceId);
const serviceId = escapePromQL(options.serviceId);
const rangeWindow = formatPromDuration(window.stepSeconds);
const totalWindow = formatPromDuration(window.durationMs / 1000);
const traefikFilter = `service=~"${serviceMatcher}"`;
const queryStart = addMilliseconds(
window.start,
Expand All @@ -345,6 +350,8 @@ export async function queryServiceMetrics(options: {
cpuResults,
memoryPercentResults,
memoryBytesResults,
totalIngressBytes,
totalEgressBytes,
] = await Promise.all([
queryRangePromQL(endpoint, {
query: `sum by (code) (increase(traefik_service_requests_total{${traefikFilter}}[${rangeWindow}]))`,
Expand Down Expand Up @@ -406,6 +413,16 @@ export async function queryServiceMetrics(options: {
end: window.end,
stepSeconds: window.stepSeconds,
}).catch(() => []),
queryInstantPromQL(
endpoint,
`sum(increase(traefik_service_requests_bytes_total{${traefikFilter}}[${totalWindow}]))`,
window.end,
).catch(() => null),
queryInstantPromQL(
endpoint,
`sum(increase(traefik_service_responses_bytes_total{${traefikFilter}}[${totalWindow}]))`,
window.end,
).catch(() => null),
]);

const buckets = createServiceMetricBuckets(window);
Expand Down Expand Up @@ -468,11 +485,40 @@ export async function queryServiceMetrics(options: {
(total, bucket) => total + bucket.totalRequests,
0,
),
totalIngressBytes,
totalEgressBytes,
statusCodes: sortStatusCodes([...statusCodes]),
buckets,
};
}

async function queryInstantPromQL(
endpoint: EndpointConfig,
query: string,
time: Date,
): Promise<number | null> {
const url = new URL(`${endpoint.url}/api/v1/query`);
url.searchParams.set("query", query);
url.searchParams.set("time", String(Math.floor(time.getTime() / 1000)));

const response = await fetch(url.toString(), buildFetchOptions(endpoint));
if (!response.ok) {
throw new Error(
`Failed to query metrics: ${response.status} ${response.statusText}`,
);
}

const data = (await response.json()) as VictoriaInstantResponse;
if (data.status !== "success") {
throw new Error(data.error || "Failed to query metrics");
}

const rawValue = data.data?.result?.[0]?.value?.[1];
if (rawValue === undefined) return null;
const value = Number.parseFloat(rawValue);
return Number.isFinite(value) ? value : null;
}

async function queryInstantMetric(
endpoint: EndpointConfig,
metricName: string,
Expand Down
70 changes: 67 additions & 3 deletions web/tests/victoria-metrics-service-metrics.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,8 @@ describe("VictoriaMetrics service metrics", () => {
windowEnd: "2026-07-02T01:00:00.000Z",
stepSeconds: 60,
totalRequests: 0,
totalIngressBytes: null,
totalEgressBytes: null,
statusCodes: [],
buckets: [],
});
Expand All @@ -52,13 +54,26 @@ describe("VictoriaMetrics service metrics", () => {

const queries: string[] = [];
const starts: string[] = [];
const instantTimes: string[] = [];
const fetchMock = vi.fn(async (input: string | URL | Request) => {
const url = new URL(String(input));
const query = url.searchParams.get("query") || "";
queries.push(query);
starts.push(url.searchParams.get("start") || "");
if (url.pathname === "/api/v1/query_range") {
starts.push(url.searchParams.get("start") || "");
}

expect(["/api/v1/query", "/api/v1/query_range"]).toContain(url.pathname);

expect(url.pathname).toBe("/api/v1/query_range");
if (url.pathname === "/api/v1/query") {
instantTimes.push(url.searchParams.get("time") || "");
if (query.includes("traefik_service_requests_bytes_total")) {
return instantJsonResponse("460");
}
if (query.includes("traefik_service_responses_bytes_total")) {
return instantJsonResponse("683051");
}
}

if (query.includes("traefik_service_requests_total")) {
return jsonResponse([
Expand Down Expand Up @@ -91,6 +106,9 @@ describe("VictoriaMetrics service metrics", () => {
if (query.includes("histogram_quantile(0.95")) {
return jsonResponse([{ metric: {}, values: [[END_TS, "0.123"]] }]);
}
if (query.includes("traefik_service_requests_bytes_total")) {
return jsonResponse([{ metric: {}, values: [[END_TS, "512"]] }]);
}
if (query.includes("traefik_service_responses_bytes_total")) {
return jsonResponse([{ metric: {}, values: [[END_TS, "2048"]] }]);
}
Expand All @@ -108,7 +126,8 @@ describe("VictoriaMetrics service metrics", () => {
now: new Date("2026-07-02T12:34:00Z"),
});

expect(fetchMock).toHaveBeenCalledTimes(10);
expect(fetchMock).toHaveBeenCalledTimes(12);
expect(instantTimes).toEqual([String(END_TS), String(END_TS)]);
expect(queries.some((query) => query.includes("LogSQL"))).toBe(false);
expect(queries).toContain(
`sum by (code) (increase(traefik_service_requests_total{service=~"^${SERVICE_ID}(@file)?$"}[5m]))`,
Expand All @@ -119,10 +138,18 @@ describe("VictoriaMetrics service metrics", () => {
expect(queries).toContain(
`sum(avg_over_time(techulus_service_memory_usage_percent{service_id="${SERVICE_ID}"}[5m]))`,
);
expect(queries).toContain(
`sum(increase(traefik_service_requests_bytes_total{service=~"^${SERVICE_ID}(@file)?$"}[1d]))`,
);
expect(queries).toContain(
`sum(increase(traefik_service_responses_bytes_total{service=~"^${SERVICE_ID}(@file)?$"}[1d]))`,
);
expect(
starts.every((start) => start === String(END_TS - 24 * 60 * 60 + 5 * 60)),
).toBe(true);
expect(stats.totalRequests).toBe(21);
expect(stats.totalIngressBytes).toBe(460);
expect(stats.totalEgressBytes).toBe(683051);
expect(stats.statusCodes).toEqual(["2xx", "3xx", "4xx", "5xx"]);
expect(stats.buckets).toHaveLength(288);
expect(stats.windowEnd).toBe("2026-07-02T12:30:00.000Z");
Expand All @@ -133,10 +160,38 @@ describe("VictoriaMetrics service metrics", () => {
totalRequests: 21,
statuses: { "2xx": 5, "3xx": 1, "4xx": 4, "5xx": 11 },
p95ResponseTimeMs: 123,
ingressBytesPerSecond: 512,
egressBytesPerSecond: 2048,
cpuUsagePercent: 42,
});
});

it("keeps service metrics available when traffic total queries fail", async () => {
vi.stubEnv("VICTORIA_METRICS_URL", "http://victoria.test");
vi.stubEnv("VICTORIA_METRICS_PRIVATE_URL", "");

vi.stubGlobal(
"fetch",
vi.fn(async (input: string | URL | Request) => {
const url = new URL(String(input));
if (url.pathname === "/api/v1/query") {
return new Response("Unavailable", { status: 503 });
}
return jsonResponse([]);
}),
);

const stats = await queryServiceMetrics({
serviceId: SERVICE_ID,
range: "1h",
now: new Date("2026-07-02T12:34:00Z"),
});

expect(stats.metricsEnabled).toBe(true);
expect(stats.totalIngressBytes).toBeNull();
expect(stats.totalEgressBytes).toBeNull();
expect(stats.buckets).toHaveLength(60);
});
});

function jsonResponse(result: unknown[]) {
Expand All @@ -147,3 +202,12 @@ function jsonResponse(result: unknown[]) {
}),
);
}

function instantJsonResponse(value: string) {
return new Response(
JSON.stringify({
status: "success",
data: { result: [{ metric: {}, value: [END_TS, value] }] },
}),
);
}
Loading