From 975ce38375ae7dd0586d5650d1a74d78ccd53a1a Mon Sep 17 00:00:00 2001 From: Miguel Cruces Date: Mon, 15 Jun 2026 12:07:26 +0200 Subject: [PATCH] feat: Add option to auto generate histograms for each endpoint --- packages/lib/src/config/config.ts | 4 +- packages/lib/src/config/config.types.ts | 1 + packages/lib/src/index.ts | 7 ++ packages/lib/src/tlm-oas/oasMiddleware.ts | 116 ++++++++++++++++++ packages/lib/src/utils/oasUtils.ts | 48 ++++++++ packages/lib/src/utils/packageUtils.ts | 59 +++++++++ packages/lib/test/e2e/definitions/metrics.ts | 18 +++ packages/lib/test/e2e/index.test.ts | 6 + packages/lib/test/e2e/servers/otTestServer.ts | 3 +- packages/lib/test/unit/config.test.ts | 10 ++ 10 files changed, 270 insertions(+), 2 deletions(-) create mode 100644 packages/lib/src/tlm-oas/oasMiddleware.ts create mode 100644 packages/lib/src/utils/oasUtils.ts create mode 100644 packages/lib/src/utils/packageUtils.ts diff --git a/packages/lib/src/config/config.ts b/packages/lib/src/config/config.ts index 07731df..857612d 100644 --- a/packages/lib/src/config/config.ts +++ b/packages/lib/src/config/config.ts @@ -39,7 +39,8 @@ const loadEnv = (): DeepPartial => { enabled: getParsedEnvVar("OASTLM_CONFIG_METRICS_MEMORY_EXPORTER_ENABLED", (v) => v === "true"), retentionTimeSeconds: getParsedEnvVar("OASTLM_CONFIG_METRICS_MEMORY_EXPORTER_RETENTION_TIME_SECONDS", (v) => parseInt(v, 10)), // filters NOT settable via env - } + }, + autoGenerateEndpointHistograms: getParsedEnvVar("OASTLM_CONFIG_METRICS_AUTO_GENERATE_ENDPOINT_HISTOGRAMS", (v) => v === "true"), }, logs: { memoryExporter: { @@ -104,6 +105,7 @@ export const defaultConfig = { retentionTimeSeconds: 60 * 60, // 1 hour }, filters: [] as any[], // future feature, currently not used + autoGenerateEndpointHistograms: false, }, logs: { extraExporters: [] as LogRecordExporter[], // e.g. [new ConsoleLogRecordExporter()] diff --git a/packages/lib/src/config/config.types.ts b/packages/lib/src/config/config.types.ts index c0606f7..0d581a4 100644 --- a/packages/lib/src/config/config.types.ts +++ b/packages/lib/src/config/config.types.ts @@ -32,6 +32,7 @@ export type UserConfig = { extraReaders?: IMetricReader[]; // required shape extraViews?: ViewOptions[]; memoryExporter?: Partial; + autoGenerateEndpointHistograms?: boolean; }; logs?: { extraExporters?: LogRecordExporter[]; diff --git a/packages/lib/src/index.ts b/packages/lib/src/index.ts index 859de78..6be945c 100644 --- a/packages/lib/src/index.ts +++ b/packages/lib/src/index.ts @@ -8,6 +8,7 @@ import { UserConfig } from './config/config.types.js'; import { configureTelemetry } from './telemetry/telemetryConfigurator.js'; import { isTelemetryConfigured, setTelemetryRouter, getTelemetryRouter } from './telemetry/telemetryRegistry.js'; import { bootEnvVariables } from "./config/bootConfig.js"; +import { getAutoEndpointMetricsMiddleware } from './tlm-oas/oasMiddleware.js'; /** * Returns the OAS-Telemetry middleware. @@ -23,6 +24,12 @@ function oasTelemetry(oasTlmInputConfig?: UserConfig) { return router; } const oasTlmConfig = getConfig(oasTlmInputConfig); + + // Register the auto endpoint metrics middleware at the root level of the router + if (oasTlmConfig.metrics.autoGenerateEndpointHistograms) { + router.use(getAutoEndpointMetricsMiddleware(oasTlmConfig)); + } + logger.info("BaseUrl: ", bootEnvVariables.OASTLM_BOOT_BASE_URL); configureTelemetry(oasTlmConfig); configureRoutes(router, oasTlmConfig); diff --git a/packages/lib/src/tlm-oas/oasMiddleware.ts b/packages/lib/src/tlm-oas/oasMiddleware.ts new file mode 100644 index 0000000..29740c6 --- /dev/null +++ b/packages/lib/src/tlm-oas/oasMiddleware.ts @@ -0,0 +1,116 @@ +import { metrics, Histogram } from '@opentelemetry/api'; +import { Request, Response, NextFunction } from 'express'; +import { OasTlmConfig } from '../config/config.types.js'; +import { bootEnvVariables } from '../config/bootConfig.js'; +import { getPackageVersion } from '../utils/packageUtils.js'; +import { loadApiSpec, normalizeSpecPath, normalizeExpressPath } from '../utils/oasUtils.js'; + +const histogramCache = new Map(); + +// Determine library version dynamically +// @ts-ignore -- import.meta is replaced in CJS builds but TypeScript requires it here +const packageVersion = getPackageVersion(import.meta.url); + +export function getAutoEndpointMetricsMiddleware(config: OasTlmConfig) { + // Load the spec at startup + const spec = loadApiSpec(config); + + // Map of normalized path -> { originalPath, methods } for highly efficient O(1) matching + const normalizedSpecMap = new Map(); + if (spec && spec.paths) { + for (const specPath of Object.keys(spec.paths)) { + const normalized = normalizeSpecPath(specPath); + const methods = Object.keys(spec.paths[specPath]) + .filter(key => ['get', 'post', 'put', 'delete', 'options', 'head', 'patch', 'trace'].includes(key.toLowerCase())) + .map(m => m.toLowerCase()); + normalizedSpecMap.set(normalized, { originalPath: specPath, methods }); + } + } + + let initialized = false; + const initializeHistograms = () => { + if (initialized) return; + initialized = true; + + const meter = metrics.getMeter('oas-telemetry-auto-endpoint-metrics', packageVersion); + for (const entry of normalizedSpecMap.values()) { + for (const method of entry.methods) { + const cleanEndpoint = entry.originalPath + .replace(/^\/+|\/+$/g, '') // Remove leading/trailing slashes + .replace(/[{}]/g, '') // Remove braces { } + .replace(/:/g, '') // Remove colons if any + .replace(/\//g, '.') // Replace slashes with dots + .replace(/\.\./g, '.') // Prevent double dots + || 'root'; + const metricName = `oas-telemetry.auto.${method}.${cleanEndpoint}.ms`; + + if (!histogramCache.has(metricName)) { + const histogram = meter.createHistogram(metricName, { + description: `Auto-generated histogram for spec endpoint ${method.toUpperCase()} ${entry.originalPath}`, + unit: 'ms', + }); + histogramCache.set(metricName, histogram); + } + } + } + }; + + return function autoEndpointMetricsMiddleware(req: Request, res: Response, next: NextFunction) { + if (!config.metrics.autoGenerateEndpointHistograms) { + return next(); + } + + // Initialize histograms on the first request to ensure the Telemetry SDK has started + initializeHistograms(); + + const telemetryBaseUrl = bootEnvVariables.OASTLM_BOOT_BASE_URL; + const path = req.path || ''; + + // Ignore internal telemetry routes to avoid self-profiling noise + if (telemetryBaseUrl && path.includes(telemetryBaseUrl)) { + return next(); + } + + const start = process.hrtime(); + + res.on('finish', () => { + const routePath = req.route?.path; + if (!routePath) { + return; + } + + const verb = req.method.toLowerCase(); + const normalizedRoute = normalizeExpressPath(routePath); + const matchedSpec = normalizedSpecMap.get(normalizedRoute); + + // If the endpoint or method is not defined in the API spec, do not record metric + if (!matchedSpec || !matchedSpec.methods.includes(verb)) { + return; + } + + const diff = process.hrtime(start); + const durationMs = (diff[0] * 1e3) + (diff[1] * 1e-6); + + const cleanEndpoint = matchedSpec.originalPath + .replace(/^\/+|\/+$/g, '') // Remove leading/trailing slashes + .replace(/[{}]/g, '') // Remove braces { } + .replace(/:/g, '') // Remove colons if any + .replace(/\//g, '.') // Replace slashes with dots + .replace(/\.\./g, '.') // Prevent double dots + || 'root'; + + const metricName = `oas-telemetry.auto.${verb}.${cleanEndpoint}.ms`; + + const histogram = histogramCache.get(metricName); + if (histogram) { + histogram.record(durationMs, { + http_method: req.method, + http_route: matchedSpec.originalPath, + http_status_code: res.statusCode.toString(), + }); + } + }); + + next(); + }; +} diff --git a/packages/lib/src/utils/oasUtils.ts b/packages/lib/src/utils/oasUtils.ts new file mode 100644 index 0000000..77c12dd --- /dev/null +++ b/packages/lib/src/utils/oasUtils.ts @@ -0,0 +1,48 @@ +import { existsSync, readFileSync } from 'fs'; +import yaml from 'js-yaml'; +import logger from './logger.js'; + +/** + * Loads the API specification from config. + */ +export function loadApiSpec(config: any): any { + if (config.general?.specFileName && existsSync(config.general.specFileName)) { + try { + const data = readFileSync(config.general.specFileName, 'utf8'); + if (config.general.specFileName.endsWith('.yaml') || config.general.specFileName.endsWith('.yml')) { + return yaml.load(data); + } + return JSON.parse(data); + } catch (e: any) { + logger.warn(`[oasUtils] Failed to load spec file: ${e.message}`); + } + } + if (config.general?.spec) { + try { + return JSON.parse(config.general.spec); + } catch (ej) { + try { + return yaml.load(config.general.spec); + } catch (ey: any) { + logger.warn(`[oasUtils] Failed to parse spec string: ${ey.message}`); + } + } + } + return null; +} + +/** + * Normalizes an OpenAPI spec path by replacing parameter placeholders with '{}' + * e.g., "/api/v1/pets/{petName}" -> "/api/v1/pets/{}" + */ +export function normalizeSpecPath(path: string): string { + return path.replace(/{[^}]+}/g, '{}'); +} + +/** + * Normalizes an Express route path by replacing parameter placeholders with '{}' + * e.g., "/api/v1/pets/:name" -> "/api/v1/pets/{}" + */ +export function normalizeExpressPath(path: string): string { + return path.replace(/:[^/]+/g, '{}'); +} diff --git a/packages/lib/src/utils/packageUtils.ts b/packages/lib/src/utils/packageUtils.ts new file mode 100644 index 0000000..f21606d --- /dev/null +++ b/packages/lib/src/utils/packageUtils.ts @@ -0,0 +1,59 @@ +import { existsSync, readFileSync } from 'fs'; +import { dirname, join } from 'path'; +import { fileURLToPath } from 'url'; +import logger from './logger.js'; + +/** + * Recursively search upwards for the library's package.json file. + */ +export function findPackageJson(startDir: string): string | null { + let dir = startDir; + while (true) { + const file = join(dir, 'package.json'); + if (existsSync(file)) { + try { + const content = JSON.parse(readFileSync(file, 'utf8')); + if (content.name === '@oas-tools/oas-telemetry') { + return file; + } + } catch (e) { + // Ignore parsing/read errors and continue upwards + } + } + const parent = dirname(dir); + if (parent === dir) { + break; + } + dir = parent; + } + return null; +} + +const fallbackVersion = '0.8.0'; + +/** + * Dynamically resolves the package version of the library. + * Expects the caller's import.meta.url or equivalent to be passed. + */ +export function getPackageVersion(moduleUrl: string): string { + try { + const isCjs = typeof __filename !== 'undefined' && typeof __dirname !== 'undefined'; + // @ts-ignore -- import.meta does not exist in CJS build + const currentDir = isCjs ? __dirname : dirname(fileURLToPath(moduleUrl)); + const packageJsonPath = findPackageJson(currentDir); + if (packageJsonPath) { + const packageJson = JSON.parse(readFileSync(packageJsonPath, 'utf8')); + if (packageJson.version == null) { + logger.warn(`[packageUtils] Valid package.json found at ${packageJsonPath} does not have a version field, using fallback version (${fallbackVersion}).`); + return fallbackVersion; + } + return packageJson.version; + } else { + logger.warn(`[packageUtils] Could not find package.json for @oas-tools/oas-telemetry, using fallback version (${fallbackVersion}).`); + return fallbackVersion; + } + } catch (e: any) { + logger.warn(`[packageUtils] Failed to read package.json, falling back to default version (${fallbackVersion}). Error: ${e.message}`); + return fallbackVersion; + } +} diff --git a/packages/lib/test/e2e/definitions/metrics.ts b/packages/lib/test/e2e/definitions/metrics.ts index ca8ceb5..c67b644 100644 --- a/packages/lib/test/e2e/definitions/metrics.ts +++ b/packages/lib/test/e2e/definitions/metrics.ts @@ -341,5 +341,23 @@ export function defineMetricsApiTests(config: E2ETestConfig) { expect(singleResponse.data.scopeMetrics[0].scope.name).toBe(firstMetric.scope.name); expect(singleResponse.data.scopeMetrics[0].descriptor.name).toBe(firstMetric.descriptor.name); }); + + it('[e2e][Metrics:AutoHistograms][+] should automatically record histogram metric per endpoint', async () => { + // First, call a standard endpoint (like GET /api/v1/pets) to trigger telemetry + const triggerResponse = await axios.get(`${baseUrl}/api/v1/pets`).catch((err) => err.response); + expect(triggerResponse.status).toBe(200); + + // Wait/retry to make sure metrics are exported (interval is 250ms) + await retry(async () => { + const metricsResponse = await axios.get(metricsDataUrl).catch((err) => err.response); + expect(metricsResponse.status).toBe(200); + + // Find our automatically generated endpoint histogram metric + const foundMetric = metricsResponse.data.scopeMetrics.find((sm: any) => + sm.descriptor.name === 'oas-telemetry.auto.get.api.v1.pets.ms' + ); + expect(foundMetric).toBeDefined(); + }, { timeout: 3000, interval: 100 }); + }); }); } diff --git a/packages/lib/test/e2e/index.test.ts b/packages/lib/test/e2e/index.test.ts index 4117fcb..7e5baf5 100644 --- a/packages/lib/test/e2e/index.test.ts +++ b/packages/lib/test/e2e/index.test.ts @@ -10,6 +10,9 @@ const cjsConfig: E2ETestConfig = { serverScript: 'test/e2e/servers/otTestServer.cjs', port: '3231', telemetryPath: '/telemetry', + additionalEnv: { + OASTLM_CONFIG_METRICS_AUTO_GENERATE_ENDPOINT_HISTOGRAMS: 'true', + } }; const esmConfig: E2ETestConfig = { @@ -17,6 +20,9 @@ const esmConfig: E2ETestConfig = { serverScript: 'test/e2e/servers/otTestServer.mjs', port: '3232', telemetryPath: '/telemetry', + additionalEnv: { + OASTLM_CONFIG_METRICS_AUTO_GENERATE_ENDPOINT_HISTOGRAMS: 'true', + } }; diff --git a/packages/lib/test/e2e/servers/otTestServer.ts b/packages/lib/test/e2e/servers/otTestServer.ts index fedc96c..1e65375 100644 --- a/packages/lib/test/e2e/servers/otTestServer.ts +++ b/packages/lib/test/e2e/servers/otTestServer.ts @@ -95,13 +95,14 @@ const oasTlmConfig: UserConfig = { // exportIntervalMillis: 1000 * 30, // 30 seconds // exporter: new ConsoleMetricExporter() // })], + autoGenerateEndpointHistograms: true, }, logs: { // extraExporters: [new ConsoleLogRecordExporter()], // extraProcessors: [new SimpleLogRecordProcessor(new ConsoleLogRecordExporter())], }, auth: { - // enabled: true, + // enabled: true, // jwtSecret: "secret", // password: "password", // accessTokenMaxAge: 1000 * 60 * 2 , // 2 minutes diff --git a/packages/lib/test/unit/config.test.ts b/packages/lib/test/unit/config.test.ts index d1fed91..c9775d7 100644 --- a/packages/lib/test/unit/config.test.ts +++ b/packages/lib/test/unit/config.test.ts @@ -55,4 +55,14 @@ describe("Config Tests", () => { const config = getConfig(undefined, defaultConfig, envConfig); expect(config.general.specFileName).toBe(defaultConfig.general.specFileName); }); + + it("[Unit][Config][+] should default autoGenerateEndpointHistograms to false", () => { + const config = getConfig(); + expect(config.metrics.autoGenerateEndpointHistograms).toBe(false); + }); + + it("[Unit][Config][+] should override autoGenerateEndpointHistograms with userConfig", () => { + const config = getConfig({ metrics: { autoGenerateEndpointHistograms: true } }); + expect(config.metrics.autoGenerateEndpointHistograms).toBe(true); + }); });