diff --git a/.changeset/datadog-adapter-init.md b/.changeset/datadog-adapter-init.md new file mode 100644 index 0000000..b47a811 --- /dev/null +++ b/.changeset/datadog-adapter-init.md @@ -0,0 +1,11 @@ +--- +"@cruncher/adapter-datadog": minor +--- + +Add Datadog adapter with browser-session auth, query support, and facet/index autocomplete. + +- Cookie + CSRF-token based authentication via Electron auth window +- Translates QQL queries to Datadog Lucene syntax +- Fetches log index names for `index=` autocomplete +- Fetches facet metadata from `/api/ui/event-platform/logs/facets` for param name suggestions +- Aggregates top values per dynamic facet for param value suggestions diff --git a/apps/cruncher/package.json b/apps/cruncher/package.json index 35c685b..d9a670a 100644 --- a/apps/cruncher/package.json +++ b/apps/cruncher/package.json @@ -67,6 +67,7 @@ "@babel/runtime": "^7.28.6", "@chakra-ui/react": "^3.34.0", "@cruncher/adapter-coralogix": "workspace:*", + "@cruncher/adapter-datadog": "workspace:*", "@cruncher/adapter-docker": "workspace:*", "@cruncher/adapter-grafana-loki-browser": "workspace:*", "@cruncher/adapter-k8s": "workspace:*", diff --git a/apps/cruncher/src/processes/main/main.ts b/apps/cruncher/src/processes/main/main.ts index 66aff07..e8ddae1 100644 --- a/apps/cruncher/src/processes/main/main.ts +++ b/apps/cruncher/src/processes/main/main.ts @@ -191,23 +191,34 @@ function startServerProcess() { const authUrl = msg.authUrl as string; const requestedCookies = msg.cookies as string[]; const jobId = msg.jobId as string; + const scriptExtractors = (msg.scriptExtractors ?? []) as { + key: string; + js: string; + waitForResult?: boolean; + runOnNavigation?: boolean; + }[]; console.log( "Received authentication request from server process, sending cookies...", ); try { - await createAuthWindow(authUrl, requestedCookies, async (cookies) => { - const result = await requestFromServer<{ - type: string; - status: boolean; - }>( - port2, - { type: "authResult", jobId: jobId, cookies: cookies }, - "authResult", - ); - - return result.status; - }); + await createAuthWindow( + authUrl, + requestedCookies, + async (cookies) => { + const result = await requestFromServer<{ + type: string; + status: boolean; + }>( + port2, + { type: "authResult", jobId: jobId, cookies: cookies }, + "authResult", + ); + + return result.status; + }, + scriptExtractors, + ); } catch (error) { if (processActive) { console.error("Error during authentication", error); diff --git a/apps/cruncher/src/processes/main/utils/auth.ts b/apps/cruncher/src/processes/main/utils/auth.ts index 4e643a5..7b20497 100644 --- a/apps/cruncher/src/processes/main/utils/auth.ts +++ b/apps/cruncher/src/processes/main/utils/auth.ts @@ -4,14 +4,20 @@ export const createAuthWindow = async ( url: string, requestedCookies: string[], checkValidCookies: (cookies: Record) => Promise, + scriptExtractors: { + key: string; + js: string; + waitForResult?: boolean; + runOnNavigation?: boolean; + }[] = [], ) => { const authWindow = new BrowserWindow({ width: 400, height: 600, title: "Auth", - show: false, // Start hidden + show: false, webPreferences: { - partition: "persist:auth-fetcher", // Persist cookies + partition: "persist:auth-fetcher", nodeIntegration: false, contextIsolation: true, }, @@ -21,42 +27,124 @@ export const createAuthWindow = async ( console.log("Auth Window created, waiting for login..."); return new Promise((resolve, reject) => { - // add timeout to reject if login takes too long const timeout = setTimeout(() => { console.error("Login timeout, closing auth window..."); - authWindow.webContents.off("did-frame-navigate", eventHandler); // Remove the event listener + authWindow.webContents.off("did-frame-navigate", eventHandler); reject(new Error("Login timeout")); authWindow.close(); - }, 120000); // 120 seconds timeout + }, 120000); + + // Only one eventHandler should proceed to extract + validate. + let settled = false; + + // Values captured from runOnNavigation extractor calls (keyed by extractor key). + // These are merged into the values dict before result-extractor polling starts, + // so the CSRF interceptor result seen during an earlier navigation is reused. + const navCaptured: Record = {}; + + const navigationExtractors = scriptExtractors.filter( + (e) => e.runOnNavigation, + ); + + // Run result extractors, starting from anything already in navCaptured. + const runExtractors = async (values: Record) => { + for (const { key, js, waitForResult } of scriptExtractors) { + if (waitForResult) { + // Prefer a value already captured during a navigation run. + let result = navCaptured[key] ?? ""; + if (!result) { + const deadline = Date.now() + 10_000; + while (!result && Date.now() < deadline) { + try { + const r = await authWindow.webContents.executeJavaScript(js); + if (r) { + result = String(r); + break; + } + } catch { + break; // window destroyed + } + await new Promise((r) => setTimeout(r, 300)); + } + } + values[key] = result; + } else { + try { + const r = await authWindow.webContents.executeJavaScript(js); + if (r != null) values[key] = String(r); + } catch (e) { + console.warn(`Auth extractor "${key}" failed:`, e); + } + } + } + }; const eventHandler = async () => { + // 1. Run setup extractors on every navigation (idempotent interceptor install). + // Also capture any value already available (e.g., CSRF seen in an early call). + for (const { key, js } of navigationExtractors) { + try { + const r = await authWindow.webContents.executeJavaScript(js); + if (r && !navCaptured[key]) navCaptured[key] = String(r); + } catch { + // page may not be ready on very early navigations + } + } + + // 2. Read all session cookies. const session = authWindow.webContents.session; const cookies = await session.cookies.get({ url: url }); - const values = requestedCookies.reduce( - (acc, cookieName) => { - const cookie = cookies.find((c) => c.name === cookieName); - if (cookie) { - acc[cookieName] = cookie.value; - } + const values = cookies.reduce( + (acc, cookie) => { + acc[cookie.name] = cookie.value; return acc; }, {} as Record, ); - const validatedCookies = await checkValidCookies(values); - if (validatedCookies) { + // 3. Quick LOCAL check (no IPC round-trip): are the required session cookies + // present? If not, show the window so the user can log in. + const hasCookies = + requestedCookies.length === 0 || + requestedCookies.some((k) => !!values[k]); + + if (!hasCookies) { + authWindow.show(); + return; + } + + // 4. Session cookies look good — claim ownership. + if (settled) return; + settled = true; + authWindow.webContents.off("did-frame-navigate", eventHandler); + clearTimeout(timeout); + + // 5. Merge anything already captured during navigation events. + Object.assign(values, navCaptured); + + // 6. Run result extractors BEFORE calling the server so the enriched dict + // (including the CSRF token) is what gets sent over IPC and ultimately + // returned from getCookies(). + await runExtractors(values); + + // 7. Validate with server — values now contains both cookies AND extractor results. + const valid = await checkValidCookies(values); + if (valid) { console.log("Login successful, capturing cookies..."); - authWindow.webContents.off("did-frame-navigate", eventHandler); // Remove the event listener - clearTimeout(timeout); // Clear the timeout since we have a valid session - authWindow.close(); + if (!authWindow.isDestroyed()) authWindow.close(); resolve(); } else { - console.info("Cookies not found - prompting user to login again."); - authWindow.show(); // Show the window to prompt user to login again + // Validation failed after enrichment (shouldn't happen if hasCookies passed, + // but handle gracefully by resetting and letting the user retry). + console.warn( + "Auth: server rejected enriched cookies, showing window for retry.", + ); + settled = false; + authWindow.webContents.on("did-frame-navigate", eventHandler); + authWindow.show(); } }; - // OPTIONAL: Detect login completion by checking URL change authWindow.webContents.on("did-frame-navigate", eventHandler); }); }; diff --git a/apps/cruncher/src/processes/server/externalAuthProvider.ts b/apps/cruncher/src/processes/server/externalAuthProvider.ts index 6b2be53..8d97250 100644 --- a/apps/cruncher/src/processes/server/externalAuthProvider.ts +++ b/apps/cruncher/src/processes/server/externalAuthProvider.ts @@ -1,4 +1,4 @@ -import { ExternalAuthProvider } from "@cruncher/adapter-utils"; +import { ExternalAuthProvider, ScriptExtractor } from "@cruncher/adapter-utils"; import { IPCMessage } from "./types"; export class ElectronExternalAuthProvider implements ExternalAuthProvider { @@ -7,6 +7,7 @@ export class ElectronExternalAuthProvider implements ExternalAuthProvider { requestedUrl: string, cookies: string[], validate: (cookies: Record) => Promise, + scriptExtractors?: ScriptExtractor[], ): Promise> { return new Promise((resolve, reject) => { const jobId = crypto.randomUUID(); @@ -58,6 +59,7 @@ export class ElectronExternalAuthProvider implements ExternalAuthProvider { authUrl: requestedUrl, jobId: jobId, cookies, + scriptExtractors: scriptExtractors ?? [], }); }); } diff --git a/apps/cruncher/src/processes/server/main.ts b/apps/cruncher/src/processes/server/main.ts index e5bcfdd..b2a6b1e 100644 --- a/apps/cruncher/src/processes/server/main.ts +++ b/apps/cruncher/src/processes/server/main.ts @@ -9,6 +9,7 @@ import loki from "@cruncher/adapter-loki"; import grafanaLokiBrowser from "@cruncher/adapter-grafana-loki-browser"; import mock from "@cruncher/adapter-mock"; import coralogix from "@cruncher/adapter-coralogix"; +import datadog from "@cruncher/adapter-datadog"; import { Engine } from "./engineV2/engine"; import { DefaultExternalAuthProvider, @@ -120,6 +121,7 @@ const initializeServer = async (authProvider: ExternalAuthProvider) => { engineV2.registerPlugin(docker); engineV2.registerPlugin(k8s); engineV2.registerPlugin(coralogix); + engineV2.registerPlugin(datadog); // const routes = await getRoutes(engineV2); // await setupEngine(serverContainer, routes); diff --git a/docs/src/content/docs/adapters/datadog.mdx b/docs/src/content/docs/adapters/datadog.mdx new file mode 100644 index 0000000..beab12d --- /dev/null +++ b/docs/src/content/docs/adapters/datadog.mdx @@ -0,0 +1,82 @@ +--- +title: Datadog +description: How to use Cruncher with Datadog Logs. +--- +import ParamItem from '../../../components/ParamItem.astro'; + +import { Badge } from '@astrojs/starlight/components'; + + + +## Params + + + The Datadog site to connect to. Choose the site that matches your Datadog organization. + + Supported values: + - `datadoghq.com` — US1 (default) + - `us3.datadoghq.com` — US3 + - `us5.datadoghq.com` — US5 + - `datadoghq.eu` — EU1 + - `ap1.datadoghq.com` — AP1 + + + + List of Datadog log index names to query. An empty list (the default) searches across all indexes (`*`). + + + + A Lucene query string that is always prepended to every search. Useful for scoping a connector to a specific service, environment, or team without requiring users to type it every time. + + Example: `service:my-service AND env:production` + + +## Examples + +### Minimal Configuration + +```yaml +connectors: + - type: datadog + name: my_datadog +``` + +### With a Specific Site + +```yaml +connectors: + - type: datadog + name: my_datadog_eu + params: + site: "datadoghq.eu" +``` + +### Scoped to Specific Indexes and a Query Prefix + +```yaml +connectors: + - type: datadog + name: my_datadog + params: + site: "datadoghq.com" + indexes: + - "main" + - "prod-logs" + query_prefix: "env:production" +``` + +## Authentication + +The Datadog adapter uses **browser-session authentication**. When you run a query for the first time, Cruncher opens a browser window pointing to your Datadog site. Log in as usual (including SSO/SAML if required), and Cruncher will automatically capture the session cookies and CSRF token it needs. + +Once authenticated, the session is reused for all subsequent queries. If the session expires, Cruncher will automatically re-open the login window. + +## Facet Autocomplete + +Cruncher automatically fetches your Datadog log facets and index names to provide autocomplete suggestions in the query editor. Facets with a fixed set of values (e.g. `status`) are resolved directly; facets with dynamic values (e.g. `kube_namespace`) are populated by querying the top 10 most frequent values from the last 15 minutes. + +## Usage Notes + +- The `indexes` param scopes queries to specific log indexes. Leave it empty to search all indexes. +- The `query_prefix` is combined with the QQL search term using `AND`, so it always applies. +- The `index` controller param (set inline in the QQL query bar) overrides the `indexes` config param for that query. diff --git a/packages/adapters/datadog/package.json b/packages/adapters/datadog/package.json new file mode 100644 index 0000000..3a7f767 --- /dev/null +++ b/packages/adapters/datadog/package.json @@ -0,0 +1,51 @@ +{ + "name": "@cruncher/adapter-datadog", + "type": "module", + "version": "0.1.0", + "repository": { + "type": "git", + "url": "https://github.com/IamShobe/cruncher" + }, + "description": "Datadog adapter for Cruncher.", + "publishConfig": { + "access": "public" + }, + "main": "dist/index.js", + "types": "dist/index.d.ts", + "exports": { + ".": { + "default": "./dist/index.js", + "types": "./dist/index.d.ts" + } + }, + "files": [ + "dist" + ], + "scripts": { + "build": "tsc --project tsconfig.json", + "format": "oxfmt src/", + "format:check": "oxfmt --check src/", + "prepublishOnly": "npm run build" + }, + "keywords": [ + "cruncher", + "qql", + "query-language", + "datadog", + "adapter" + ], + "author": { + "name": "Elran Shefer" + }, + "license": "GPL-3.0-only", + "dependencies": { + "@cruncher/adapter-utils": "workspace:*", + "@cruncher/qql": "workspace:*", + "async-mutex": "^0.5.0", + "zod": "^4.3.6" + }, + "devDependencies": { + "@types/node": "^25.3.5", + "typescript": "^5.9.3" + } +} diff --git a/packages/adapters/datadog/src/controller.ts b/packages/adapters/datadog/src/controller.ts new file mode 100644 index 0000000..0cdb3dd --- /dev/null +++ b/packages/adapters/datadog/src/controller.ts @@ -0,0 +1,395 @@ +import { + AdapterContext, + QueryOptions, + QueryProvider, +} from "@cruncher/adapter-utils"; +import { ControllerIndexParam, Search } from "@cruncher/qql/grammar"; +import { Mutex } from "async-mutex"; +import { z } from "zod/v4"; +import type { DatadogParams } from "."; +import { getAppUrl } from "."; +import { + buildDatadogQuery, + buildRequestPayload, + processDatadogResponse, + DatadogLogsResponse, +} from "./query"; + +// --------------------------------------------------------------------------- +// Response schemas for getControllerParams +// --------------------------------------------------------------------------- + +const IndexesResponseSchema = z.object({ + indexes: z.array(z.object({ name: z.string() })).optional(), +}); + +const FacetValueSchema = z.object({ + id: z.string().optional(), + name: z.string().optional(), +}); + +const FacetEntrySchema = z.object({ + path: z.string().optional(), + name: z.string().optional(), + values: z.array(FacetValueSchema).optional(), +}); + +const FacetsResponseSchema = z.object({ + facets: z.record(z.string(), z.array(FacetEntrySchema)).optional(), +}); + +// aggregate endpoint: { result: { buckets: [{ by: { "": "" } }] } } +const AggregateBucketSchema = z.object({ + by: z.record(z.string(), z.string()), +}); + +const AggregateResponseSchema = z.object({ + result: z + .object({ + buckets: z.array(AggregateBucketSchema).optional(), + }) + .optional(), +}); + +// --------------------------------------------------------------------------- +// CSRF interception +// --------------------------------------------------------------------------- + +// Key used in the cookies dict to carry the CSRF token extracted from the page. +const CSRF_KEY = "__dd_csrf__"; + +// JS snippet injected into the auth window to capture the CSRF token from +// Datadog's own outgoing requests. It is idempotent (safe to run on every +// navigation) and works with both fetch and XMLHttpRequest. +// +// runOnNavigation:true → auth.ts runs this on every did-frame-navigate so +// interceptors are in place before the SPA fires its initial API calls. +// waitForResult:true → auth.ts polls every 300 ms (up to 10 s) until a +// truthy value is returned. +const CSRF_JS = ` + (function() { + if (window.__capturedCsrf) return window.__capturedCsrf; + + if (!window.__csrfInterceptorInstalled) { + window.__csrfInterceptorInstalled = true; + + var _fetch = window.fetch; + window.fetch = function(input, init) { + try { + var h = init && init.headers; + if (h) { + var v = (h instanceof Headers) + ? (h.get('x-csrf-token') || h.get('X-CSRF-Token')) + : (h['x-csrf-token'] || h['X-Csrf-Token'] || h['X-CSRF-Token']); + if (v) window.__capturedCsrf = v; + } + } catch(e) {} + return _fetch.apply(this, arguments); + }; + + var _setHeader = XMLHttpRequest.prototype.setRequestHeader; + XMLHttpRequest.prototype.setRequestHeader = function(name, value) { + if (name && name.toLowerCase() === 'x-csrf-token' && value) + window.__capturedCsrf = value; + return _setHeader.apply(this, arguments); + }; + } + + return window.__capturedCsrf || ''; + })() +`.trim(); + +// --------------------------------------------------------------------------- +// Controller +// --------------------------------------------------------------------------- + +// The primary session cookie set by Datadog after a successful SSO login. +const SESSION_COOKIE = "dogweb"; + +// Module-level mutex ensures only one auth window opens at a time across all +// controller instances (e.g. if multiple queries are fired concurrently). +const mutex = new Mutex(); + +type RequestInitModified = Omit & { + headers?: Record; +}; + +type SessionState = { + all: Record; // all session cookies + csrf: string; // CSRF token captured from the page +}; + +export class DatadogController implements QueryProvider { + private session: SessionState | null = null; + + constructor( + private ctx: AdapterContext, + private params: DatadogParams, + ) {} + + async getControllerParams(): Promise> { + const appUrl = getAppUrl(this.params.site); + const result: Record = {}; + + // Phase 1: fetch index names and facet metadata (path + pre-defined values). + // Facets with bounded enum values (e.g. "status") are fully resolved here. + // Facets with dynamic values (e.g. "kube_namespace") get an empty array for + // now — phase 2 will fill them in via aggregate queries. + const dynamicFacetPaths: string[] = []; + + await Promise.all([ + // Fetch available log index names. + this._fetchWrapper(`${appUrl}/api/v1/logs/indexes`) + .then(async (r) => { + if (!r.ok) return; + const parsed = IndexesResponseSchema.safeParse(await r.json()); + if (!parsed.success) return; + const names = (parsed.data.indexes ?? []) + .map((i) => i.name) + .filter(Boolean); + if (names.length) result.index = names; + }) + .catch(() => {}), + + // Fetch configured log facets. + // Response shape: { facets: { [category]: FacetEntry[] } } + this._fetchWrapper( + `${appUrl}/api/ui/event-platform/logs/facets?type=logs`, + ) + .then(async (r) => { + if (!r.ok) return; + const parsed = FacetsResponseSchema.safeParse(await r.json()); + if (!parsed.success) return; + for (const category of Object.values(parsed.data.facets ?? {})) { + for (const facet of category) { + const key = facet.path ?? facet.name; + if (!key || key in result) continue; + const predefined = (facet.values ?? []) + .map((v) => v.id ?? v.name ?? "") + .filter(Boolean); + result[key] = predefined; + if (predefined.length === 0) dynamicFacetPaths.push(key); + } + } + }) + .catch(() => {}), + ]); + + // Phase 2: for facets with no pre-defined values, query the aggregate + // endpoint to surface the most common values from recent logs. + const csrf = await this.getCsrf(); + await Promise.all( + dynamicFacetPaths.map((path) => + this._fetchFacetTopValues(appUrl, path, csrf) + .then((values) => { + if (values.length) result[path] = values; + }) + .catch(() => {}), + ), + ); + + return result; + } + + // Queries the aggregate endpoint for the top-N most frequent values of a + // single facet over the last 15 minutes. Returns [] on any failure. + private async _fetchFacetTopValues( + appUrl: string, + facetPath: string, + csrfToken: string, + ): Promise { + const now = Date.now(); + const from = now - 15 * 60 * 1000; + + const response = await this._fetchWrapper( + `${appUrl}/api/v1/logs-analytics/aggregate?type=logs`, + { + method: "POST", + body: JSON.stringify({ + aggregate: { + groupBy: [ + { + facet: facetPath, + limit: 10, + sort: { aggregation: "count", order: "desc" }, + }, + ], + compute: [{ aggregation: "count" }], + search: { query: "*" }, + time: { from, to: now }, + indexes: this.params.indexes?.length ? this.params.indexes : ["*"], + }, + querySourceId: "logs_explorer", + _authentication_token: csrfToken, + }), + }, + ); + + if (!response.ok) { + console.error( + `[datadog] aggregate for ${facetPath} returned ${response.status}`, + ); + return []; + } + + const raw = (await response.json()) as unknown; + console.log( + `[datadog] aggregate response for ${facetPath}:`, + JSON.stringify(raw), + ); + + const parsed = AggregateResponseSchema.safeParse(raw); + if (!parsed.success) return []; + + return (parsed.data.result?.buckets ?? []) + .map((b) => b.by[facetPath] ?? "") + .filter(Boolean); + } + + async query( + controllerParams: ControllerIndexParam[], + searchTerm: Search, + queryOptions: QueryOptions, + ): Promise { + const { fromTime, toTime, limit, cancelToken, onBatchDone } = queryOptions; + const apiUrl = `${getAppUrl(this.params.site)}/api/v1/logs-analytics/list?type=logs`; + + // Extract `index` params and route them to the payload's `indexes` field. + // All other params are serialized into the Lucene query string. + const indexValues = controllerParams + .filter((p) => p.name === "index" && p.operator === "=") + .map( + (p) => (p.value.type === "regex" ? p.value.pattern : p.value) as string, + ); + const otherParams = controllerParams.filter((p) => p.name !== "index"); + + const luceneQuery = buildDatadogQuery( + otherParams, + searchTerm, + this.params.query_prefix, + ); + + // User-specified indexes override the profile-level defaults. + const indexes = indexValues.length > 0 ? indexValues : this.params.indexes; + + let cursor: string | undefined; + let totalProcessed = 0; + + while (true) { + if (cancelToken.aborted) break; + const pageLimit = Math.min(1000, limit - totalProcessed); + // Ensure session is established before building the payload so the CSRF + // token is always fresh (getCsrf() calls _ensureSession internally). + const csrf = await this.getCsrf(); + const payload = buildRequestPayload( + luceneQuery, + indexes, + fromTime, + toTime, + pageLimit, + csrf, + cursor, + ); + + const response = await this._fetchWrapper(apiUrl, { + method: "POST", + body: JSON.stringify(payload), + signal: cancelToken, + }); + + if (!response.ok) { + throw new Error( + `Datadog query failed: ${response.status} ${response.statusText}`, + ); + } + + const data: DatadogLogsResponse = await response.json(); + const logs = processDatadogResponse(data); + totalProcessed += logs.length; + onBatchDone(logs); + + cursor = data.result?.nextPageToken; + if (!cursor || logs.length === 0 || totalProcessed >= limit) break; + } + } + + private async getCsrf(): Promise { + await this._ensureSession(); + return this.session!.csrf; + } + + private _ensureSession = async (): Promise => { + await mutex.runExclusive(async () => { + if (this.session) return; + + const allCookies = await this.ctx.externalAuthProvider.getCookies( + getAppUrl(this.params.site), + [SESSION_COOKIE], + (cookies) => Promise.resolve(!!cookies[SESSION_COOKIE]), + [ + { + key: CSRF_KEY, + js: CSRF_JS, + waitForResult: true, + runOnNavigation: true, + }, + ], + ); + + const csrf = allCookies[CSRF_KEY] ?? ""; + delete allCookies[CSRF_KEY]; + this.session = { all: allCookies, csrf }; + }); + }; + + private _fetchWrapper = async ( + url: string, + options: RequestInitModified = {}, + retryAmount: number = 0, + ): ReturnType => { + if (retryAmount > 2) { + throw new Error("Failed to authenticate after multiple attempts"); + } + + await this._ensureSession(); + + const appUrl = getAppUrl(this.params.site); + const headers = options.headers ?? {}; + + // Attach session cookies and required headers for Datadog's CSRF protection. + headers["Cookie"] = Object.entries(this.session!.all) + .map(([k, v]) => `${k}=${v}`) + .join("; "); + headers["X-CSRF-Token"] = this.session!.csrf; + headers["Content-Type"] = "application/json"; + // Rails CSRF validation requires Origin to match the expected app host. + headers["Origin"] = appUrl; + headers["Referer"] = `${appUrl}/logs`; + options.headers = headers; + + // On retry the session CSRF may have changed — keep the request body in sync. + if (options.body && typeof options.body === "string") { + try { + const bodyObj = JSON.parse(options.body) as Record; + if ("_authentication_token" in bodyObj) { + bodyObj._authentication_token = this.session!.csrf; + options.body = JSON.stringify(bodyObj); + } + } catch { + // body wasn't JSON — leave as-is + } + } + + const response = await fetch(url, options); + + if (response.status === 401 || response.status === 403) { + // Session has expired — clear it so _ensureSession re-authenticates. + await mutex.runExclusive(async () => { + this.session = null; + }); + return this._fetchWrapper(url, options, retryAmount + 1); + } + + return response; + }; +} diff --git a/packages/adapters/datadog/src/index.ts b/packages/adapters/datadog/src/index.ts new file mode 100644 index 0000000..7be75a3 --- /dev/null +++ b/packages/adapters/datadog/src/index.ts @@ -0,0 +1,38 @@ +import { z } from "zod/v4"; +import { Adapter, newPluginRef, QueryProvider } from "@cruncher/adapter-utils"; +import { DatadogController } from "./controller"; + +const DD_SITES = [ + "datadoghq.com", // US1 — app.datadoghq.com + "us3.datadoghq.com", // US3 + "us5.datadoghq.com", // US5 + "datadoghq.eu", // EU1 — app.datadoghq.eu + "ap1.datadoghq.com", // AP1 +] as const; + +const paramsSchema = z.object({ + site: z.enum(DD_SITES).default("datadoghq.com"), + indexes: z.array(z.string()).default([]), // empty = all indexes ("*") + query_prefix: z.string().default(""), // always-applied Lucene filter +}); + +export type DatadogParams = z.infer; + +/** Returns the web app base URL for a given Datadog site. */ +export function getAppUrl(site: string): string { + if (site === "datadoghq.com") return "https://app.datadoghq.com"; + if (site === "datadoghq.eu") return "https://app.datadoghq.eu"; + return `https://${site}`; +} + +const adapter: Adapter = { + ref: newPluginRef("datadog"), + name: "Datadog Adapter", + description: "Adapter for Datadog Logs", + version: "0.1.0", + params: paramsSchema, + factory: (ctx, { params }): QueryProvider => + new DatadogController(ctx, paramsSchema.parse(params)), +}; + +export default adapter; diff --git a/packages/adapters/datadog/src/query.ts b/packages/adapters/datadog/src/query.ts new file mode 100644 index 0000000..6e9a729 --- /dev/null +++ b/packages/adapters/datadog/src/query.ts @@ -0,0 +1,204 @@ +import { ControllerIndexParam, Search } from "@cruncher/qql/grammar"; +import { + ProcessedData, + ObjectFields, + processField, +} from "@cruncher/adapter-utils/logTypes"; +import { + buildSearchTreeCallback, + SearchTreeBuilder, +} from "@cruncher/qql/searchTree"; + +// --------------------------------------------------------------------------- +// API types +// Response shape for app.datadoghq.com/api/v1/logs-analytics/list?type=logs +// --------------------------------------------------------------------------- + +// Columns returned per event, matching the order requested in buildRequestPayload: +// [0] status_line [1] timestamp [2] host [3] service [4] content +type EventColumns = [unknown, unknown, unknown, unknown, unknown]; + +export type DatadogEvent = { + id?: string; + event_id?: string; + columns: EventColumns; + event?: Record; +}; + +export type DatadogLogsResponse = { + result?: { + events?: DatadogEvent[]; + nextPageToken?: string; + }; + hitCount?: number; +}; + +// --------------------------------------------------------------------------- +// Query building +// --------------------------------------------------------------------------- + +function buildSearchPattern(search: Search): string { + const luceneBuilder: SearchTreeBuilder = { + buildAnd: (leftCallback, searchAnd) => (item: string) => { + const left = leftCallback(item); + const right = buildSearchTreeCallback( + searchAnd.right, + luceneBuilder, + )(item); + return [left, right].filter(Boolean).join(" AND "); + }, + buildOr: (leftCallback, searchOr) => (item: string) => { + const left = leftCallback(item); + const right = buildSearchTreeCallback( + searchOr.right, + luceneBuilder, + )(item); + return [left, right].filter(Boolean).join(" OR "); + }, + buildLiteral: (searchLiteral) => () => { + if (!searchLiteral.tokens.length) return ""; + return searchLiteral.tokens + .map((token) => `"${String(token)}"`) + .join(" AND "); + }, + }; + return buildSearchTreeCallback(search, luceneBuilder)(""); +} + +export function buildDatadogQuery( + controllerParams: ControllerIndexParam[], + searchTerm: Search, + queryPrefix: string, +): string { + const paramParts = controllerParams.map((param) => { + const val = + param.value.type === "regex" ? param.value.pattern : param.value; + if (param.operator === "=") return `${param.name}:"${val}"`; + if (param.operator === "!=") return `NOT ${param.name}:"${val}"`; + return ""; + }); + + const searchText = buildSearchPattern(searchTerm); + return [queryPrefix, ...paramParts, searchText].filter(Boolean).join(" AND "); +} + +// --------------------------------------------------------------------------- +// Request payload +// --------------------------------------------------------------------------- + +export function buildRequestPayload( + query: string, + indexes: string[], + fromTime: Date, + toTime: Date, + pageLimit: number, + csrfToken: string, + cursor?: string, +) { + return { + list: { + // Column order determines the indices in DatadogEvent.columns. + columns: [ + { field: { path: "status_line" } }, + { field: { path: "timestamp" } }, + { field: { path: "host" } }, + { field: { path: "service" } }, + { field: { path: "content" } }, + ], + sorts: [{ time: { order: "desc" } }], + limit: pageLimit, + time: { + from: fromTime.getTime(), + to: toTime.getTime(), + }, + search: { query }, + includeEvents: true, + computeCount: false, + indexes: indexes.length > 0 ? indexes : ["*"], + executionInfo: {}, + ...(cursor ? { startAt: cursor } : {}), + }, + querySourceId: "logs_explorer", + userQueryId: crypto.randomUUID(), + _authentication_token: csrfToken, + }; +} + +// --------------------------------------------------------------------------- +// Response processing +// --------------------------------------------------------------------------- + +export function processDatadogResponse( + data: DatadogLogsResponse, +): ProcessedData[] { + const events = data.result?.events ?? []; + + return events.map((entry) => { + // Destructure columns in the order they were requested. + const [statusLine, rawTimestamp, host, service] = entry.columns; + + const ts = + typeof rawTimestamp === "number" + ? rawTimestamp + : typeof rawTimestamp === "string" + ? new Date(rawTimestamp).getTime() + : Date.now(); + + const uniqueId = entry.id ?? entry.event_id ?? ""; + const eventData = entry.event ?? {}; + + const message = + typeof eventData["message"] === "string" + ? eventData["message"] + : typeof eventData["msg"] === "string" + ? eventData["msg"] + : JSON.stringify(eventData); + + const objectFields: ObjectFields = { + _time: { type: "date", value: ts }, + _sortBy: { type: "number", value: ts }, + _uniqueId: { type: "string", value: uniqueId }, + id: { type: "string", value: uniqueId }, + _raw: { type: "string", value: JSON.stringify(entry) }, + }; + + if (statusLine != null) + objectFields.status = { type: "string", value: String(statusLine) }; + if (host != null) + objectFields.host = { type: "string", value: String(host) }; + if (service != null) + objectFields.service = { type: "string", value: String(service) }; + + // If the message is a JSON object, flatten its fields into objectFields. + if (typeof message === "string") { + try { + const parsed = JSON.parse(message); + if ( + parsed !== null && + typeof parsed === "object" && + !Array.isArray(parsed) + ) { + for (const [key, value] of Object.entries( + parsed as Record, + )) { + if (!(key in objectFields)) { + objectFields[key] = processField(value); + } + } + } + } catch { + // message wasn't JSON — keep as-is + } + } + + // Flatten all remaining event fields, skipping already-mapped ones. + const skip = new Set(["message", "msg"]); + for (const [key, value] of Object.entries(eventData)) { + if (!skip.has(key) && !(key in objectFields)) { + objectFields[key] = processField(value); + } + } + + return { object: objectFields, message }; + }); +} diff --git a/packages/adapters/datadog/tsconfig.json b/packages/adapters/datadog/tsconfig.json new file mode 100644 index 0000000..f0ecb97 --- /dev/null +++ b/packages/adapters/datadog/tsconfig.json @@ -0,0 +1,15 @@ +{ + "extends": "../../../tsconfig.modules.json", + "compilerOptions": { + "baseUrl": ".", + "outDir": "dist", + "rootDir": "src" + }, + "include": [ + "src/**/*" + ], + "exclude": [ + "dist", + "node_modules" + ] +} diff --git a/packages/adapters/utils/src/index.ts b/packages/adapters/utils/src/index.ts index 6dab90b..c9e3e2c 100644 --- a/packages/adapters/utils/src/index.ts +++ b/packages/adapters/utils/src/index.ts @@ -63,11 +63,25 @@ export type FactoryParams = { export type PluginRef = Brand; // A unique identifier for a plugin +export type ScriptExtractor = { + key: string; // key added to the returned cookies dict + js: string; // JS expression evaluated in the auth window; result must be a string + // If true, auth.ts retries the JS expression every 300 ms (up to 10 s) until it + // returns a truthy value. Use this when the value is set asynchronously by the + // page's JavaScript (e.g. React SPA initialisation, dynamic CSRF token injection). + waitForResult?: boolean; + // If true, the JS expression is also executed on every did-frame-navigate event + // (before cookie validation). Use this to install interceptors early so they are + // in place before the SPA makes its initial authenticated API calls. + runOnNavigation?: boolean; +}; + export type ExternalAuthProvider = { getCookies( requestedUrl: string, cookies: string[], validate: (cookies: Record) => Promise, + scriptExtractors?: ScriptExtractor[], ): Promise>; }; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 68e6e28..eb7ab03 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -71,6 +71,9 @@ importers: '@cruncher/adapter-coralogix': specifier: workspace:* version: link:../../packages/adapters/coralogix + '@cruncher/adapter-datadog': + specifier: workspace:* + version: link:../../packages/adapters/datadog '@cruncher/adapter-docker': specifier: workspace:* version: link:../../packages/adapters/docker @@ -410,6 +413,28 @@ importers: specifier: ^5.9.3 version: 5.9.3 + packages/adapters/datadog: + dependencies: + '@cruncher/adapter-utils': + specifier: workspace:* + version: link:../utils + '@cruncher/qql': + specifier: workspace:* + version: link:../../qql + async-mutex: + specifier: ^0.5.0 + version: 0.5.0 + zod: + specifier: ^4.3.6 + version: 4.3.6 + devDependencies: + '@types/node': + specifier: ^25.3.5 + version: 25.3.5 + typescript: + specifier: ^5.9.3 + version: 5.9.3 + packages/adapters/docker: dependencies: '@cruncher/adapter-utils': diff --git a/scripts/npm-placeholder-publish.sh b/scripts/npm-placeholder-publish.sh new file mode 100755 index 0000000..04e2c77 --- /dev/null +++ b/scripts/npm-placeholder-publish.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +set -euo pipefail + +usage() { + echo "Usage: $0 [description]" + echo " Publishes a minimal placeholder to npm to claim the package name." + echo " After publishing, configure a trusted publisher at:" + echo " https://www.npmjs.com/package/" + exit 1 +} + +NAME="${1:-}" +DESCRIPTION="${2:-}" + +[ -z "$NAME" ] && usage + +[ -z "$DESCRIPTION" ] && DESCRIPTION="Placeholder package for ${NAME}" + +TMPDIR="$(mktemp -d)" +trap 'rm -rf "$TMPDIR"' EXIT + +cd "$TMPDIR" + +cat > package.json << EOF +{ + "name": "${NAME}", + "version": "0.0.1", + "description": "${DESCRIPTION}", + "main": "index.js", + "license": "GPL-3.0-only", + "publishConfig": { "access": "public" }, + "repository": { + "type": "git", + "url": "https://github.com/IamShobe/cruncher" + } +} +EOF + +echo "// placeholder" > index.js + +echo "Publishing placeholder for ${NAME}..." +npm publish --access public + +echo "" +echo "Done! Configure trusted publisher at:" +echo " https://www.npmjs.com/package/${NAME}"