feat: Datadog adapter

.changeset/datadog-adapter-init.md

@@ -0,0 +1,11 @@
+---
+"@cruncher/adapter-datadog": minor
+---
+
+Add Datadog adapter with browser-session auth, query support, and facet/index autocomplete.
+
+- Cookie + CSRF-token based authentication via Electron auth window
+- Translates QQL queries to Datadog Lucene syntax
+- Fetches log index names for `index=` autocomplete
+- Fetches facet metadata from `/api/ui/event-platform/logs/facets` for param name suggestions
+- Aggregates top values per dynamic facet for param value suggestions

apps/cruncher/package.json

@@ -67,6 +67,7 @@
     "@babel/runtime": "^7.28.6",
     "@chakra-ui/react": "^3.34.0",
     "@cruncher/adapter-coralogix": "workspace:*",
+    "@cruncher/adapter-datadog": "workspace:*",
     "@cruncher/adapter-docker": "workspace:*",
     "@cruncher/adapter-grafana-loki-browser": "workspace:*",
     "@cruncher/adapter-k8s": "workspace:*",

apps/cruncher/src/processes/main/main.ts

@@ -191,23 +191,34 @@ function startServerProcess() {
       const authUrl = msg.authUrl as string;
       const requestedCookies = msg.cookies as string[];
       const jobId = msg.jobId as string;
+      const scriptExtractors = (msg.scriptExtractors ?? []) as {
+        key: string;
+        js: string;
+        waitForResult?: boolean;
+        runOnNavigation?: boolean;
+      }[];
       console.log(
         "Received authentication request from server process, sending cookies...",
       );
 
       try {
-        await createAuthWindow(authUrl, requestedCookies, async (cookies) => {
-          const result = await requestFromServer<{
-            type: string;
-            status: boolean;
-          }>(
-            port2,
-            { type: "authResult", jobId: jobId, cookies: cookies },
-            "authResult",
-          );
-
-          return result.status;
-        });
+        await createAuthWindow(
+          authUrl,
+          requestedCookies,
+          async (cookies) => {
+            const result = await requestFromServer<{
+              type: string;
+              status: boolean;
+            }>(
+              port2,
+              { type: "authResult", jobId: jobId, cookies: cookies },
+              "authResult",
+            );
+
+            return result.status;
+          },
+          scriptExtractors,
+        );
       } catch (error) {
         if (processActive) {
           console.error("Error during authentication", error);

apps/cruncher/src/processes/main/utils/auth.ts

@@ -4,14 +4,20 @@ export const createAuthWindow = async (
   url: string,
   requestedCookies: string[],
   checkValidCookies: (cookies: Record<string, string>) => Promise<boolean>,
+  scriptExtractors: {
+    key: string;
+    js: string;
+    waitForResult?: boolean;
+    runOnNavigation?: boolean;
+  }[] = [],
 ) => {
   const authWindow = new BrowserWindow({
     width: 400,
     height: 600,
     title: "Auth",
-    show: false, // Start hidden
+    show: false,
     webPreferences: {
-      partition: "persist:auth-fetcher", // Persist cookies
+      partition: "persist:auth-fetcher",
       nodeIntegration: false,
       contextIsolation: true,
     },
@@ -21,42 +27,124 @@ export const createAuthWindow = async (
 
   console.log("Auth Window created, waiting for login...");
   return new Promise<void>((resolve, reject) => {
-    // add timeout to reject if login takes too long
     const timeout = setTimeout(() => {
       console.error("Login timeout, closing auth window...");
-      authWindow.webContents.off("did-frame-navigate", eventHandler); // Remove the event listener
+      authWindow.webContents.off("did-frame-navigate", eventHandler);
       reject(new Error("Login timeout"));
       authWindow.close();
-    }, 120000); // 120 seconds timeout
+    }, 120000);
+
+    // Only one eventHandler should proceed to extract + validate.
+    let settled = false;
+
+    // Values captured from runOnNavigation extractor calls (keyed by extractor key).
+    // These are merged into the values dict before result-extractor polling starts,
+    // so the CSRF interceptor result seen during an earlier navigation is reused.
+    const navCaptured: Record<string, string> = {};
+
+    const navigationExtractors = scriptExtractors.filter(
+      (e) => e.runOnNavigation,
+    );
+
+    // Run result extractors, starting from anything already in navCaptured.
+    const runExtractors = async (values: Record<string, string>) => {
+      for (const { key, js, waitForResult } of scriptExtractors) {
+        if (waitForResult) {
+          // Prefer a value already captured during a navigation run.
+          let result = navCaptured[key] ?? "";
+          if (!result) {
+            const deadline = Date.now() + 10_000;
+            while (!result && Date.now() < deadline) {
+              try {
+                const r = await authWindow.webContents.executeJavaScript(js);
+                if (r) {
+                  result = String(r);
+                  break;
+                }
+              } catch {
+                break; // window destroyed
+              }
+              await new Promise<void>((r) => setTimeout(r, 300));
+            }
+          }
+          values[key] = result;
+        } else {
+          try {
+            const r = await authWindow.webContents.executeJavaScript(js);
+            if (r != null) values[key] = String(r);
+          } catch (e) {
+            console.warn(`Auth extractor "${key}" failed:`, e);
+          }
+        }
+      }
+    };
 
     const eventHandler = async () => {
+      // 1. Run setup extractors on every navigation (idempotent interceptor install).
+      //    Also capture any value already available (e.g., CSRF seen in an early call).
+      for (const { key, js } of navigationExtractors) {
+        try {
+          const r = await authWindow.webContents.executeJavaScript(js);
+          if (r && !navCaptured[key]) navCaptured[key] = String(r);
+        } catch {
+          // page may not be ready on very early navigations
+        }
+      }
+
+      // 2. Read all session cookies.
       const session = authWindow.webContents.session;
       const cookies = await session.cookies.get({ url: url });
-      const values = requestedCookies.reduce(
-        (acc, cookieName) => {
-          const cookie = cookies.find((c) => c.name === cookieName);
-          if (cookie) {
-            acc[cookieName] = cookie.value;
-          }
+      const values = cookies.reduce(
+        (acc, cookie) => {
+          acc[cookie.name] = cookie.value;
           return acc;
         },
         {} as Record<string, string>,
       );
 
-      const validatedCookies = await checkValidCookies(values);
-      if (validatedCookies) {
+      // 3. Quick LOCAL check (no IPC round-trip): are the required session cookies
+      //    present? If not, show the window so the user can log in.
+      const hasCookies =
+        requestedCookies.length === 0 ||
+        requestedCookies.some((k) => !!values[k]);
+
+      if (!hasCookies) {
+        authWindow.show();
+        return;
+      }
+
+      // 4. Session cookies look good — claim ownership.
+      if (settled) return;
+      settled = true;
+      authWindow.webContents.off("did-frame-navigate", eventHandler);
+      clearTimeout(timeout);
+
+      // 5. Merge anything already captured during navigation events.
+      Object.assign(values, navCaptured);
+
+      // 6. Run result extractors BEFORE calling the server so the enriched dict
+      //    (including the CSRF token) is what gets sent over IPC and ultimately
+      //    returned from getCookies().
+      await runExtractors(values);
+
+      // 7. Validate with server — values now contains both cookies AND extractor results.
+      const valid = await checkValidCookies(values);
+      if (valid) {
         console.log("Login successful, capturing cookies...");
-        authWindow.webContents.off("did-frame-navigate", eventHandler); // Remove the event listener
-        clearTimeout(timeout); // Clear the timeout since we have a valid session
-        authWindow.close();
+        if (!authWindow.isDestroyed()) authWindow.close();
         resolve();
       } else {
-        console.info("Cookies not found - prompting user to login again.");
-        authWindow.show(); // Show the window to prompt user to login again
+        // Validation failed after enrichment (shouldn't happen if hasCookies passed,
+        // but handle gracefully by resetting and letting the user retry).
+        console.warn(
+          "Auth: server rejected enriched cookies, showing window for retry.",
+        );
+        settled = false;
+        authWindow.webContents.on("did-frame-navigate", eventHandler);
+        authWindow.show();
       }
     };
 
-    // OPTIONAL: Detect login completion by checking URL change
     authWindow.webContents.on("did-frame-navigate", eventHandler);
   });
 };

apps/cruncher/src/processes/server/externalAuthProvider.ts

@@ -1,4 +1,4 @@
-import { ExternalAuthProvider } from "@cruncher/adapter-utils";
+import { ExternalAuthProvider, ScriptExtractor } from "@cruncher/adapter-utils";
 import { IPCMessage } from "./types";
 
 export class ElectronExternalAuthProvider implements ExternalAuthProvider {
@@ -7,6 +7,7 @@ export class ElectronExternalAuthProvider implements ExternalAuthProvider {
     requestedUrl: string,
     cookies: string[],
     validate: (cookies: Record<string, string>) => Promise<boolean>,
+    scriptExtractors?: ScriptExtractor[],
   ): Promise<Record<string, string>> {
     return new Promise((resolve, reject) => {
       const jobId = crypto.randomUUID();
@@ -58,6 +59,7 @@ export class ElectronExternalAuthProvider implements ExternalAuthProvider {
         authUrl: requestedUrl,
         jobId: jobId,
         cookies,
+        scriptExtractors: scriptExtractors ?? [],
       });
     });
   }

apps/cruncher/src/processes/server/main.ts

@@ -9,6 +9,7 @@ import loki from "@cruncher/adapter-loki";
 import grafanaLokiBrowser from "@cruncher/adapter-grafana-loki-browser";
 import mock from "@cruncher/adapter-mock";
 import coralogix from "@cruncher/adapter-coralogix";
+import datadog from "@cruncher/adapter-datadog";
 import { Engine } from "./engineV2/engine";
 import {
   DefaultExternalAuthProvider,
@@ -120,6 +121,7 @@ const initializeServer = async (authProvider: ExternalAuthProvider) => {
   engineV2.registerPlugin(docker);
   engineV2.registerPlugin(k8s);
   engineV2.registerPlugin(coralogix);
+  engineV2.registerPlugin(datadog);
 
   //   const routes = await getRoutes(engineV2);
   //   await setupEngine(serverContainer, routes);

docs/src/content/docs/adapters/datadog.mdx

@@ -0,0 +1,82 @@
+---
+title: Datadog
+description: How to use Cruncher with Datadog Logs.
+---
+import ParamItem from '../../../components/ParamItem.astro';
+
+import { Badge } from '@astrojs/starlight/components';
+
+<Badge text="datadog" />
+
+## Params
+
+<ParamItem label="site" type="string" optional default="datadoghq.com">
+  The Datadog site to connect to. Choose the site that matches your Datadog organization.
+
+  Supported values:
+  - `datadoghq.com` — US1 (default)
+  - `us3.datadoghq.com` — US3
+  - `us5.datadoghq.com` — US5
+  - `datadoghq.eu` — EU1
+  - `ap1.datadoghq.com` — AP1
+</ParamItem>
+
+<ParamItem label="indexes" type="string[]" optional default="[]">
+  List of Datadog log index names to query. An empty list (the default) searches across all indexes (`*`).
+</ParamItem>
+
+<ParamItem label="query_prefix" type="string" optional default='""'>
+  A Lucene query string that is always prepended to every search. Useful for scoping a connector to a specific service, environment, or team without requiring users to type it every time.
+
+  Example: `service:my-service AND env:production`
+</ParamItem>
+
+## Examples
+
+### Minimal Configuration
+
+```yaml
+connectors:
+  - type: datadog
+    name: my_datadog
+```
+
+### With a Specific Site
+
+```yaml
+connectors:
+  - type: datadog
+    name: my_datadog_eu
+    params:
+      site: "datadoghq.eu"
+```
+
+### Scoped to Specific Indexes and a Query Prefix
+
+```yaml
+connectors:
+  - type: datadog
+    name: my_datadog
+    params:
+      site: "datadoghq.com"
+      indexes:
+        - "main"
+        - "prod-logs"
+      query_prefix: "env:production"
+```
+
+## Authentication
+
+The Datadog adapter uses **browser-session authentication**. When you run a query for the first time, Cruncher opens a browser window pointing to your Datadog site. Log in as usual (including SSO/SAML if required), and Cruncher will automatically capture the session cookies and CSRF token it needs.
+
+Once authenticated, the session is reused for all subsequent queries. If the session expires, Cruncher will automatically re-open the login window.
+
+## Facet Autocomplete
+
+Cruncher automatically fetches your Datadog log facets and index names to provide autocomplete suggestions in the query editor. Facets with a fixed set of values (e.g. `status`) are resolved directly; facets with dynamic values (e.g. `kube_namespace`) are populated by querying the top 10 most frequent values from the last 15 minutes.
+
+## Usage Notes
+
+- The `indexes` param scopes queries to specific log indexes. Leave it empty to search all indexes.
+- The `query_prefix` is combined with the QQL search term using `AND`, so it always applies.
+- The `index` controller param (set inline in the QQL query bar) overrides the `indexes` config param for that query.