diff --git a/CHANGELOG.md b/CHANGELOG.md index 5194af3..87b2ddb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,9 @@ ## 0.0.11-beta.2 — 2026-05-21 +### Features +- Expand PostHog telemetry coverage to close the 16 server-side and 12 web-UI gaps surfaced by the May audit (#376). Server-side adds `cli_install_success` / `cli_install_failure` / `cli_uninstall_success` / `cli_uninstall_failure` / `cli_list_invoked` / `cli_parse_error` / `cli_unexpected_error` / `hook_dispatch_error` (CLI lifecycle outcomes in `bin/failproofai.mjs`), `hook_stdin_error` / `hook_payload_parse_error` (hook handler input errors in `src/hooks/handler.ts`), `policy_evaluation_error` (builtin policy crashes in `src/hooks/policy-evaluator.ts`, distinct from the existing `custom_hook_error`), `custom_policy_validation_failed` / `custom_hooks_load_error` / `policy_params_validation_warning` / `scope_validation_failed` / `hook_write_failed` / `multi_scope_warning_shown` / `cli_detection_summary` / `beta_policies_installed` (manager / loader / install-prompt internals), and `first_install` / `version_changed` (lifecycle detection in `scripts/postinstall.mjs` via a new `~/.failproofai/last-version` file). Web-UI adds `policies_tab_switched` / `activity_filter_changed` (debounced) / `activity_row_toggled` / `activity_copy_clicked` / `activity_pagination_changed` / `cli_selection_toggled` / `cli_install_remove_submitted` / `cli_reinstall_submitted` / `policy_config_modal_opened` / `policy_config_modal_closed` / `action_error_displayed` / `hooks_install_from_error_clicked` via `usePostHog()` in `app/policies/hooks-client.tsx`. The deny-/instruct-only condition at `handler.ts:344` (allow-path tracking) is intentionally left unchanged. All events go through the existing helpers (`trackHookEvent`, `trackInstallEvent`, `captureClientEvent`) and honor `FAILPROOFAI_TELEMETRY_DISABLED=1`. + ### Breaking - Remove the undocumented cloud auth + event relay subsystem ahead of a from-scratch redesign. Deletes `src/auth/` (OAuth 2.0 device-flow login against `api.befailproof.ai`, `~/.failproofai/auth.json` token store) and `src/relay/` (WebSocket event relay daemon, sanitized JSONL queue at `~/.failproofai/cache/server-queue/`, PID tracking). Strips the `failproofai login` / `logout` / `whoami` / `relay start|stop|status` / `sync` subcommands and the internal `--relay-daemon` mode from `bin/failproofai.mjs`, along with their `--help` entries and "did you mean" suggestions. Removes the fire-and-forget `appendToServerQueue` + `ensureRelayRunning` calls from `src/hooks/handler.ts` so hook evaluation no longer enqueues events or lazy-spawns a daemon. The whole subsystem had zero references in `README.md`, `docs/`, `examples/`, or `__tests__/`, and only had internal cross-imports — `tsc`, `eslint`, `vitest` (1623 tests), and the `bun run build` bundles all stay green. Users who ran `failproofai login` should also wipe `~/.failproofai/{auth.json,cache/server-queue,relay.pid}` and stop any running relay daemon by hand; new auth/cloud surface will land in a follow-up. diff --git a/__tests__/hooks/new-telemetry.test.ts b/__tests__/hooks/new-telemetry.test.ts new file mode 100644 index 0000000..349ada0 --- /dev/null +++ b/__tests__/hooks/new-telemetry.test.ts @@ -0,0 +1,211 @@ +// @vitest-environment node +/** + * Coverage for telemetry events added as part of the gap-closing audit. + * Each test stubs trackHookEvent and asserts the right event name + props + * fire at the trigger site. Keep one focused case per event. + */ +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; +import { readFileSync, writeFileSync, existsSync } from "node:fs"; +import { execSync } from "node:child_process"; +import { resolve } from "node:path"; +import { homedir } from "node:os"; + +vi.mock("node:fs", () => ({ + readFileSync: vi.fn(), + writeFileSync: vi.fn(), + existsSync: vi.fn(), + mkdirSync: vi.fn(), + readdirSync: vi.fn(() => []), +})); + +vi.mock("node:child_process", () => ({ + execSync: vi.fn(), +})); + +vi.mock("../../src/hooks/install-prompt", async () => { + const actual = await vi.importActual( + "../../src/hooks/install-prompt", + ); + return { + ...actual, + promptPolicySelection: vi.fn(() => Promise.resolve(["block-sudo"])), + }; +}); + +vi.mock("../../src/hooks/integrations", () => ({ + detectInstalledClis: vi.fn(() => ["claude"]), + getIntegration: vi.fn((id: string) => ({ + displayName: id, + scopes: id === "codex" ? ["user", "project"] : ["user", "project", "local"], + eventTypes: [], + getSettingsPath: vi.fn(), + hooksInstalledInSettings: vi.fn(), + readSettings: vi.fn(() => ({})), + writeSettings: vi.fn(), + writeHookEntries: vi.fn(), + removeHooksFromFile: vi.fn(() => 0), + })), + claudeCode: { + getSettingsPath: vi.fn(() => "/tmp/.claude/settings.json"), + hooksInstalledInSettings: vi.fn(() => false), + }, + listIntegrations: vi.fn(() => []), +})); + +vi.mock("../../src/hooks/hooks-config", () => ({ + readHooksConfig: vi.fn(() => ({ enabledPolicies: [] })), + readMergedHooksConfig: vi.fn(() => ({ enabledPolicies: [] })), + writeHooksConfig: vi.fn(), + readScopedHooksConfig: vi.fn(() => ({ enabledPolicies: [] })), + writeScopedHooksConfig: vi.fn(), + findProjectConfigDir: vi.fn((cwd: string) => cwd), + getConfigPathForScope: vi.fn(() => "/tmp/policies-config.json"), +})); + +vi.mock("../../src/hooks/hook-telemetry", () => ({ + trackHookEvent: vi.fn(() => Promise.resolve()), +})); + +vi.mock("../../lib/telemetry-id", () => ({ + getInstanceId: vi.fn(() => "test-instance-id"), + hashToId: vi.fn((raw: string) => `hashed:${raw}`), +})); + +vi.mock("../../src/hooks/custom-hooks-loader", async () => { + const actual = await vi.importActual( + "../../src/hooks/custom-hooks-loader", + ); + return { + ...actual, + loadCustomHooks: vi.fn(() => Promise.resolve([])), + discoverPolicyFiles: vi.fn(() => []), + }; +}); + +describe("new telemetry events — manager", () => { + beforeEach(() => { + vi.resetAllMocks(); + vi.mocked(execSync).mockReturnValue("/usr/local/bin/failproofai\n"); + vi.spyOn(console, "log").mockImplementation(() => {}); + vi.spyOn(console, "error").mockImplementation(() => {}); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + it("fires scope_validation_failed when scope is unsupported for the CLI", async () => { + vi.mocked(existsSync).mockReturnValue(true); + vi.mocked(readFileSync).mockReturnValue("{}"); + + const { installHooks } = await import("../../src/hooks/manager"); + const { trackHookEvent } = await import("../../src/hooks/hook-telemetry"); + + // codex does not support the "local" scope (see src/hooks/integrations.ts) + await expect(installHooks(["block-sudo"], "local" as never, undefined, false, undefined, undefined, false, [ + "codex", + ])).rejects.toThrow(/Scope "local" is not supported/); + + expect(trackHookEvent).toHaveBeenCalledWith( + "test-instance-id", + "scope_validation_failed", + expect.objectContaining({ + cli: "codex", + scope: "local", + supported_scopes: expect.any(Array), + }), + ); + }); + + it("fires custom_policy_validation_failed when the custom file throws", async () => { + vi.mocked(existsSync).mockReturnValue(true); + vi.mocked(readFileSync).mockReturnValue("{}"); + const { loadCustomHooks } = await import("../../src/hooks/custom-hooks-loader"); + vi.mocked(loadCustomHooks).mockRejectedValueOnce(new Error("Custom hooks file not found: /tmp/missing.js")); + const exitSpy = vi.spyOn(process, "exit").mockImplementation(((_code?: string | number | null | undefined) => undefined as never) as never); + + const { installHooks } = await import("../../src/hooks/manager"); + const { trackHookEvent } = await import("../../src/hooks/hook-telemetry"); + + await installHooks(["block-sudo"], "user", undefined, false, undefined, "/tmp/missing.js"); + + expect(trackHookEvent).toHaveBeenCalledWith( + "test-instance-id", + "custom_policy_validation_failed", + expect.objectContaining({ + scope: "user", + error_type: "file_not_found", + }), + ); + exitSpy.mockRestore(); + }); + + it("fires policy_params_validation_warning when an unknown key is in policyParams", async () => { + vi.mocked(existsSync).mockReturnValue(false); + const { readMergedHooksConfig } = await import("../../src/hooks/hooks-config"); + vi.mocked(readMergedHooksConfig).mockReturnValue({ + enabledPolicies: [], + policyParams: { "nonexistent-policy": { hint: "test" } }, + }); + + const { listHooks } = await import("../../src/hooks/manager"); + const { trackHookEvent } = await import("../../src/hooks/hook-telemetry"); + + await listHooks(); + + expect(trackHookEvent).toHaveBeenCalledWith( + "test-instance-id", + "policy_params_validation_warning", + expect.objectContaining({ + unknown_keys_count: 1, + unknown_keys: ["nonexistent-policy"], + }), + ); + }); + + it("respects FAILPROOFAI_TELEMETRY_DISABLED — the underlying helper is mocked, but the call still happens (test verifies the trigger fires unconditionally)", async () => { + // The helper itself short-circuits when FAILPROOFAI_TELEMETRY_DISABLED=1 + // (verified in __tests__/lib/telemetry.test.ts). Here we just confirm the + // trigger call site still runs — the env-var check lives in the helper. + process.env.FAILPROOFAI_TELEMETRY_DISABLED = "1"; + vi.mocked(existsSync).mockReturnValue(true); + vi.mocked(readFileSync).mockReturnValue("{}"); + + try { + const { installHooks } = await import("../../src/hooks/manager"); + const { trackHookEvent } = await import("../../src/hooks/hook-telemetry"); + + await installHooks(["block-sudo"], "user"); + + // The call site fires; the helper internally no-ops. This is the same + // contract as the existing hooks_installed test. + expect(trackHookEvent).toHaveBeenCalled(); + } finally { + delete process.env.FAILPROOFAI_TELEMETRY_DISABLED; + } + }); +}); + +describe("new telemetry events — install-prompt", () => { + beforeEach(() => { + vi.resetAllMocks(); + }); + + it("fires cli_detection_summary with resolution_mode=explicit when --cli is passed", async () => { + const { resolveTargetClis } = await import("../../src/hooks/install-prompt"); + const { trackHookEvent } = await import("../../src/hooks/hook-telemetry"); + + const result = await resolveTargetClis(["codex"], "install"); + expect(result).toEqual(["codex"]); + expect(trackHookEvent).toHaveBeenCalledWith( + "test-instance-id", + "cli_detection_summary", + expect.objectContaining({ + action: "install", + explicit_clis: ["codex"], + selected_clis: ["codex"], + resolution_mode: "explicit", + }), + ); + }); +}); diff --git a/app/policies/hooks-client.tsx b/app/policies/hooks-client.tsx index 70a1059..79ac210 100644 --- a/app/policies/hooks-client.tsx +++ b/app/policies/hooks-client.tsx @@ -26,6 +26,7 @@ import { togglePolicyAction } from "@/app/actions/update-hooks-config"; import { installHooksWebAction, removeHooksWebAction } from "@/app/actions/install-hooks-web"; import { updatePolicyParamsAction } from "@/app/actions/update-policy-params"; import { useAutoRefresh } from "@/contexts/AutoRefreshContext"; +import { usePostHog } from "@/contexts/PostHogContext"; import { useUrlParams } from "@/lib/use-url-params"; import { pageToParam, paramToPage } from "@/lib/url-filter-serializers"; import { getCliLabel, getCliBadgeClasses, KNOWN_CLI_IDS, isKnownCli, type CliId } from "@/lib/cli-registry"; @@ -208,10 +209,12 @@ function DurationDisplay({ ms }: { ms: number }) { // -- Copy Button -- -function CopyButton({ text }: { text: string }) { +function CopyButton({ text, field }: { text: string; field?: string }) { const [copied, setCopied] = useState(false); + const { capture } = usePostHog(); const handleCopy = async (e: React.MouseEvent) => { e.stopPropagation(); + capture("activity_copy_clicked", { field: field ?? "unknown" }); try { await navigator.clipboard.writeText(text); setCopied(true); @@ -321,12 +324,12 @@ function DetailPanel({ {item.sessionId ?? "\u2014"} - {item.sessionId && } + {item.sessionId && }
CWD: {item.cwd ?? "\u2014"} - {item.cwd && } + {item.cwd && }
Transcript: @@ -366,6 +369,7 @@ function ActivityTab({ onSwitchTab?: (tab: "activity" | "policies") => void; }) { const { intervalSec } = useAutoRefresh(); + const { capture } = usePostHog(); const url = useUrlParams(); const mountedRef = useRef(false); @@ -385,6 +389,7 @@ function ActivityTab({ return isKnownCli(v) ? v : ""; }); const debounceRef = useRef | null>(null); + const filterTelemetryFirstRunRef = useRef(true); const filtersRef = useRef({ filterDecision, filterEventType, filterPolicy, filterSessionId, filterCli }); filtersRef.current = { filterDecision, filterEventType, filterPolicy, filterSessionId, filterCli }; @@ -444,6 +449,28 @@ function ActivityTab({ setPage(1); setExpandedRow(null); fetchData(1); + // Skip the initial render — filters are initialized from URL params, not + // from a user action. Only fire on real user-driven changes. + if (filterTelemetryFirstRunRef.current) { + filterTelemetryFirstRunRef.current = false; + return; + } + capture("activity_filter_changed", { + active_filter_count: + (filterDecision !== "" ? 1 : 0) + + (filterEventType !== "" ? 1 : 0) + + (filterPolicy !== "" ? 1 : 0) + + (filterSessionId !== "" ? 1 : 0) + + (filterCli !== "" ? 1 : 0), + has_decision: filterDecision !== "", + has_event_type: filterEventType !== "", + has_policy: filterPolicy !== "", + has_session: filterSessionId !== "", + has_cli: filterCli !== "", + decision: filterDecision || null, + event_type: filterEventType || null, + cli: filterCli || null, + }); }, 300); return () => { if (debounceRef.current) clearTimeout(debounceRef.current); @@ -455,7 +482,17 @@ function ActivityTab({ const totalPages = data?.totalPages ?? 1; const toggleRow = (idx: number) => { - setExpandedRow((prev) => (prev === idx ? null : idx)); + setExpandedRow((prev) => { + const next = prev === idx ? null : idx; + const item = items[idx]; + capture("activity_row_toggled", { + expanded: next !== null, + decision: item?.decision ?? null, + policy_name: item?.policyName ?? null, + event_type: item?.eventType ?? null, + }); + return next; + }); }; return ( @@ -652,6 +689,11 @@ function ActivityTab({ currentPage={page} totalPages={totalPages} onPageChange={(p) => { + capture("activity_pagination_changed", { + from_page: page, + to_page: p, + total_pages: totalPages, + }); setPage(p); setExpandedRow(null); }} @@ -886,6 +928,7 @@ function ErrorToast({ onInstall: () => void; isPending: boolean; }) { + const { capture } = usePostHog(); return createPortal(