diff --git a/.gitignore b/.gitignore index 47384918..fadc3506 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,5 @@ node_modules -.env \ No newline at end of file +.env +/drift-report/ diff --git a/package.json b/package.json index a36dce76..24478e1a 100644 --- a/package.json +++ b/package.json @@ -16,7 +16,8 @@ "gen-api-pages:read": "cd src && mintlify-scrape openapi-file -o reference read.json | tail -n +2 > reference/read.json", "gen-api-pages:write": "cd src && mintlify-scrape openapi-file -o reference write.json | tail -n +2 > reference/write.json", "gen-api-pages:search": "cd src && mintlify-scrape openapi-file -o reference search.json | tail -n +2 > reference/search.json", - "gen-docs": "cd src && tsx generate-docs.ts docs.json" + "gen-docs": "cd src && tsx generate-docs.ts docs.json", + "drift-check": "tsx scripts/drift-check/index.ts" }, "devDependencies": { "@mintlify/scraping": "3.0.137", diff --git a/scripts/drift-check/README.md b/scripts/drift-check/README.md new file mode 100644 index 00000000..18b6d988 --- /dev/null +++ b/scripts/drift-check/README.md @@ -0,0 +1,66 @@ +# Drift check + +Compares provider-guide claims against the generated [`amp-labs/connectors`](https://github.com/amp-labs/connectors/blob/main/internal/generated/catalog.json) catalog. Static, no credentials, no runtime calls. + +## Quick start + +```shell +pnpm run drift-check # full sweep, default ./drift-report/ +pnpm run drift-check -- --out /tmp/out # custom output directory +pnpm run drift-check -- --mode provider --provider hubspot +pnpm run drift-check -- --fail-on-error # exit non-zero on any error finding +``` + +Both `--out X` and `-- --out X` (the pnpm forwarding form) work. Output is written to `drift-report.json` (canonical) and `drift-report.md` (human-readable rollup). + +## Modes + +- `full` (default): scan every guide; run the undocumented-provider check. +- `per-pr --changed ...`: scan only the listed `.mdx` files. Skips the undocumented check (needs a global view). +- `provider --provider `: scan a single guide. Useful while iterating. + +## Finding types + +| Kind | Severity | Triggers | +|---|---|---| +| `undocumented-provider` | error | Catalog has a provider; no guide; no allowance | +| `undocumented-provider-allowance-expired` | warning | An entry in `undocumentedAllowed` has passed its `until` date | +| `guide-without-catalog-entry` | error | A guide resolves to a catalog key that does not exist | +| `doc-overclaims-{read,write,subscribe,proxy,search}` | error | Guide claims an action; neither provider-level nor any module-level support flag is true | +| `broken-sample-link` | error | Guide links to a `samples` manifest that 404s | + +Module-aware: a claim is valid if either the provider-level flag is true *or* any module-level flag is true. Multi-module provider findings carry `"module": "all"` since v1 does not attribute the claim to a specific module. + +## What this does not prove + +This check uses the generated connectors catalog from `main`. It does **not** validate the deployed `/v1/providers` API against its declared shape, and it does **not** prove that read/write/proxy/subscribe calls succeed at runtime. Several drift surfaces are deliberately out of scope: + +| Surface | Why out of scope | +|---|---| +| Connector Go source vs generated catalog | Regeneration lag; lives in the connectors repo | +| Generated catalog vs live `/v1/providers` | Deployment lag; separate cheap check | +| Live API declared shape vs runtime behavior | Requires sandbox accounts; separate program | +| Guides vs connector source object names | Requires parsing Go; deferred to a follow-up PR | +| Guides vs provider's own developer docs | Per-provider HTML parsing | +| Guides vs provider product UIs (screenshots, GIFs) | Manual surface; account + MFA | + +## Adding a recipe + +Edit `recipes.ts`. Two surfaces, both intentionally sparse. + +**`slugOverrides`** — when the filename slug and the catalog key are different *strings* (not just different cases — case differences are handled automatically): + +```ts +slugOverrides: { + instantly: 'instantlyAI', + jira: 'atlassian', +} +``` + +**`undocumentedAllowed`** — when a catalog entry intentionally has no guide. Every entry needs a reason and a time-bounded `until` so the allowance cannot persist forever: + +```ts +{ provider: 'adyenTest', reason: 'Internal test variant.', until: '2027-01-01' } +``` + +When `until` passes, the entry becomes an `undocumented-provider-allowance-expired` warning until renewed or removed. diff --git a/scripts/drift-check/catalog.ts b/scripts/drift-check/catalog.ts new file mode 100644 index 00000000..81318c58 --- /dev/null +++ b/scripts/drift-check/catalog.ts @@ -0,0 +1,97 @@ +const CATALOG_URL = + 'https://raw.githubusercontent.com/amp-labs/connectors/main/internal/generated/catalog.json'; + +export type Capability = 'read' | 'write' | 'proxy' | 'subscribe' | 'search'; + +interface SearchSupport { + operators?: Record; +} + +interface SupportFlags { + read?: boolean; + write?: boolean; + proxy?: boolean; + subscribe?: boolean; + search?: SearchSupport; +} + +interface ModuleEntry { + baseURL?: string; + displayName?: string; + support?: SupportFlags; +} + +export interface CatalogProvider { + name?: string; + displayName?: string; + authType?: string; + baseURL?: string; + defaultModule?: string; + support?: SupportFlags; + modules?: Record; +} + +export interface Catalog { + data: Record; + sourceUrl: string; + fetchedAt: string; +} + +export async function fetchCatalog(): Promise { + const res = await fetch(CATALOG_URL); + if (!res.ok) { + throw new Error(`Failed to fetch catalog: ${res.status} ${res.statusText}`); + } + const json = (await res.json()) as { catalog: Record }; + return { + data: json.catalog, + sourceUrl: CATALOG_URL, + fetchedAt: new Date().toISOString(), + }; +} + +// Module-aware: returns true if the provider-level flag is true, or any +// module-level flag is true. Catches cases like Google, where the provider +// has subscribe=false but modules.gmail has subscribe=true. +export function supports( + provider: CatalogProvider | undefined, + capability: Capability, +): boolean { + if (!provider) return false; + if (capability === 'search') { + if (supportsSearch(provider.support?.search)) return true; + for (const mod of Object.values(provider.modules ?? {})) { + if (supportsSearch(mod.support?.search)) return true; + } + return false; + } + if (provider.support?.[capability]) return true; + for (const mod of Object.values(provider.modules ?? {})) { + if (mod.support?.[capability]) return true; + } + return false; +} + +function supportsSearch(s: SearchSupport | undefined): boolean { + if (!s?.operators) return false; + return Object.values(s.operators).some(Boolean); +} + +export function hasModules(provider: CatalogProvider | undefined): boolean { + return !!provider?.modules && Object.keys(provider.modules).length > 0; +} + +export function modulesSupporting( + provider: CatalogProvider | undefined, + capability: Capability, +): string[] { + if (!provider?.modules) return []; + if (capability === 'search') { + return Object.entries(provider.modules) + .filter(([, mod]) => supportsSearch(mod.support?.search)) + .map(([name]) => name); + } + return Object.entries(provider.modules) + .filter(([, mod]) => mod.support?.[capability]) + .map(([name]) => name); +} diff --git a/scripts/drift-check/docs.ts b/scripts/drift-check/docs.ts new file mode 100644 index 00000000..caa36ed3 --- /dev/null +++ b/scripts/drift-check/docs.ts @@ -0,0 +1,78 @@ +import { promises as fs } from 'node:fs'; +import path from 'node:path'; +import matter from 'gray-matter'; + +const PROVIDER_GUIDES_DIR = path.join(process.cwd(), 'src', 'provider-guides'); + +// Files in src/provider-guides/ that are not provider guides. +const NON_GUIDE_FILES = new Set(['overview.mdx', 'AGENTS.md', 'CLAUDE.md']); + +export type Capability = 'read' | 'write' | 'proxy' | 'subscribe' | 'search'; + +export interface GuideDoc { + docPath: string; + slug: string; + providerKeyFromFrontmatter?: string; + guideType?: string; + claimedActions: Set; + sampleLinks: string[]; +} + +const ACTION_PATTERNS: Array<{ cap: Capability; re: RegExp }> = [ + { cap: 'read', re: /\[Read Actions\]\(\/read-actions\)/ }, + { cap: 'write', re: /\[Write Actions\]\(\/write-actions\)/ }, + { cap: 'subscribe', re: /\[Subscribe Actions\]\(\/subscribe-actions\)/ }, + { cap: 'proxy', re: /\[Proxy Actions\]\(\/proxy-actions\)/ }, + { cap: 'search', re: /\[Search Actions\]\(\/search-actions\)/ }, +]; + +const SAMPLE_LINK_RE = + /https:\/\/github\.com\/amp-labs\/samples\/blob\/[^/]+\/[^/]+\/amp\.ya?ml/g; + +export async function scanGuides(): Promise { + const entries = await fs.readdir(PROVIDER_GUIDES_DIR); + const guides: GuideDoc[] = []; + for (const entry of entries) { + if (!entry.endsWith('.mdx')) continue; + if (NON_GUIDE_FILES.has(entry)) continue; + const full = path.join(PROVIDER_GUIDES_DIR, entry); + const raw = await fs.readFile(full, 'utf8'); + const parsed = matter(raw); + const slug = entry.replace(/\.mdx$/, ''); + const claimedActions = new Set(); + for (const { cap, re } of ACTION_PATTERNS) { + if (re.test(parsed.content)) claimedActions.add(cap); + } + const sampleLinks = Array.from( + new Set(parsed.content.match(SAMPLE_LINK_RE) ?? []), + ); + guides.push({ + docPath: path.relative(process.cwd(), full), + slug, + providerKeyFromFrontmatter: + typeof parsed.data.provider === 'string' ? parsed.data.provider : undefined, + guideType: + typeof parsed.data.guide_type === 'string' ? parsed.data.guide_type : undefined, + claimedActions, + sampleLinks, + }); + } + return guides; +} + +// Resolution priority: frontmatter `provider` > slugOverrides > case-insensitive +// match against catalog keys > filename slug verbatim. +export function resolveProviderKey( + guide: GuideDoc, + slugOverrides: Record, + catalogKeys?: readonly string[], +): string { + if (guide.providerKeyFromFrontmatter) return guide.providerKeyFromFrontmatter; + if (slugOverrides[guide.slug]) return slugOverrides[guide.slug]; + if (catalogKeys) { + const ci = guide.slug.toLowerCase(); + const match = catalogKeys.find((k) => k.toLowerCase() === ci); + if (match) return match; + } + return guide.slug; +} diff --git a/scripts/drift-check/index.ts b/scripts/drift-check/index.ts new file mode 100644 index 00000000..6876ddee --- /dev/null +++ b/scripts/drift-check/index.ts @@ -0,0 +1,174 @@ +#!/usr/bin/env tsx +import { parseArgs } from 'node:util'; +import path from 'node:path'; +import { + fetchCatalog, + hasModules, + supports, + modulesSupporting, + type Capability, + type CatalogProvider, +} from './catalog'; +import { scanGuides, resolveProviderKey } from './docs'; +import { checkSampleLink } from './samples'; +import { recipes } from './recipes'; +import { writeReport, type Finding, type Report } from './report'; + +async function main() { + // Accept both `drift-check --out X` and `pnpm run drift-check -- --out X`. + const args = process.argv.slice(2); + if (args[0] === '--') args.shift(); + const { values } = parseArgs({ + args, + options: { + mode: { type: 'string', default: 'full' }, + provider: { type: 'string' }, + changed: { type: 'string', multiple: true }, + out: { type: 'string', default: './drift-report' }, + 'fail-on-error': { type: 'boolean', default: false }, + }, + }); + + const mode = values.mode as 'full' | 'per-pr' | 'provider'; + + const catalog = await fetchCatalog(); + const catalogKeys = Object.keys(catalog.data); + let guides = await scanGuides(); + + if (mode === 'per-pr') { + const changedSet = new Set((values.changed ?? []).map((p) => path.normalize(p))); + guides = guides.filter((g) => changedSet.has(path.normalize(g.docPath))); + } else if (mode === 'provider') { + if (!values.provider) { + throw new Error('--provider required with --mode provider'); + } + guides = guides.filter( + (g) => + resolveProviderKey(g, recipes.slugOverrides, catalogKeys) === values.provider, + ); + } + + const findings: Finding[] = []; + + // The undocumented-provider check requires a global view, so it only runs in full mode. + if (mode === 'full') { + const documentedKeys = new Set( + guides.map((g) => resolveProviderKey(g, recipes.slugOverrides, catalogKeys)), + ); + const today = new Date(); + for (const catalogKey of catalogKeys) { + if (documentedKeys.has(catalogKey)) continue; + const allow = recipes.undocumentedAllowed.find((a) => a.provider === catalogKey); + if (allow) { + if (new Date(allow.until) < today) { + findings.push({ + provider: catalogKey, + severity: 'warning', + kind: 'undocumented-provider-allowance-expired', + suggestedFix: `Renew or remove the allowance for "${catalogKey}" in scripts/drift-check/recipes.ts (current until=${allow.until}).`, + }); + } + continue; + } + findings.push({ + provider: catalogKey, + severity: 'error', + kind: 'undocumented-provider', + suggestedFix: `Add a provider guide at src/provider-guides/${catalogKey}.mdx, or add "${catalogKey}" to undocumentedAllowed in scripts/drift-check/recipes.ts with a reason and an "until" date.`, + }); + } + } + + const sampleChecks: Promise[] = []; + for (const guide of guides) { + const providerKey = resolveProviderKey(guide, recipes.slugOverrides, catalogKeys); + const provider = catalog.data[providerKey]; + if (!provider) { + findings.push({ + provider: providerKey, + severity: 'error', + kind: 'guide-without-catalog-entry', + docPath: guide.docPath, + suggestedFix: `The guide resolves to provider key "${providerKey}" but the catalog has no such entry. Check the frontmatter "provider" field or add a slugOverride in recipes.ts.`, + }); + continue; + } + for (const cap of guide.claimedActions) { + if (!supports(provider, cap as Capability)) { + findings.push( + withModule(provider, { + provider: providerKey, + severity: 'error', + kind: `doc-overclaims-${cap}`, + docPath: guide.docPath, + docClaim: `Guide includes "[${capitalize(cap)} Actions](/${cap}-actions)".`, + catalogTruth: { + providerLevel: providerLevelSupport(provider, cap as Capability), + modulesSupporting: modulesSupporting(provider, cap as Capability), + }, + suggestedFix: `Remove the ${cap} action claim, or update the connector if support was added but not flagged.`, + }), + ); + } + } + for (const sampleLink of guide.sampleLinks) { + sampleChecks.push( + checkSampleLink(sampleLink).then((r) => { + if (!r.exists) { + findings.push( + withModule(provider, { + provider: providerKey, + severity: 'error', + kind: 'broken-sample-link', + docPath: guide.docPath, + docClaim: r.url, + catalogTruth: { httpStatus: r.status ?? 'fetch failed' }, + suggestedFix: `Update or remove the sample link. The referenced file does not exist at ${r.url}.`, + }), + ); + } + }), + ); + } + } + await Promise.all(sampleChecks); + + const report: Report = { + catalogSource: `${catalog.sourceUrl} (fetched from amp-labs/connectors@main)`, + checkedAt: catalog.fetchedAt, + mode, + findings, + }; + await writeReport(report, values.out!); + + const errorCount = findings.filter((f) => f.severity === 'error').length; + console.log( + `\n${findings.length} findings (${errorCount} error, ${ + findings.filter((f) => f.severity === 'warning').length + } warning, ${findings.filter((f) => f.severity === 'info').length} info).`, + ); + if (values['fail-on-error'] && errorCount > 0) { + process.exit(1); + } +} + +function capitalize(s: string): string { + return s.charAt(0).toUpperCase() + s.slice(1); +} + +// Default module to "all" on findings for multi-module providers; v1 does not attribute claims to a specific module. +function withModule(provider: CatalogProvider, finding: Finding): Finding { + if (finding.module !== undefined) return finding; + if (hasModules(provider)) return { ...finding, module: 'all' }; + return finding; +} + +function providerLevelSupport(provider: CatalogProvider, cap: Capability): unknown { + if (cap === 'search') return provider.support?.search ?? false; + return provider.support?.[cap] ?? false; +} + +main().catch((err) => { + console.error(err); + process.exit(2); +}); diff --git a/scripts/drift-check/recipes.ts b/scripts/drift-check/recipes.ts new file mode 100644 index 00000000..0e31d34b --- /dev/null +++ b/scripts/drift-check/recipes.ts @@ -0,0 +1,57 @@ +// Sparse overrides and allowances. Only encode exceptions here; convention covers the rest. + +export interface UndocumentedAllowance { + provider: string; + reason: string; + until: string; // ISO date; allowance expires after this +} + +export interface Recipes { + slugOverrides: Record; + undocumentedAllowed: UndocumentedAllowance[]; +} + +export const recipes: Recipes = { + slugOverrides: { + // instantly.mdx documents the V2 catalog entry (instantlyAI); the V1 entry + // is allowed undocumented below. + instantly: 'instantlyAI', + 'google-workspace-delegation': 'googleWorkspaceDelegation', + 'netsuite-m2m': 'netsuiteM2M', + 'salesforce-jwt': 'salesforceJWT', + // jira.mdx is an alias guide pointing readers to the Atlassian guide. + jira: 'atlassian', + // highlevel.mdx covers the Standard variant; the WhiteLabel variant is + // allowed undocumented below. + highlevel: 'highLevelStandard', + }, + + undocumentedAllowed: [ + { + provider: 'instantly', + reason: 'Legacy V1 connector; current public guide covers instantlyAI.', + until: '2026-09-01', + }, + { provider: 'adyenTest', reason: 'Internal test variant.', until: '2027-01-01' }, + { provider: 'gladlyQA', reason: 'QA-only variant.', until: '2027-01-01' }, + { + provider: 'docusignDeveloper', + reason: 'Developer-environment variant; setup covered by docusign.mdx.', + until: '2026-09-01', + }, + { provider: 'deelSandbox', reason: 'Sandbox variant.', until: '2027-01-01' }, + { provider: 'gustoDemo', reason: 'Demo variant.', until: '2027-01-01' }, + { provider: 'paddleSandbox', reason: 'Sandbox variant.', until: '2027-01-01' }, + { provider: 'payPalSandBox', reason: 'Sandbox variant.', until: '2027-01-01' }, + { provider: 'procoreSandbox', reason: 'Sandbox variant.', until: '2027-01-01' }, + { provider: 'quickbooksSandbox', reason: 'Sandbox variant.', until: '2027-01-01' }, + { provider: 'rampDemo', reason: 'Demo variant.', until: '2027-01-01' }, + { provider: 'ironcladDemo', reason: 'Demo variant.', until: '2027-01-01' }, + { provider: 'leverSandbox', reason: 'Sandbox variant.', until: '2027-01-01' }, + { + provider: 'highLevelWhiteLabel', + reason: 'WhiteLabel variant; highlevel.mdx covers the Standard variant.', + until: '2026-09-01', + }, + ], +}; diff --git a/scripts/drift-check/report.ts b/scripts/drift-check/report.ts new file mode 100644 index 00000000..44328c2a --- /dev/null +++ b/scripts/drift-check/report.ts @@ -0,0 +1,72 @@ +import { promises as fs } from 'node:fs'; +import path from 'node:path'; + +export type Severity = 'error' | 'warning' | 'info'; + +export interface Finding { + provider: string; // exact catalog key, case preserved + module?: string | 'all'; // required for multi-module providers when known + severity: Severity; + kind: string; + docPath?: string; + docClaim?: string; + catalogTruth?: unknown; + suggestedFix?: string; +} + +export interface Report { + catalogSource: string; + checkedAt: string; // ISO timestamp + mode: 'per-pr' | 'full' | 'provider'; + findings: Finding[]; +} + +export async function writeReport(report: Report, outDir: string): Promise { + await fs.mkdir(outDir, { recursive: true }); + const jsonPath = path.join(outDir, 'drift-report.json'); + const mdPath = path.join(outDir, 'drift-report.md'); + await fs.writeFile(jsonPath, JSON.stringify(report, null, 2) + '\n'); + await fs.writeFile(mdPath, toMarkdown(report)); + console.log(`Wrote ${jsonPath}`); + console.log(`Wrote ${mdPath}`); +} + +function toMarkdown(report: Report): string { + const bySeverity: Record = { error: [], warning: [], info: [] }; + for (const f of report.findings) bySeverity[f.severity].push(f); + + const lines: string[] = []; + lines.push('# Drift check report'); + lines.push(''); + lines.push(`- Catalog source: \`${report.catalogSource}\``); + lines.push(`- Checked at: \`${report.checkedAt}\``); + lines.push(`- Mode: \`${report.mode}\``); + lines.push(`- Totals: ${bySeverity.error.length} error, ${bySeverity.warning.length} warning, ${bySeverity.info.length} info`); + lines.push(''); + + for (const sev of ['error', 'warning', 'info'] as const) { + const group = bySeverity[sev]; + if (group.length === 0) continue; + lines.push(`## ${capitalize(sev)} (${group.length})`); + lines.push(''); + for (const f of group.sort(sortByProvider)) { + const mod = f.module ? ` [${f.module}]` : ''; + lines.push(`### ${f.provider}${mod} — ${f.kind}`); + if (f.docPath) lines.push(`- File: \`${f.docPath}\``); + if (f.docClaim) lines.push(`- Doc says: ${f.docClaim}`); + if (f.catalogTruth !== undefined) lines.push(`- Catalog says: \`${JSON.stringify(f.catalogTruth)}\``); + if (f.suggestedFix) lines.push(`- Fix: ${f.suggestedFix}`); + lines.push(''); + } + } + return lines.join('\n'); +} + +function sortByProvider(a: Finding, b: Finding): number { + if (a.provider !== b.provider) return a.provider.localeCompare(b.provider); + return (a.kind ?? '').localeCompare(b.kind ?? ''); +} + +function capitalize(s: string): string { + return s.charAt(0).toUpperCase() + s.slice(1); +} diff --git a/scripts/drift-check/samples.ts b/scripts/drift-check/samples.ts new file mode 100644 index 00000000..861447f6 --- /dev/null +++ b/scripts/drift-check/samples.ts @@ -0,0 +1,33 @@ +// blob URLs return a 200 HTML page for missing files; raw URLs return 404. +function toRawUrl(blobUrl: string): string { + return blobUrl + .replace('github.com/', 'raw.githubusercontent.com/') + .replace('/blob/', '/'); +} + +export interface SampleCheckResult { + url: string; + exists: boolean; + status?: number; +} + +// Trusts 404 immediately; retries once on transient errors (5xx, network blips). +export async function checkSampleLink(blobUrl: string): Promise { + const rawUrl = toRawUrl(blobUrl); + const MAX_ATTEMPTS = 2; + let lastStatus: number | undefined; + for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) { + try { + const res = await fetch(rawUrl, { method: 'HEAD' }); + if (res.ok) return { url: blobUrl, exists: true, status: res.status }; + if (res.status === 404) return { url: blobUrl, exists: false, status: 404 }; + lastStatus = res.status; + } catch { + // network error; retry + } + if (attempt < MAX_ATTEMPTS) { + await new Promise((r) => setTimeout(r, 250 * attempt)); + } + } + return { url: blobUrl, exists: false, status: lastStatus }; +}