From dd1232ff66a91208a9c645828a06fe03ca2ba7b6 Mon Sep 17 00:00:00 2001 From: Caio Pizzol Date: Sun, 17 May 2026 18:02:35 -0300 Subject: [PATCH 1/8] scripts: add drift-check v1 detector for docs vs catalog Static three-source diff between provider guides and the connectors catalog. v1 scope (static only, no runtime, no provider public-doc scraping). Module-aware: a guide claim is valid if either the provider-level or any module-level support flag matches. Files: - scripts/drift-check/index.ts CLI entry; modes full / per-pr / provider - scripts/drift-check/catalog.ts fetch connectors catalog from main; module-aware capability helpers - scripts/drift-check/docs.ts scan src/provider-guides/*.mdx, extract frontmatter and the four canonical action-link patterns; case- insensitive resolution of file slug to catalog key - scripts/drift-check/samples.ts HEAD-check each sample manifest link - scripts/drift-check/recipes.ts sparse slugOverrides for hyphen-vs- camelCase and family aliases; undocumentedAllowed list with reasons and expirations - scripts/drift-check/report.ts Finding type, JSON canonical output, Markdown derived First sweep on main surfaces 42 findings: 32 undocumented providers, 7 broken sample links, 2 write overclaims, 1 proxy overclaim. All appear legitimate; no false positives observed. Adds pnpm run drift-check. Default mode is full; runs report-only (does not fail CI). PR 3 will add connector-source inspection (connectors.ts) and CI wiring. --- .gitignore | 3 +- package.json | 3 +- scripts/drift-check/catalog.ts | 83 ++++++++++++++++ scripts/drift-check/docs.ts | 86 ++++++++++++++++ scripts/drift-check/index.ts | 174 +++++++++++++++++++++++++++++++++ scripts/drift-check/recipes.ts | 107 ++++++++++++++++++++ scripts/drift-check/report.ts | 72 ++++++++++++++ scripts/drift-check/samples.ts | 26 +++++ 8 files changed, 552 insertions(+), 2 deletions(-) create mode 100644 scripts/drift-check/catalog.ts create mode 100644 scripts/drift-check/docs.ts create mode 100644 scripts/drift-check/index.ts create mode 100644 scripts/drift-check/recipes.ts create mode 100644 scripts/drift-check/report.ts create mode 100644 scripts/drift-check/samples.ts diff --git a/.gitignore b/.gitignore index 47384918..fd79120e 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,5 @@ node_modules -.env \ No newline at end of file +.env +drift-report/ diff --git a/package.json b/package.json index a36dce76..24478e1a 100644 --- a/package.json +++ b/package.json @@ -16,7 +16,8 @@ "gen-api-pages:read": "cd src && mintlify-scrape openapi-file -o reference read.json | tail -n +2 > reference/read.json", "gen-api-pages:write": "cd src && mintlify-scrape openapi-file -o reference write.json | tail -n +2 > reference/write.json", "gen-api-pages:search": "cd src && mintlify-scrape openapi-file -o reference search.json | tail -n +2 > reference/search.json", - "gen-docs": "cd src && tsx generate-docs.ts docs.json" + "gen-docs": "cd src && tsx generate-docs.ts docs.json", + "drift-check": "tsx scripts/drift-check/index.ts" }, "devDependencies": { "@mintlify/scraping": "3.0.137", diff --git a/scripts/drift-check/catalog.ts b/scripts/drift-check/catalog.ts new file mode 100644 index 00000000..e1ea8640 --- /dev/null +++ b/scripts/drift-check/catalog.ts @@ -0,0 +1,83 @@ +const CATALOG_URL = + 'https://raw.githubusercontent.com/amp-labs/connectors/main/internal/generated/catalog.json'; + +export type Capability = 'read' | 'write' | 'proxy' | 'subscribe'; + +interface SupportFlags { + read?: boolean; + write?: boolean; + proxy?: boolean; + subscribe?: boolean; +} + +interface ModuleEntry { + baseURL?: string; + displayName?: string; + support?: SupportFlags; +} + +export interface CatalogProvider { + name?: string; + displayName?: string; + authType?: string; + baseURL?: string; + defaultModule?: string; + support?: SupportFlags; + modules?: Record; +} + +export interface Catalog { + data: Record; + sourceUrl: string; + fetchedAt: string; +} + +export async function fetchCatalog(): Promise { + const res = await fetch(CATALOG_URL); + if (!res.ok) { + throw new Error(`Failed to fetch catalog: ${res.status} ${res.statusText}`); + } + const json = (await res.json()) as { catalog: Record }; + return { + data: json.catalog, + sourceUrl: CATALOG_URL, + fetchedAt: new Date().toISOString(), + }; +} + +/** + * Module-aware capability check. + * + * Returns true if either the provider-level flag is true, or any module-level + * flag is true. This is permissive: a guide that claims support without + * specifying a module is considered correct as long as some part of the + * provider supports it. Tightening to require module-specific claims is a + * PR 3 concern. + */ +export function supports( + provider: CatalogProvider | undefined, + capability: Capability, +): boolean { + if (!provider) return false; + if (provider.support?.[capability]) return true; + for (const mod of Object.values(provider.modules ?? {})) { + if (mod.support?.[capability]) return true; + } + return false; +} + +/** True if the provider has any module entries. */ +export function hasModules(provider: CatalogProvider | undefined): boolean { + return !!provider?.modules && Object.keys(provider.modules).length > 0; +} + +/** Lists modules that support the given capability. Useful for advisory output. */ +export function modulesSupporting( + provider: CatalogProvider | undefined, + capability: Capability, +): string[] { + if (!provider?.modules) return []; + return Object.entries(provider.modules) + .filter(([, mod]) => mod.support?.[capability]) + .map(([name]) => name); +} diff --git a/scripts/drift-check/docs.ts b/scripts/drift-check/docs.ts new file mode 100644 index 00000000..9ad070b2 --- /dev/null +++ b/scripts/drift-check/docs.ts @@ -0,0 +1,86 @@ +import { promises as fs } from 'node:fs'; +import path from 'node:path'; +import matter from 'gray-matter'; + +const PROVIDER_GUIDES_DIR = path.join(process.cwd(), 'src', 'provider-guides'); + +// Files in src/provider-guides/ that are not provider guides. +const NON_GUIDE_FILES = new Set(['overview.mdx', 'AGENTS.md', 'CLAUDE.md']); + +export type Capability = 'read' | 'write' | 'proxy' | 'subscribe'; + +export interface GuideDoc { + docPath: string; // path relative to repo root + slug: string; // filename without .mdx + providerKeyFromFrontmatter?: string; + guideType?: string; + claimedActions: Set; + sampleLink?: string; // first samples link found, if any +} + +/** Fixed link patterns used in provider guides. */ +const ACTION_PATTERNS: Array<{ cap: Capability; re: RegExp }> = [ + { cap: 'read', re: /\[Read Actions\]\(\/read-actions\)/ }, + { cap: 'write', re: /\[Write Actions\]\(\/write-actions\)/ }, + { cap: 'subscribe', re: /\[Subscribe Actions\]\(\/subscribe-actions\)/ }, + { cap: 'proxy', re: /\[Proxy Actions\]\(\/proxy-actions\)/ }, +]; + +const SAMPLE_LINK_RE = + /https:\/\/github\.com\/amp-labs\/samples\/blob\/[^/]+\/([^/]+)\/amp\.yaml/; + +export async function scanGuides(): Promise { + const entries = await fs.readdir(PROVIDER_GUIDES_DIR); + const guides: GuideDoc[] = []; + for (const entry of entries) { + if (!entry.endsWith('.mdx')) continue; + if (NON_GUIDE_FILES.has(entry)) continue; + const full = path.join(PROVIDER_GUIDES_DIR, entry); + const raw = await fs.readFile(full, 'utf8'); + const parsed = matter(raw); + const slug = entry.replace(/\.mdx$/, ''); + const claimedActions = new Set(); + for (const { cap, re } of ACTION_PATTERNS) { + if (re.test(parsed.content)) claimedActions.add(cap); + } + const sampleMatch = parsed.content.match(SAMPLE_LINK_RE); + guides.push({ + docPath: path.relative(process.cwd(), full), + slug, + providerKeyFromFrontmatter: + typeof parsed.data.provider === 'string' ? parsed.data.provider : undefined, + guideType: + typeof parsed.data.guide_type === 'string' ? parsed.data.guide_type : undefined, + claimedActions, + sampleLink: sampleMatch ? sampleMatch[0] : undefined, + }); + } + return guides; +} + +/** + * Resolve the catalog provider key a guide documents. + * + * Priority: + * 1. frontmatter `provider` field (case preserved) + * 2. slugOverrides recipe + * 3. case-insensitive match against catalog keys (handles the systemic + * "doc files lowercase, catalog mixed-case" convention split, e.g. + * aweber.mdx -> aWeber) + * 4. filename slug verbatim (will surface as guide-without-catalog-entry + * if not in catalog) + */ +export function resolveProviderKey( + guide: GuideDoc, + slugOverrides: Record, + catalogKeys?: readonly string[], +): string { + if (guide.providerKeyFromFrontmatter) return guide.providerKeyFromFrontmatter; + if (slugOverrides[guide.slug]) return slugOverrides[guide.slug]; + if (catalogKeys) { + const ci = guide.slug.toLowerCase(); + const match = catalogKeys.find((k) => k.toLowerCase() === ci); + if (match) return match; + } + return guide.slug; +} diff --git a/scripts/drift-check/index.ts b/scripts/drift-check/index.ts new file mode 100644 index 00000000..47e2339b --- /dev/null +++ b/scripts/drift-check/index.ts @@ -0,0 +1,174 @@ +#!/usr/bin/env tsx +/** + * Drift check v1. + * + * Static comparison of provider guides against the connectors catalog. + * Three checks: + * - undocumented-provider: catalog has a provider with no guide and no + * allowance in recipes.ts + * - doc-overclaims-: guide claims an action the catalog + * (module-aware) does not support + * - broken-sample-link: guide links to a sample manifest that 404s + * + * Modes: + * --mode full scan every provider guide (default) + * --mode per-pr scan only .mdx files passed via --changed + * --mode provider scan a single provider via --provider + * + * Output: + * --out write drift-report.json and drift-report.md + * (default ./drift-report) + */ +import { parseArgs } from 'node:util'; +import path from 'node:path'; +import { + fetchCatalog, + supports, + modulesSupporting, + type Capability, +} from './catalog'; +import { scanGuides, resolveProviderKey, type GuideDoc } from './docs'; +import { checkSampleLink } from './samples'; +import { recipes } from './recipes'; +import { writeReport, type Finding, type Report } from './report'; + +async function main() { + const { values } = parseArgs({ + options: { + mode: { type: 'string', default: 'full' }, + provider: { type: 'string' }, + changed: { type: 'string', multiple: true }, + out: { type: 'string', default: './drift-report' }, + 'fail-on-error': { type: 'boolean', default: false }, + }, + }); + + const mode = values.mode as 'full' | 'per-pr' | 'provider'; + + const catalog = await fetchCatalog(); + const catalogKeys = Object.keys(catalog.data); + let guides = await scanGuides(); + + if (mode === 'per-pr') { + const changedSet = new Set((values.changed ?? []).map((p) => path.normalize(p))); + guides = guides.filter((g) => changedSet.has(path.normalize(g.docPath))); + } else if (mode === 'provider') { + if (!values.provider) { + throw new Error('--provider required with --mode provider'); + } + guides = guides.filter( + (g) => + resolveProviderKey(g, recipes.slugOverrides, catalogKeys) === values.provider, + ); + } + + const findings: Finding[] = []; + + // Check: undocumented providers. Full mode only; otherwise the per-PR slice + // can't tell what's missing. + if (mode === 'full') { + const documentedKeys = new Set( + (await scanGuides()).map((g) => resolveProviderKey(g, recipes.slugOverrides, catalogKeys)), + ); + const today = new Date(); + for (const catalogKey of Object.keys(catalog.data)) { + if (documentedKeys.has(catalogKey)) continue; + const allow = recipes.undocumentedAllowed.find((a) => a.provider === catalogKey); + if (allow) { + if (new Date(allow.until) < today) { + findings.push({ + provider: catalogKey, + severity: 'warning', + kind: 'undocumented-provider-allowance-expired', + suggestedFix: `Renew or remove the allowance for "${catalogKey}" in scripts/drift-check/recipes.ts (current until=${allow.until}).`, + }); + } + continue; + } + findings.push({ + provider: catalogKey, + severity: 'error', + kind: 'undocumented-provider', + suggestedFix: `Add a provider guide at src/provider-guides/${catalogKey}.mdx, or add "${catalogKey}" to undocumentedAllowed in scripts/drift-check/recipes.ts with a reason and an "until" date.`, + }); + } + } + + // Check: doc-overclaims- + collect sample-link checks. + const sampleChecks: Promise[] = []; + for (const guide of guides) { + const providerKey = resolveProviderKey(guide, recipes.slugOverrides, catalogKeys); + const cp = catalog.data[providerKey]; + if (!cp) { + findings.push({ + provider: providerKey, + severity: 'error', + kind: 'guide-without-catalog-entry', + docPath: guide.docPath, + suggestedFix: `The guide resolves to provider key "${providerKey}" but the catalog has no such entry. Check the frontmatter "provider" field or add a slugOverride in recipes.ts.`, + }); + continue; + } + for (const cap of guide.claimedActions) { + if (!supports(cp, cap as Capability)) { + findings.push({ + provider: providerKey, + severity: 'error', + kind: `doc-overclaims-${cap}`, + docPath: guide.docPath, + docClaim: `Guide includes "[${capitalize(cap)} Actions](/${cap}-actions)".`, + catalogTruth: { + providerLevel: cp.support?.[cap as Capability] ?? false, + modulesSupporting: modulesSupporting(cp, cap as Capability), + }, + suggestedFix: `Remove the ${cap} action claim, or update the connector if support was added but not flagged.`, + }); + } + } + if (guide.sampleLink) { + sampleChecks.push( + checkSampleLink(guide.sampleLink).then((r) => { + if (!r.exists) { + findings.push({ + provider: providerKey, + severity: 'error', + kind: 'broken-sample-link', + docPath: guide.docPath, + docClaim: r.url, + catalogTruth: { httpStatus: r.status ?? 'fetch failed' }, + suggestedFix: `Update or remove the sample link. The referenced file does not exist at ${r.url}.`, + }); + } + }), + ); + } + } + await Promise.all(sampleChecks); + + const report: Report = { + catalogSource: `${catalog.sourceUrl} (fetched from amp-labs/connectors@main)`, + checkedAt: catalog.fetchedAt, + mode, + findings, + }; + await writeReport(report, values.out!); + + const errorCount = findings.filter((f) => f.severity === 'error').length; + console.log( + `\n${findings.length} findings (${errorCount} error, ${ + findings.filter((f) => f.severity === 'warning').length + } warning, ${findings.filter((f) => f.severity === 'info').length} info).`, + ); + if (values['fail-on-error'] && errorCount > 0) { + process.exit(1); + } +} + +function capitalize(s: string): string { + return s.charAt(0).toUpperCase() + s.slice(1); +} + +main().catch((err) => { + console.error(err); + process.exit(2); +}); diff --git a/scripts/drift-check/recipes.ts b/scripts/drift-check/recipes.ts new file mode 100644 index 00000000..73984b21 --- /dev/null +++ b/scripts/drift-check/recipes.ts @@ -0,0 +1,107 @@ +// Sparse overrides and allowances for the drift check. +// Only encode exceptions here. Every other case should be conventional. + +export interface UndocumentedAllowance { + provider: string; // exact catalog key + reason: string; + until: string; // ISO date; allowance expires after this +} + +export interface Recipes { + // Maps a doc file slug (filename without .mdx) to the catalog provider key + // when they differ. Use only when the conventional `.mdx` lookup + // does not apply. + slugOverrides: Record; + + // Catalog provider keys that are intentionally undocumented. Anything not + // listed here that lacks a guide will produce an `undocumented-provider` + // error finding. + undocumentedAllowed: UndocumentedAllowance[]; +} + +export const recipes: Recipes = { + slugOverrides: { + // The instantly.mdx guide currently documents the instantlyAI catalog + // entry (V2 API). The instantly catalog entry (Legacy V1) is allowed + // undocumented below. + instantly: 'instantlyAI', + // Filename uses hyphens; catalog keys are camelCase without hyphens. + 'google-workspace-delegation': 'googleWorkspaceDelegation', + 'netsuite-m2m': 'netsuiteM2M', + 'salesforce-jwt': 'salesforceJWT', + // jira.mdx is an alias guide that points readers to the Atlassian guide. + // Mapping it to atlassian lets it inherit the same catalog-side checks. + jira: 'atlassian', + // highlevel.mdx documents the Standard variant; the WhiteLabel variant + // is intentionally undocumented (see undocumentedAllowed below). + highlevel: 'highLevelStandard', + }, + + undocumentedAllowed: [ + { + provider: 'instantly', + reason: 'Legacy V1 connector; current public guide covers instantlyAI.', + until: '2026-09-01', + }, + { + provider: 'adyenTest', + reason: 'Internal test variant; not user-facing.', + until: '2027-01-01', + }, + { + provider: 'gladlyQA', + reason: 'QA-only variant of gladly; not user-facing.', + until: '2027-01-01', + }, + { + provider: 'deelSandbox', + reason: 'Sandbox variant; share docs with deel.', + until: '2027-01-01', + }, + { + provider: 'gustoDemo', + reason: 'Demo variant; share docs with gusto.', + until: '2027-01-01', + }, + { + provider: 'paddleSandbox', + reason: 'Sandbox variant.', + until: '2027-01-01', + }, + { + provider: 'paypalSandbox', + reason: 'Sandbox variant.', + until: '2027-01-01', + }, + { + provider: 'procoreSandbox', + reason: 'Sandbox variant.', + until: '2027-01-01', + }, + { + provider: 'quickbooksSandbox', + reason: 'Sandbox variant.', + until: '2027-01-01', + }, + { + provider: 'rampDemo', + reason: 'Demo variant.', + until: '2027-01-01', + }, + { + provider: 'ironcladDemo', + reason: 'Demo variant.', + until: '2027-01-01', + }, + { + provider: 'leverSandbox', + reason: 'Sandbox variant.', + until: '2027-01-01', + }, + { + provider: 'highLevelWhiteLabel', + reason: 'WhiteLabel variant; the highlevel.mdx guide covers the Standard variant.', + until: '2026-09-01', + }, + ], +}; diff --git a/scripts/drift-check/report.ts b/scripts/drift-check/report.ts new file mode 100644 index 00000000..44328c2a --- /dev/null +++ b/scripts/drift-check/report.ts @@ -0,0 +1,72 @@ +import { promises as fs } from 'node:fs'; +import path from 'node:path'; + +export type Severity = 'error' | 'warning' | 'info'; + +export interface Finding { + provider: string; // exact catalog key, case preserved + module?: string | 'all'; // required for multi-module providers when known + severity: Severity; + kind: string; + docPath?: string; + docClaim?: string; + catalogTruth?: unknown; + suggestedFix?: string; +} + +export interface Report { + catalogSource: string; + checkedAt: string; // ISO timestamp + mode: 'per-pr' | 'full' | 'provider'; + findings: Finding[]; +} + +export async function writeReport(report: Report, outDir: string): Promise { + await fs.mkdir(outDir, { recursive: true }); + const jsonPath = path.join(outDir, 'drift-report.json'); + const mdPath = path.join(outDir, 'drift-report.md'); + await fs.writeFile(jsonPath, JSON.stringify(report, null, 2) + '\n'); + await fs.writeFile(mdPath, toMarkdown(report)); + console.log(`Wrote ${jsonPath}`); + console.log(`Wrote ${mdPath}`); +} + +function toMarkdown(report: Report): string { + const bySeverity: Record = { error: [], warning: [], info: [] }; + for (const f of report.findings) bySeverity[f.severity].push(f); + + const lines: string[] = []; + lines.push('# Drift check report'); + lines.push(''); + lines.push(`- Catalog source: \`${report.catalogSource}\``); + lines.push(`- Checked at: \`${report.checkedAt}\``); + lines.push(`- Mode: \`${report.mode}\``); + lines.push(`- Totals: ${bySeverity.error.length} error, ${bySeverity.warning.length} warning, ${bySeverity.info.length} info`); + lines.push(''); + + for (const sev of ['error', 'warning', 'info'] as const) { + const group = bySeverity[sev]; + if (group.length === 0) continue; + lines.push(`## ${capitalize(sev)} (${group.length})`); + lines.push(''); + for (const f of group.sort(sortByProvider)) { + const mod = f.module ? ` [${f.module}]` : ''; + lines.push(`### ${f.provider}${mod} — ${f.kind}`); + if (f.docPath) lines.push(`- File: \`${f.docPath}\``); + if (f.docClaim) lines.push(`- Doc says: ${f.docClaim}`); + if (f.catalogTruth !== undefined) lines.push(`- Catalog says: \`${JSON.stringify(f.catalogTruth)}\``); + if (f.suggestedFix) lines.push(`- Fix: ${f.suggestedFix}`); + lines.push(''); + } + } + return lines.join('\n'); +} + +function sortByProvider(a: Finding, b: Finding): number { + if (a.provider !== b.provider) return a.provider.localeCompare(b.provider); + return (a.kind ?? '').localeCompare(b.kind ?? ''); +} + +function capitalize(s: string): string { + return s.charAt(0).toUpperCase() + s.slice(1); +} diff --git a/scripts/drift-check/samples.ts b/scripts/drift-check/samples.ts new file mode 100644 index 00000000..4791aad7 --- /dev/null +++ b/scripts/drift-check/samples.ts @@ -0,0 +1,26 @@ +/** + * Resolve a github.com/...blob/... URL to its raw equivalent, since blob URLs + * return a 200 HTML page even for missing files. The raw URL returns 404 on + * missing files, which is what we want for existence checking. + */ +function toRawUrl(blobUrl: string): string { + return blobUrl + .replace('github.com/', 'raw.githubusercontent.com/') + .replace('/blob/', '/'); +} + +export interface SampleCheckResult { + url: string; + exists: boolean; + status?: number; +} + +export async function checkSampleLink(blobUrl: string): Promise { + const rawUrl = toRawUrl(blobUrl); + try { + const res = await fetch(rawUrl, { method: 'HEAD' }); + return { url: blobUrl, exists: res.ok, status: res.status }; + } catch { + return { url: blobUrl, exists: false }; + } +} From c5d1c1be9f74f2509c5b79f577c37b418f3f5a8f Mon Sep 17 00:00:00 2001 From: Caio Pizzol Date: Sun, 17 May 2026 18:10:49 -0300 Subject: [PATCH 2/8] scripts(drift-check): correctness fixes from review - recipes.ts: fix payPalSandBox case (catalog uses payPalSandBox, not paypalSandbox); the typo caused a false-positive undocumented-provider - catalog.ts: add 'search' to Capability with object-shape handling (search support is {operators: {equals: bool}}, not a flat boolean) - docs.ts: extract all distinct sample links per guide, not just the first. Accept both amp.yaml and amp.yml extensions. Surfaces google, netsuite-m2m, and salesforce multi-sample cases - docs.ts: add Search Actions to the action-link pattern set - index.ts: enforce module-aware schema by injecting module: 'all' when the provider has modules and the finding does not specify one - index.ts: accept both 'drift-check --out X' and the 'pnpm run drift-check -- --out X' forms by stripping leading '--' Findings: 42 -> 41 (false positive removed). aws and seismic findings now carry module: 'all'. CLI forms both work; --fail-on-error exits nonzero when errors are present. --- scripts/drift-check/catalog.ts | 24 +++++++++- scripts/drift-check/docs.ts | 16 ++++--- scripts/drift-check/index.ts | 82 ++++++++++++++++++++++++---------- scripts/drift-check/recipes.ts | 2 +- 4 files changed, 93 insertions(+), 31 deletions(-) diff --git a/scripts/drift-check/catalog.ts b/scripts/drift-check/catalog.ts index e1ea8640..2ec0b9de 100644 --- a/scripts/drift-check/catalog.ts +++ b/scripts/drift-check/catalog.ts @@ -1,13 +1,18 @@ const CATALOG_URL = 'https://raw.githubusercontent.com/amp-labs/connectors/main/internal/generated/catalog.json'; -export type Capability = 'read' | 'write' | 'proxy' | 'subscribe'; +export type Capability = 'read' | 'write' | 'proxy' | 'subscribe' | 'search'; + +interface SearchSupport { + operators?: Record; +} interface SupportFlags { read?: boolean; write?: boolean; proxy?: boolean; subscribe?: boolean; + search?: SearchSupport; } interface ModuleEntry { @@ -59,6 +64,13 @@ export function supports( capability: Capability, ): boolean { if (!provider) return false; + if (capability === 'search') { + if (hasAnySearchOperator(provider.support?.search)) return true; + for (const mod of Object.values(provider.modules ?? {})) { + if (hasAnySearchOperator(mod.support?.search)) return true; + } + return false; + } if (provider.support?.[capability]) return true; for (const mod of Object.values(provider.modules ?? {})) { if (mod.support?.[capability]) return true; @@ -66,6 +78,11 @@ export function supports( return false; } +function hasAnySearchOperator(s: SearchSupport | undefined): boolean { + if (!s?.operators) return false; + return Object.values(s.operators).some(Boolean); +} + /** True if the provider has any module entries. */ export function hasModules(provider: CatalogProvider | undefined): boolean { return !!provider?.modules && Object.keys(provider.modules).length > 0; @@ -77,6 +94,11 @@ export function modulesSupporting( capability: Capability, ): string[] { if (!provider?.modules) return []; + if (capability === 'search') { + return Object.entries(provider.modules) + .filter(([, mod]) => hasAnySearchOperator(mod.support?.search)) + .map(([name]) => name); + } return Object.entries(provider.modules) .filter(([, mod]) => mod.support?.[capability]) .map(([name]) => name); diff --git a/scripts/drift-check/docs.ts b/scripts/drift-check/docs.ts index 9ad070b2..d73c5a13 100644 --- a/scripts/drift-check/docs.ts +++ b/scripts/drift-check/docs.ts @@ -7,7 +7,7 @@ const PROVIDER_GUIDES_DIR = path.join(process.cwd(), 'src', 'provider-guides'); // Files in src/provider-guides/ that are not provider guides. const NON_GUIDE_FILES = new Set(['overview.mdx', 'AGENTS.md', 'CLAUDE.md']); -export type Capability = 'read' | 'write' | 'proxy' | 'subscribe'; +export type Capability = 'read' | 'write' | 'proxy' | 'subscribe' | 'search'; export interface GuideDoc { docPath: string; // path relative to repo root @@ -15,7 +15,7 @@ export interface GuideDoc { providerKeyFromFrontmatter?: string; guideType?: string; claimedActions: Set; - sampleLink?: string; // first samples link found, if any + sampleLinks: string[]; // all distinct samples links found } /** Fixed link patterns used in provider guides. */ @@ -24,10 +24,12 @@ const ACTION_PATTERNS: Array<{ cap: Capability; re: RegExp }> = [ { cap: 'write', re: /\[Write Actions\]\(\/write-actions\)/ }, { cap: 'subscribe', re: /\[Subscribe Actions\]\(\/subscribe-actions\)/ }, { cap: 'proxy', re: /\[Proxy Actions\]\(\/proxy-actions\)/ }, + { cap: 'search', re: /\[Search Actions\]\(\/search-actions\)/ }, ]; -const SAMPLE_LINK_RE = - /https:\/\/github\.com\/amp-labs\/samples\/blob\/[^/]+\/([^/]+)\/amp\.yaml/; +// Match every distinct samples link, allowing both amp.yaml and amp.yml. +const SAMPLE_LINK_RE_G = + /https:\/\/github\.com\/amp-labs\/samples\/blob\/[^/]+\/[^/]+\/amp\.ya?ml/g; export async function scanGuides(): Promise { const entries = await fs.readdir(PROVIDER_GUIDES_DIR); @@ -43,7 +45,9 @@ export async function scanGuides(): Promise { for (const { cap, re } of ACTION_PATTERNS) { if (re.test(parsed.content)) claimedActions.add(cap); } - const sampleMatch = parsed.content.match(SAMPLE_LINK_RE); + const sampleLinks = Array.from( + new Set(parsed.content.match(SAMPLE_LINK_RE_G) ?? []), + ); guides.push({ docPath: path.relative(process.cwd(), full), slug, @@ -52,7 +56,7 @@ export async function scanGuides(): Promise { guideType: typeof parsed.data.guide_type === 'string' ? parsed.data.guide_type : undefined, claimedActions, - sampleLink: sampleMatch ? sampleMatch[0] : undefined, + sampleLinks, }); } return guides; diff --git a/scripts/drift-check/index.ts b/scripts/drift-check/index.ts index 47e2339b..2c72fc63 100644 --- a/scripts/drift-check/index.ts +++ b/scripts/drift-check/index.ts @@ -23,9 +23,11 @@ import { parseArgs } from 'node:util'; import path from 'node:path'; import { fetchCatalog, + hasModules, supports, modulesSupporting, type Capability, + type CatalogProvider, } from './catalog'; import { scanGuides, resolveProviderKey, type GuideDoc } from './docs'; import { checkSampleLink } from './samples'; @@ -33,7 +35,12 @@ import { recipes } from './recipes'; import { writeReport, type Finding, type Report } from './report'; async function main() { + // Accept both `drift-check --out X` and `pnpm run drift-check -- --out X` + // by stripping a leading `--` separator if present. + const args = process.argv.slice(2); + if (args[0] === '--') args.shift(); const { values } = parseArgs({ + args, options: { mode: { type: 'string', default: 'full' }, provider: { type: 'string' }, @@ -111,33 +118,37 @@ async function main() { } for (const cap of guide.claimedActions) { if (!supports(cp, cap as Capability)) { - findings.push({ - provider: providerKey, - severity: 'error', - kind: `doc-overclaims-${cap}`, - docPath: guide.docPath, - docClaim: `Guide includes "[${capitalize(cap)} Actions](/${cap}-actions)".`, - catalogTruth: { - providerLevel: cp.support?.[cap as Capability] ?? false, - modulesSupporting: modulesSupporting(cp, cap as Capability), - }, - suggestedFix: `Remove the ${cap} action claim, or update the connector if support was added but not flagged.`, - }); + findings.push( + withModule(cp, { + provider: providerKey, + severity: 'error', + kind: `doc-overclaims-${cap}`, + docPath: guide.docPath, + docClaim: `Guide includes "[${capitalize(cap)} Actions](/${cap}-actions)".`, + catalogTruth: { + providerLevel: capabilityValueAt(cp, cap as Capability, 'provider'), + modulesSupporting: modulesSupporting(cp, cap as Capability), + }, + suggestedFix: `Remove the ${cap} action claim, or update the connector if support was added but not flagged.`, + }), + ); } } - if (guide.sampleLink) { + for (const sampleLink of guide.sampleLinks) { sampleChecks.push( - checkSampleLink(guide.sampleLink).then((r) => { + checkSampleLink(sampleLink).then((r) => { if (!r.exists) { - findings.push({ - provider: providerKey, - severity: 'error', - kind: 'broken-sample-link', - docPath: guide.docPath, - docClaim: r.url, - catalogTruth: { httpStatus: r.status ?? 'fetch failed' }, - suggestedFix: `Update or remove the sample link. The referenced file does not exist at ${r.url}.`, - }); + findings.push( + withModule(cp, { + provider: providerKey, + severity: 'error', + kind: 'broken-sample-link', + docPath: guide.docPath, + docClaim: r.url, + catalogTruth: { httpStatus: r.status ?? 'fetch failed' }, + suggestedFix: `Update or remove the sample link. The referenced file does not exist at ${r.url}.`, + }), + ); } }), ); @@ -168,6 +179,31 @@ function capitalize(s: string): string { return s.charAt(0).toUpperCase() + s.slice(1); } +/** + * Enforce the module-aware schema: if the provider has modules and the + * finding does not specify one, default to "all". Single-module providers + * leave module unset. + */ +function withModule( + provider: CatalogProvider | undefined, + finding: Finding, +): Finding { + if (finding.module !== undefined) return finding; + if (provider && hasModules(provider)) { + return { ...finding, module: 'all' }; + } + return finding; +} + +function capabilityValueAt( + cp: CatalogProvider, + cap: Capability, + _level: 'provider', +): unknown { + if (cap === 'search') return cp.support?.search ?? false; + return cp.support?.[cap] ?? false; +} + main().catch((err) => { console.error(err); process.exit(2); diff --git a/scripts/drift-check/recipes.ts b/scripts/drift-check/recipes.ts index 73984b21..0ce7a482 100644 --- a/scripts/drift-check/recipes.ts +++ b/scripts/drift-check/recipes.ts @@ -69,7 +69,7 @@ export const recipes: Recipes = { until: '2027-01-01', }, { - provider: 'paypalSandbox', + provider: 'payPalSandBox', reason: 'Sandbox variant.', until: '2027-01-01', }, From 972804b207d6228ede60cd9995fef9d2541f72de Mon Sep 17 00:00:00 2001 From: Caio Pizzol Date: Sun, 17 May 2026 18:11:19 -0300 Subject: [PATCH 3/8] scripts(drift-check): add README documenting modes, recipe rules, and v1 scope --- scripts/drift-check/README.md | 110 ++++++++++++++++++++++++++++++++++ 1 file changed, 110 insertions(+) create mode 100644 scripts/drift-check/README.md diff --git a/scripts/drift-check/README.md b/scripts/drift-check/README.md new file mode 100644 index 00000000..0ecceb6b --- /dev/null +++ b/scripts/drift-check/README.md @@ -0,0 +1,110 @@ +# Drift check + +Static comparison of provider guides against the connectors catalog. Detects three classes of drift between the docs at `src/provider-guides/` and the catalog at [`amp-labs/connectors/internal/generated/catalog.json`](https://github.com/amp-labs/connectors/blob/main/internal/generated/catalog.json). + +## Quick start + +```shell +pnpm run drift-check # full sweep, default ./drift-report/ +pnpm run drift-check -- --out /tmp/out # custom output directory +pnpm run drift-check -- --mode provider --provider hubspot +pnpm run drift-check -- --fail-on-error # exit non-zero on any error finding +``` + +Both `pnpm run drift-check --out X` and `pnpm run drift-check -- --out X` work. The script writes two files to the output directory: + +- `drift-report.json`: canonical findings, one object per finding. +- `drift-report.md`: human-readable rollup grouped by severity. + +## What it checks + +| Kind | Severity | What triggers it | +|---|---|---| +| `undocumented-provider` | error | Catalog has a provider; no guide exists; no allowance in `recipes.ts` | +| `undocumented-provider-allowance-expired` | warning | An entry in `undocumentedAllowed` has passed its `until` date | +| `guide-without-catalog-entry` | error | A guide resolves to a catalog key that does not exist | +| `doc-overclaims-{read,write,subscribe,proxy,search}` | error | Guide claims an action; neither provider-level nor any module-level support flag is true | +| `broken-sample-link` | error | Guide links to a `samples` manifest that 404s | + +The check is **module-aware**: a guide claim is considered valid if either the provider-level flag is true *or* any module-level flag is true. So Google's `subscribe` claim is valid because `modules.gmail.support.subscribe == true`, even though `support.subscribe == false` at the provider level. + +For multi-module providers, findings include `"module": "all"` since the v1 detector does not attempt to attribute the claim to a specific module. + +## Modes + +- **`--mode full`** (default): scan every guide and check every catalog provider for missing docs. +- **`--mode per-pr --changed ...`**: scan only the listed `.mdx` files. Skips the undocumented-provider check (it requires a global view). Suitable for per-PR CI where you only want to validate touched files. +- **`--mode provider --provider `**: scan only the guide that resolves to the given catalog key. Useful for guide authors iterating locally. + +## Resolving guide slug to catalog key + +Guides resolve to a catalog key in this order: + +1. Frontmatter `provider` field, case preserved. +2. `slugOverrides` in `recipes.ts` (for genuine name differences, not case). +3. Case-insensitive match against catalog keys (handles `aweber.mdx` → `aWeber`). +4. Filename slug, verbatim (fails as `guide-without-catalog-entry` if no catalog entry). + +This means most case differences between filenames and catalog keys (73 of 227 catalog keys are mixed-case) need no recipe at all. + +## Adding a recipe + +Edit `recipes.ts`. Two surfaces: + +### `slugOverrides` + +Use only when the filename slug and the catalog key are genuinely different strings, not just case differences. Examples: + +```ts +slugOverrides: { + instantly: 'instantlyAI', // file documents the V2 catalog entry + 'google-workspace-delegation': 'googleWorkspaceDelegation', // hyphen vs camelCase + jira: 'atlassian', // alias guide for a family +} +``` + +### `undocumentedAllowed` + +Use when a catalog entry intentionally has no guide. Every allowance has a reason and an `until` date so it cannot persist forever. + +```ts +undocumentedAllowed: [ + { + provider: 'instantly', + reason: 'Legacy V1 connector; current public guide covers instantlyAI.', + until: '2026-09-01', + }, +] +``` + +When `until` passes, the entry becomes a warning (`undocumented-provider-allowance-expired`) until renewed or removed. + +## What v1 deliberately does not check + +These are out of scope for v1. PR 3 or later may add them. + +- **Object-name drift between guides and connector source.** The connector source under `providers//` enumerates supported objects; v1 does not parse Go. +- **Provider public-doc drift.** Comparing each guide's setup steps against the provider's own developer documentation requires per-provider HTML parsing. +- **Provider product-UI drift.** Validating screenshots and GIFs against the actual provider dashboards requires real accounts, MFA, and is explicitly out of scope. +- **Runtime smoke tests.** Actually exercising read/write/proxy against a live provider requires sandbox accounts and is a separate program. + +## Severity → CI behavior + +When wired into CI (PR 3), the intended mapping: + +- **error**: fails CI when the change is on a touched file (per-PR mode). +- **warning**: reported, never fails CI. +- **info**: reported, never fails CI. + +The script accepts `--fail-on-error` for the CI invocation. + +## Files + +| File | Purpose | +|---|---| +| `index.ts` | CLI entry; orchestrates the three checks | +| `catalog.ts` | Fetch `catalog.json`; module-aware capability helpers | +| `docs.ts` | Scan `.mdx` files; extract frontmatter and the five action-link patterns | +| `samples.ts` | HEAD-check each sample manifest URL | +| `recipes.ts` | `slugOverrides` and `undocumentedAllowed` | +| `report.ts` | Finding type, JSON canonical output, Markdown rollup | From fc08acfecf055d9e543818d51e7b9b65bbd711d4 Mon Sep 17 00:00:00 2001 From: Caio Pizzol Date: Sun, 17 May 2026 18:21:32 -0300 Subject: [PATCH 4/8] scripts(drift-check): clarify what the check does not prove The detector compares docs against the connector catalog. It does not validate the deployed API against its declared shape, and it does not prove that read/write/proxy/subscribe calls succeed at runtime. List the specific drift surfaces left out of v1 so users do not overstate what the green report means. Verified today: connector catalog.json (227 providers) and live /v1/providers (227 providers) are in lockstep across keys, authType, baseURL, displayName, defaultModule, and top-level support flags. So catalog-vs-live drift is currently theoretical; it can become a separate cheap check in a follow-up PR. --- scripts/drift-check/README.md | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/scripts/drift-check/README.md b/scripts/drift-check/README.md index 0ecceb6b..97595cfa 100644 --- a/scripts/drift-check/README.md +++ b/scripts/drift-check/README.md @@ -79,14 +79,20 @@ undocumentedAllowed: [ When `until` passes, the entry becomes a warning (`undocumented-provider-allowance-expired`) until renewed or removed. -## What v1 deliberately does not check +## What this check does not prove -These are out of scope for v1. PR 3 or later may add them. +This check validates docs against the connector catalog. It does **not** prove that read/write/proxy/subscribe calls succeed at runtime, and it does not validate the deployed Ampersand API against its declared shape. -- **Object-name drift between guides and connector source.** The connector source under `providers//` enumerates supported objects; v1 does not parse Go. -- **Provider public-doc drift.** Comparing each guide's setup steps against the provider's own developer documentation requires per-provider HTML parsing. -- **Provider product-UI drift.** Validating screenshots and GIFs against the actual provider dashboards requires real accounts, MFA, and is explicitly out of scope. -- **Runtime smoke tests.** Actually exercising read/write/proxy against a live provider requires sandbox accounts and is a separate program. +Concretely, the drift surfaces this check ignores: + +| Surface | Out of scope because | +|---|---| +| **Connector Go source vs generated `catalog.json`** | The connectors repo generates the catalog from source. Regeneration lag is the connectors team's problem, not the docs' problem. | +| **Generated `catalog.json` vs live `/v1/providers`** | Deployment lag. Catalog-vs-live is a separate cheap check (no credentials needed); track it under a follow-up PR. | +| **Live API declared shape vs actual runtime behavior** | Whether `read` actually works requires sandbox accounts, installations, and real provider credentials. That is a separate program, not a docs-repo check. | +| **Object-name drift between guides and connector source** | Requires parsing Go. Deferred to a later PR that adds `connectors.ts`. | +| **Provider public-doc drift** | Comparing guide steps against the provider's own developer documentation requires per-provider HTML parsing. | +| **Provider product-UI drift** | Validating screenshots and GIFs against live provider dashboards requires real accounts and MFA. Manual surface. | ## Severity → CI behavior From a2994dfcd2c3a6980720639487761af7aa2cb6ab Mon Sep 17 00:00:00 2001 From: Caio Pizzol Date: Sun, 17 May 2026 18:23:02 -0300 Subject: [PATCH 5/8] scripts(drift-check): retry transient sample-link failures; sharpen README samples.ts: HEAD against GitHub raw is occasionally flaky (transient 5xx or network blips). Trust a 404 immediately, but retry once on non-404 errors with a short backoff to avoid false-positive broken-sample-link findings in CI. README: add an explicit one-line summary at the top of the 'does not prove' section: this check uses the generated catalog from main, not the live /v1/providers API or runtime behavior. --- scripts/drift-check/README.md | 4 +++- scripts/drift-check/samples.ts | 26 +++++++++++++++++++++----- 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/scripts/drift-check/README.md b/scripts/drift-check/README.md index 97595cfa..cd0518bb 100644 --- a/scripts/drift-check/README.md +++ b/scripts/drift-check/README.md @@ -81,7 +81,9 @@ When `until` passes, the entry becomes a warning (`undocumented-provider-allowan ## What this check does not prove -This check validates docs against the connector catalog. It does **not** prove that read/write/proxy/subscribe calls succeed at runtime, and it does not validate the deployed Ampersand API against its declared shape. +This check uses the generated connectors catalog from `main`; it does not compare against the deployed `/v1/providers` API or prove runtime API behavior. + +It does **not** prove that read/write/proxy/subscribe calls succeed at runtime, and it does not validate the deployed Ampersand API against its declared shape. Concretely, the drift surfaces this check ignores: diff --git a/scripts/drift-check/samples.ts b/scripts/drift-check/samples.ts index 4791aad7..838c57de 100644 --- a/scripts/drift-check/samples.ts +++ b/scripts/drift-check/samples.ts @@ -15,12 +15,28 @@ export interface SampleCheckResult { status?: number; } +/** + * Trusts a 404 immediately (definitive missing). Retries once on network + * errors or non-404 non-2xx responses, since GitHub raw can produce + * transient 5xx / connection blips that would otherwise become false + * positives in CI. + */ export async function checkSampleLink(blobUrl: string): Promise { const rawUrl = toRawUrl(blobUrl); - try { - const res = await fetch(rawUrl, { method: 'HEAD' }); - return { url: blobUrl, exists: res.ok, status: res.status }; - } catch { - return { url: blobUrl, exists: false }; + const MAX_ATTEMPTS = 2; + let lastStatus: number | undefined; + for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) { + try { + const res = await fetch(rawUrl, { method: 'HEAD' }); + if (res.ok) return { url: blobUrl, exists: true, status: res.status }; + if (res.status === 404) return { url: blobUrl, exists: false, status: 404 }; + lastStatus = res.status; + } catch { + // network error; retry + } + if (attempt < MAX_ATTEMPTS) { + await new Promise((r) => setTimeout(r, 250 * attempt)); + } } + return { url: blobUrl, exists: false, status: lastStatus }; } From 32972efbdf9dcc8befaab1068e8a685d71b541da Mon Sep 17 00:00:00 2001 From: Caio Pizzol Date: Sun, 17 May 2026 19:26:25 -0300 Subject: [PATCH 6/8] scripts(drift-check): tighten docs and comments, no behavior change - README trimmed from 118 to 66 lines: cut file inventory and repeated CI behavior; compressed the out-of-scope table; kept the slugOverrides example and the does-not-prove disclaimer. - index.ts: drop the file header (README covers it); rename cp -> provider for readability; drop the unused _level parameter and rename the helper to providerLevelSupport; drop the unused GuideDoc import. - catalog.ts: shorten supports() comment to one line; rename hasAnySearchOperator -> supportsSearch. - docs.ts: shorten the resolveProviderKey comment; rename SAMPLE_LINK_RE_G -> SAMPLE_LINK_RE; drop trailing field comments. - samples.ts: compress the blob-to-raw and retry comments. - recipes.ts: drop interface/inline commentary on obvious entries; keep the comments that explain non-obvious mappings (instantly, jira, highlevel). - report.ts: unchanged. Total LOC 744 -> 572 (-23%). Detector still produces the same 41 findings against main. --- scripts/drift-check/README.md | 114 +++++++++------------------------ scripts/drift-check/catalog.ts | 22 ++----- scripts/drift-check/docs.ts | 26 ++------ scripts/drift-check/index.ts | 72 ++++++--------------- scripts/drift-check/recipes.ts | 95 ++++++--------------------- scripts/drift-check/samples.ts | 13 +--- 6 files changed, 85 insertions(+), 257 deletions(-) diff --git a/scripts/drift-check/README.md b/scripts/drift-check/README.md index cd0518bb..18b6d988 100644 --- a/scripts/drift-check/README.md +++ b/scripts/drift-check/README.md @@ -1,118 +1,66 @@ # Drift check -Static comparison of provider guides against the connectors catalog. Detects three classes of drift between the docs at `src/provider-guides/` and the catalog at [`amp-labs/connectors/internal/generated/catalog.json`](https://github.com/amp-labs/connectors/blob/main/internal/generated/catalog.json). +Compares provider-guide claims against the generated [`amp-labs/connectors`](https://github.com/amp-labs/connectors/blob/main/internal/generated/catalog.json) catalog. Static, no credentials, no runtime calls. ## Quick start ```shell -pnpm run drift-check # full sweep, default ./drift-report/ -pnpm run drift-check -- --out /tmp/out # custom output directory +pnpm run drift-check # full sweep, default ./drift-report/ +pnpm run drift-check -- --out /tmp/out # custom output directory pnpm run drift-check -- --mode provider --provider hubspot -pnpm run drift-check -- --fail-on-error # exit non-zero on any error finding +pnpm run drift-check -- --fail-on-error # exit non-zero on any error finding ``` -Both `pnpm run drift-check --out X` and `pnpm run drift-check -- --out X` work. The script writes two files to the output directory: +Both `--out X` and `-- --out X` (the pnpm forwarding form) work. Output is written to `drift-report.json` (canonical) and `drift-report.md` (human-readable rollup). -- `drift-report.json`: canonical findings, one object per finding. -- `drift-report.md`: human-readable rollup grouped by severity. +## Modes + +- `full` (default): scan every guide; run the undocumented-provider check. +- `per-pr --changed ...`: scan only the listed `.mdx` files. Skips the undocumented check (needs a global view). +- `provider --provider `: scan a single guide. Useful while iterating. -## What it checks +## Finding types -| Kind | Severity | What triggers it | +| Kind | Severity | Triggers | |---|---|---| -| `undocumented-provider` | error | Catalog has a provider; no guide exists; no allowance in `recipes.ts` | +| `undocumented-provider` | error | Catalog has a provider; no guide; no allowance | | `undocumented-provider-allowance-expired` | warning | An entry in `undocumentedAllowed` has passed its `until` date | | `guide-without-catalog-entry` | error | A guide resolves to a catalog key that does not exist | | `doc-overclaims-{read,write,subscribe,proxy,search}` | error | Guide claims an action; neither provider-level nor any module-level support flag is true | | `broken-sample-link` | error | Guide links to a `samples` manifest that 404s | -The check is **module-aware**: a guide claim is considered valid if either the provider-level flag is true *or* any module-level flag is true. So Google's `subscribe` claim is valid because `modules.gmail.support.subscribe == true`, even though `support.subscribe == false` at the provider level. - -For multi-module providers, findings include `"module": "all"` since the v1 detector does not attempt to attribute the claim to a specific module. - -## Modes - -- **`--mode full`** (default): scan every guide and check every catalog provider for missing docs. -- **`--mode per-pr --changed ...`**: scan only the listed `.mdx` files. Skips the undocumented-provider check (it requires a global view). Suitable for per-PR CI where you only want to validate touched files. -- **`--mode provider --provider `**: scan only the guide that resolves to the given catalog key. Useful for guide authors iterating locally. +Module-aware: a claim is valid if either the provider-level flag is true *or* any module-level flag is true. Multi-module provider findings carry `"module": "all"` since v1 does not attribute the claim to a specific module. -## Resolving guide slug to catalog key +## What this does not prove -Guides resolve to a catalog key in this order: +This check uses the generated connectors catalog from `main`. It does **not** validate the deployed `/v1/providers` API against its declared shape, and it does **not** prove that read/write/proxy/subscribe calls succeed at runtime. Several drift surfaces are deliberately out of scope: -1. Frontmatter `provider` field, case preserved. -2. `slugOverrides` in `recipes.ts` (for genuine name differences, not case). -3. Case-insensitive match against catalog keys (handles `aweber.mdx` → `aWeber`). -4. Filename slug, verbatim (fails as `guide-without-catalog-entry` if no catalog entry). - -This means most case differences between filenames and catalog keys (73 of 227 catalog keys are mixed-case) need no recipe at all. +| Surface | Why out of scope | +|---|---| +| Connector Go source vs generated catalog | Regeneration lag; lives in the connectors repo | +| Generated catalog vs live `/v1/providers` | Deployment lag; separate cheap check | +| Live API declared shape vs runtime behavior | Requires sandbox accounts; separate program | +| Guides vs connector source object names | Requires parsing Go; deferred to a follow-up PR | +| Guides vs provider's own developer docs | Per-provider HTML parsing | +| Guides vs provider product UIs (screenshots, GIFs) | Manual surface; account + MFA | ## Adding a recipe -Edit `recipes.ts`. Two surfaces: - -### `slugOverrides` +Edit `recipes.ts`. Two surfaces, both intentionally sparse. -Use only when the filename slug and the catalog key are genuinely different strings, not just case differences. Examples: +**`slugOverrides`** — when the filename slug and the catalog key are different *strings* (not just different cases — case differences are handled automatically): ```ts slugOverrides: { - instantly: 'instantlyAI', // file documents the V2 catalog entry - 'google-workspace-delegation': 'googleWorkspaceDelegation', // hyphen vs camelCase - jira: 'atlassian', // alias guide for a family + instantly: 'instantlyAI', + jira: 'atlassian', } ``` -### `undocumentedAllowed` - -Use when a catalog entry intentionally has no guide. Every allowance has a reason and an `until` date so it cannot persist forever. +**`undocumentedAllowed`** — when a catalog entry intentionally has no guide. Every entry needs a reason and a time-bounded `until` so the allowance cannot persist forever: ```ts -undocumentedAllowed: [ - { - provider: 'instantly', - reason: 'Legacy V1 connector; current public guide covers instantlyAI.', - until: '2026-09-01', - }, -] +{ provider: 'adyenTest', reason: 'Internal test variant.', until: '2027-01-01' } ``` -When `until` passes, the entry becomes a warning (`undocumented-provider-allowance-expired`) until renewed or removed. - -## What this check does not prove - -This check uses the generated connectors catalog from `main`; it does not compare against the deployed `/v1/providers` API or prove runtime API behavior. - -It does **not** prove that read/write/proxy/subscribe calls succeed at runtime, and it does not validate the deployed Ampersand API against its declared shape. - -Concretely, the drift surfaces this check ignores: - -| Surface | Out of scope because | -|---|---| -| **Connector Go source vs generated `catalog.json`** | The connectors repo generates the catalog from source. Regeneration lag is the connectors team's problem, not the docs' problem. | -| **Generated `catalog.json` vs live `/v1/providers`** | Deployment lag. Catalog-vs-live is a separate cheap check (no credentials needed); track it under a follow-up PR. | -| **Live API declared shape vs actual runtime behavior** | Whether `read` actually works requires sandbox accounts, installations, and real provider credentials. That is a separate program, not a docs-repo check. | -| **Object-name drift between guides and connector source** | Requires parsing Go. Deferred to a later PR that adds `connectors.ts`. | -| **Provider public-doc drift** | Comparing guide steps against the provider's own developer documentation requires per-provider HTML parsing. | -| **Provider product-UI drift** | Validating screenshots and GIFs against live provider dashboards requires real accounts and MFA. Manual surface. | - -## Severity → CI behavior - -When wired into CI (PR 3), the intended mapping: - -- **error**: fails CI when the change is on a touched file (per-PR mode). -- **warning**: reported, never fails CI. -- **info**: reported, never fails CI. - -The script accepts `--fail-on-error` for the CI invocation. - -## Files - -| File | Purpose | -|---|---| -| `index.ts` | CLI entry; orchestrates the three checks | -| `catalog.ts` | Fetch `catalog.json`; module-aware capability helpers | -| `docs.ts` | Scan `.mdx` files; extract frontmatter and the five action-link patterns | -| `samples.ts` | HEAD-check each sample manifest URL | -| `recipes.ts` | `slugOverrides` and `undocumentedAllowed` | -| `report.ts` | Finding type, JSON canonical output, Markdown rollup | +When `until` passes, the entry becomes an `undocumented-provider-allowance-expired` warning until renewed or removed. diff --git a/scripts/drift-check/catalog.ts b/scripts/drift-check/catalog.ts index 2ec0b9de..81318c58 100644 --- a/scripts/drift-check/catalog.ts +++ b/scripts/drift-check/catalog.ts @@ -50,24 +50,18 @@ export async function fetchCatalog(): Promise { }; } -/** - * Module-aware capability check. - * - * Returns true if either the provider-level flag is true, or any module-level - * flag is true. This is permissive: a guide that claims support without - * specifying a module is considered correct as long as some part of the - * provider supports it. Tightening to require module-specific claims is a - * PR 3 concern. - */ +// Module-aware: returns true if the provider-level flag is true, or any +// module-level flag is true. Catches cases like Google, where the provider +// has subscribe=false but modules.gmail has subscribe=true. export function supports( provider: CatalogProvider | undefined, capability: Capability, ): boolean { if (!provider) return false; if (capability === 'search') { - if (hasAnySearchOperator(provider.support?.search)) return true; + if (supportsSearch(provider.support?.search)) return true; for (const mod of Object.values(provider.modules ?? {})) { - if (hasAnySearchOperator(mod.support?.search)) return true; + if (supportsSearch(mod.support?.search)) return true; } return false; } @@ -78,17 +72,15 @@ export function supports( return false; } -function hasAnySearchOperator(s: SearchSupport | undefined): boolean { +function supportsSearch(s: SearchSupport | undefined): boolean { if (!s?.operators) return false; return Object.values(s.operators).some(Boolean); } -/** True if the provider has any module entries. */ export function hasModules(provider: CatalogProvider | undefined): boolean { return !!provider?.modules && Object.keys(provider.modules).length > 0; } -/** Lists modules that support the given capability. Useful for advisory output. */ export function modulesSupporting( provider: CatalogProvider | undefined, capability: Capability, @@ -96,7 +88,7 @@ export function modulesSupporting( if (!provider?.modules) return []; if (capability === 'search') { return Object.entries(provider.modules) - .filter(([, mod]) => hasAnySearchOperator(mod.support?.search)) + .filter(([, mod]) => supportsSearch(mod.support?.search)) .map(([name]) => name); } return Object.entries(provider.modules) diff --git a/scripts/drift-check/docs.ts b/scripts/drift-check/docs.ts index d73c5a13..caa36ed3 100644 --- a/scripts/drift-check/docs.ts +++ b/scripts/drift-check/docs.ts @@ -10,15 +10,14 @@ const NON_GUIDE_FILES = new Set(['overview.mdx', 'AGENTS.md', 'CLAUDE.md']); export type Capability = 'read' | 'write' | 'proxy' | 'subscribe' | 'search'; export interface GuideDoc { - docPath: string; // path relative to repo root - slug: string; // filename without .mdx + docPath: string; + slug: string; providerKeyFromFrontmatter?: string; guideType?: string; claimedActions: Set; - sampleLinks: string[]; // all distinct samples links found + sampleLinks: string[]; } -/** Fixed link patterns used in provider guides. */ const ACTION_PATTERNS: Array<{ cap: Capability; re: RegExp }> = [ { cap: 'read', re: /\[Read Actions\]\(\/read-actions\)/ }, { cap: 'write', re: /\[Write Actions\]\(\/write-actions\)/ }, @@ -27,8 +26,7 @@ const ACTION_PATTERNS: Array<{ cap: Capability; re: RegExp }> = [ { cap: 'search', re: /\[Search Actions\]\(\/search-actions\)/ }, ]; -// Match every distinct samples link, allowing both amp.yaml and amp.yml. -const SAMPLE_LINK_RE_G = +const SAMPLE_LINK_RE = /https:\/\/github\.com\/amp-labs\/samples\/blob\/[^/]+\/[^/]+\/amp\.ya?ml/g; export async function scanGuides(): Promise { @@ -46,7 +44,7 @@ export async function scanGuides(): Promise { if (re.test(parsed.content)) claimedActions.add(cap); } const sampleLinks = Array.from( - new Set(parsed.content.match(SAMPLE_LINK_RE_G) ?? []), + new Set(parsed.content.match(SAMPLE_LINK_RE) ?? []), ); guides.push({ docPath: path.relative(process.cwd(), full), @@ -62,18 +60,8 @@ export async function scanGuides(): Promise { return guides; } -/** - * Resolve the catalog provider key a guide documents. - * - * Priority: - * 1. frontmatter `provider` field (case preserved) - * 2. slugOverrides recipe - * 3. case-insensitive match against catalog keys (handles the systemic - * "doc files lowercase, catalog mixed-case" convention split, e.g. - * aweber.mdx -> aWeber) - * 4. filename slug verbatim (will surface as guide-without-catalog-entry - * if not in catalog) - */ +// Resolution priority: frontmatter `provider` > slugOverrides > case-insensitive +// match against catalog keys > filename slug verbatim. export function resolveProviderKey( guide: GuideDoc, slugOverrides: Record, diff --git a/scripts/drift-check/index.ts b/scripts/drift-check/index.ts index 2c72fc63..6876ddee 100644 --- a/scripts/drift-check/index.ts +++ b/scripts/drift-check/index.ts @@ -1,24 +1,4 @@ #!/usr/bin/env tsx -/** - * Drift check v1. - * - * Static comparison of provider guides against the connectors catalog. - * Three checks: - * - undocumented-provider: catalog has a provider with no guide and no - * allowance in recipes.ts - * - doc-overclaims-: guide claims an action the catalog - * (module-aware) does not support - * - broken-sample-link: guide links to a sample manifest that 404s - * - * Modes: - * --mode full scan every provider guide (default) - * --mode per-pr scan only .mdx files passed via --changed - * --mode provider scan a single provider via --provider - * - * Output: - * --out write drift-report.json and drift-report.md - * (default ./drift-report) - */ import { parseArgs } from 'node:util'; import path from 'node:path'; import { @@ -29,14 +9,13 @@ import { type Capability, type CatalogProvider, } from './catalog'; -import { scanGuides, resolveProviderKey, type GuideDoc } from './docs'; +import { scanGuides, resolveProviderKey } from './docs'; import { checkSampleLink } from './samples'; import { recipes } from './recipes'; import { writeReport, type Finding, type Report } from './report'; async function main() { - // Accept both `drift-check --out X` and `pnpm run drift-check -- --out X` - // by stripping a leading `--` separator if present. + // Accept both `drift-check --out X` and `pnpm run drift-check -- --out X`. const args = process.argv.slice(2); if (args[0] === '--') args.shift(); const { values } = parseArgs({ @@ -71,14 +50,13 @@ async function main() { const findings: Finding[] = []; - // Check: undocumented providers. Full mode only; otherwise the per-PR slice - // can't tell what's missing. + // The undocumented-provider check requires a global view, so it only runs in full mode. if (mode === 'full') { const documentedKeys = new Set( - (await scanGuides()).map((g) => resolveProviderKey(g, recipes.slugOverrides, catalogKeys)), + guides.map((g) => resolveProviderKey(g, recipes.slugOverrides, catalogKeys)), ); const today = new Date(); - for (const catalogKey of Object.keys(catalog.data)) { + for (const catalogKey of catalogKeys) { if (documentedKeys.has(catalogKey)) continue; const allow = recipes.undocumentedAllowed.find((a) => a.provider === catalogKey); if (allow) { @@ -101,12 +79,11 @@ async function main() { } } - // Check: doc-overclaims- + collect sample-link checks. const sampleChecks: Promise[] = []; for (const guide of guides) { const providerKey = resolveProviderKey(guide, recipes.slugOverrides, catalogKeys); - const cp = catalog.data[providerKey]; - if (!cp) { + const provider = catalog.data[providerKey]; + if (!provider) { findings.push({ provider: providerKey, severity: 'error', @@ -117,17 +94,17 @@ async function main() { continue; } for (const cap of guide.claimedActions) { - if (!supports(cp, cap as Capability)) { + if (!supports(provider, cap as Capability)) { findings.push( - withModule(cp, { + withModule(provider, { provider: providerKey, severity: 'error', kind: `doc-overclaims-${cap}`, docPath: guide.docPath, docClaim: `Guide includes "[${capitalize(cap)} Actions](/${cap}-actions)".`, catalogTruth: { - providerLevel: capabilityValueAt(cp, cap as Capability, 'provider'), - modulesSupporting: modulesSupporting(cp, cap as Capability), + providerLevel: providerLevelSupport(provider, cap as Capability), + modulesSupporting: modulesSupporting(provider, cap as Capability), }, suggestedFix: `Remove the ${cap} action claim, or update the connector if support was added but not flagged.`, }), @@ -139,7 +116,7 @@ async function main() { checkSampleLink(sampleLink).then((r) => { if (!r.exists) { findings.push( - withModule(cp, { + withModule(provider, { provider: providerKey, severity: 'error', kind: 'broken-sample-link', @@ -179,29 +156,16 @@ function capitalize(s: string): string { return s.charAt(0).toUpperCase() + s.slice(1); } -/** - * Enforce the module-aware schema: if the provider has modules and the - * finding does not specify one, default to "all". Single-module providers - * leave module unset. - */ -function withModule( - provider: CatalogProvider | undefined, - finding: Finding, -): Finding { +// Default module to "all" on findings for multi-module providers; v1 does not attribute claims to a specific module. +function withModule(provider: CatalogProvider, finding: Finding): Finding { if (finding.module !== undefined) return finding; - if (provider && hasModules(provider)) { - return { ...finding, module: 'all' }; - } + if (hasModules(provider)) return { ...finding, module: 'all' }; return finding; } -function capabilityValueAt( - cp: CatalogProvider, - cap: Capability, - _level: 'provider', -): unknown { - if (cap === 'search') return cp.support?.search ?? false; - return cp.support?.[cap] ?? false; +function providerLevelSupport(provider: CatalogProvider, cap: Capability): unknown { + if (cap === 'search') return provider.support?.search ?? false; + return provider.support?.[cap] ?? false; } main().catch((err) => { diff --git a/scripts/drift-check/recipes.ts b/scripts/drift-check/recipes.ts index 0ce7a482..ebbcc7e4 100644 --- a/scripts/drift-check/recipes.ts +++ b/scripts/drift-check/recipes.ts @@ -1,39 +1,28 @@ -// Sparse overrides and allowances for the drift check. -// Only encode exceptions here. Every other case should be conventional. +// Sparse overrides and allowances. Only encode exceptions here; convention covers the rest. export interface UndocumentedAllowance { - provider: string; // exact catalog key + provider: string; reason: string; - until: string; // ISO date; allowance expires after this + until: string; // ISO date; allowance expires after this } export interface Recipes { - // Maps a doc file slug (filename without .mdx) to the catalog provider key - // when they differ. Use only when the conventional `.mdx` lookup - // does not apply. slugOverrides: Record; - - // Catalog provider keys that are intentionally undocumented. Anything not - // listed here that lacks a guide will produce an `undocumented-provider` - // error finding. undocumentedAllowed: UndocumentedAllowance[]; } export const recipes: Recipes = { slugOverrides: { - // The instantly.mdx guide currently documents the instantlyAI catalog - // entry (V2 API). The instantly catalog entry (Legacy V1) is allowed - // undocumented below. + // instantly.mdx documents the V2 catalog entry (instantlyAI); the V1 entry + // is allowed undocumented below. instantly: 'instantlyAI', - // Filename uses hyphens; catalog keys are camelCase without hyphens. 'google-workspace-delegation': 'googleWorkspaceDelegation', 'netsuite-m2m': 'netsuiteM2M', 'salesforce-jwt': 'salesforceJWT', - // jira.mdx is an alias guide that points readers to the Atlassian guide. - // Mapping it to atlassian lets it inherit the same catalog-side checks. + // jira.mdx is an alias guide pointing readers to the Atlassian guide. jira: 'atlassian', - // highlevel.mdx documents the Standard variant; the WhiteLabel variant - // is intentionally undocumented (see undocumentedAllowed below). + // highlevel.mdx covers the Standard variant; the WhiteLabel variant is + // allowed undocumented below. highlevel: 'highLevelStandard', }, @@ -43,64 +32,20 @@ export const recipes: Recipes = { reason: 'Legacy V1 connector; current public guide covers instantlyAI.', until: '2026-09-01', }, - { - provider: 'adyenTest', - reason: 'Internal test variant; not user-facing.', - until: '2027-01-01', - }, - { - provider: 'gladlyQA', - reason: 'QA-only variant of gladly; not user-facing.', - until: '2027-01-01', - }, - { - provider: 'deelSandbox', - reason: 'Sandbox variant; share docs with deel.', - until: '2027-01-01', - }, - { - provider: 'gustoDemo', - reason: 'Demo variant; share docs with gusto.', - until: '2027-01-01', - }, - { - provider: 'paddleSandbox', - reason: 'Sandbox variant.', - until: '2027-01-01', - }, - { - provider: 'payPalSandBox', - reason: 'Sandbox variant.', - until: '2027-01-01', - }, - { - provider: 'procoreSandbox', - reason: 'Sandbox variant.', - until: '2027-01-01', - }, - { - provider: 'quickbooksSandbox', - reason: 'Sandbox variant.', - until: '2027-01-01', - }, - { - provider: 'rampDemo', - reason: 'Demo variant.', - until: '2027-01-01', - }, - { - provider: 'ironcladDemo', - reason: 'Demo variant.', - until: '2027-01-01', - }, - { - provider: 'leverSandbox', - reason: 'Sandbox variant.', - until: '2027-01-01', - }, + { provider: 'adyenTest', reason: 'Internal test variant.', until: '2027-01-01' }, + { provider: 'gladlyQA', reason: 'QA-only variant.', until: '2027-01-01' }, + { provider: 'deelSandbox', reason: 'Sandbox variant.', until: '2027-01-01' }, + { provider: 'gustoDemo', reason: 'Demo variant.', until: '2027-01-01' }, + { provider: 'paddleSandbox', reason: 'Sandbox variant.', until: '2027-01-01' }, + { provider: 'payPalSandBox', reason: 'Sandbox variant.', until: '2027-01-01' }, + { provider: 'procoreSandbox', reason: 'Sandbox variant.', until: '2027-01-01' }, + { provider: 'quickbooksSandbox', reason: 'Sandbox variant.', until: '2027-01-01' }, + { provider: 'rampDemo', reason: 'Demo variant.', until: '2027-01-01' }, + { provider: 'ironcladDemo', reason: 'Demo variant.', until: '2027-01-01' }, + { provider: 'leverSandbox', reason: 'Sandbox variant.', until: '2027-01-01' }, { provider: 'highLevelWhiteLabel', - reason: 'WhiteLabel variant; the highlevel.mdx guide covers the Standard variant.', + reason: 'WhiteLabel variant; highlevel.mdx covers the Standard variant.', until: '2026-09-01', }, ], diff --git a/scripts/drift-check/samples.ts b/scripts/drift-check/samples.ts index 838c57de..861447f6 100644 --- a/scripts/drift-check/samples.ts +++ b/scripts/drift-check/samples.ts @@ -1,8 +1,4 @@ -/** - * Resolve a github.com/...blob/... URL to its raw equivalent, since blob URLs - * return a 200 HTML page even for missing files. The raw URL returns 404 on - * missing files, which is what we want for existence checking. - */ +// blob URLs return a 200 HTML page for missing files; raw URLs return 404. function toRawUrl(blobUrl: string): string { return blobUrl .replace('github.com/', 'raw.githubusercontent.com/') @@ -15,12 +11,7 @@ export interface SampleCheckResult { status?: number; } -/** - * Trusts a 404 immediately (definitive missing). Retries once on network - * errors or non-404 non-2xx responses, since GitHub raw can produce - * transient 5xx / connection blips that would otherwise become false - * positives in CI. - */ +// Trusts 404 immediately; retries once on transient errors (5xx, network blips). export async function checkSampleLink(blobUrl: string): Promise { const rawUrl = toRawUrl(blobUrl); const MAX_ATTEMPTS = 2; From b3c2765eeb942088e5216228a71bd97bbbce82e2 Mon Sep 17 00:00:00 2001 From: Caio Pizzol Date: Sun, 17 May 2026 19:34:06 -0300 Subject: [PATCH 7/8] scripts(drift-check): pin drift-report gitignore to repo root only --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index fd79120e..fadc3506 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,4 @@ node_modules .env -drift-report/ +/drift-report/ From fc9ada9a239aaa54d875062db7ba44a7d5bbd1fa Mon Sep 17 00:00:00 2001 From: Caio Pizzol Date: Sun, 17 May 2026 19:36:42 -0300 Subject: [PATCH 8/8] scripts(drift-check): allowance for docusignDeveloper Full-content audit of the 31 undocumented findings turned up one real alias miss: the docusign.mdx guide explicitly walks through creating a DocuSign Developer Account, so the docusignDeveloper catalog entry is covered there. Same pattern as instantly/instantlyAI. The other audit candidates were false positives from shared docs domains: - adobe -> marketo.mdx (Adobe owns Marketo's docs URLs) - greenhouse -> greenhouseJobBoard.mdx (Harvest API vs Job Board API, different connectors) - payPal -> braintree.mdx (PayPal owns Braintree's docs URLs) microsoftClientCredentials surfaced in microsoft.mdx but only as a generic mention; the existing guide documents the Authorization Code flow against Microsoft Graph, not Client Credentials. Treat as genuinely missing (precedent: salesforce-jwt.mdx exists separately from salesforce.mdx). --- scripts/drift-check/recipes.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/drift-check/recipes.ts b/scripts/drift-check/recipes.ts index ebbcc7e4..0e31d34b 100644 --- a/scripts/drift-check/recipes.ts +++ b/scripts/drift-check/recipes.ts @@ -34,6 +34,11 @@ export const recipes: Recipes = { }, { provider: 'adyenTest', reason: 'Internal test variant.', until: '2027-01-01' }, { provider: 'gladlyQA', reason: 'QA-only variant.', until: '2027-01-01' }, + { + provider: 'docusignDeveloper', + reason: 'Developer-environment variant; setup covered by docusign.mdx.', + until: '2026-09-01', + }, { provider: 'deelSandbox', reason: 'Sandbox variant.', until: '2027-01-01' }, { provider: 'gustoDemo', reason: 'Demo variant.', until: '2027-01-01' }, { provider: 'paddleSandbox', reason: 'Sandbox variant.', until: '2027-01-01' },