From 456ef7250b6f79f6864a7277778ffc518b7939f1 Mon Sep 17 00:00:00 2001 From: chitcommit <208086304+chitcommit@users.noreply.github.com> Date: Thu, 11 Jun 2026 01:02:40 +0000 Subject: [PATCH] feat(vendor-charge): ingest ChittyScrape portal charges into the books MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Canonical "ChittyScrape feeds the cost flow" wiring. ChittyScrape (scrape.chitty.cc) extracts vendor charges from portals with no API (registered-agent fees, utilities, mortgage). This adds a real ChittyFinance ingest that turns a ChittyScrape result envelope + resolved charge into an idempotent expense transaction, reusing the PR #133 comptroller pattern. - storage.upsertVendorCharge: idempotent by external_id `scrape:{portalId}:{period}:{vendor}`, type='expense', enforced by the partial unique index transactions_tenant_external_idx (external_id NOT NULL). amount is REQUIRED (> 0) and decimal(12,2); exact USD + paymentStatus + category preserved in metadata. onConflictDoUpdate targetWhere matches the partial arbiter. - POST /api/vendor-charge/ingest: validates the envelope + charge (zod), rejects success=false (422) and a missing/zero amount (400, never defaults to 0), maps vendor category -> real COA code, books against the account by external_id (default chittyos-infra), audit-logs via ledgerLog. - COA mapping (verified against seeded COA on Neon solitary-rice-14149088, no invented codes): registered-agent -> 5050 Legal & Professional Fees; utilities -> 5100/5110/5120/5130/5140; mortgage -> 5300 Mortgage Interest. nw-registered-agent mapping: scraper annualFeeUsd -> charge.amountUsd, paymentStatus ('ok'|'failed') -> charge.paymentStatus, portal 'nw-registered-agent' -> category 'registered-agent' -> COA 5050, period = filing year. Verified end-to-end on real Neon (solitary-rice-14149088): upserted a real Northwest Registered Agent $125/yr charge via the exact storage SQL, selected it back (COA 5050, type expense), re-ran the upsert — same row id, 1 row, no duplicate (idempotency proven). npm run check clean. NOTE: the live portal scrape is credential-gated (NW login via ChittyConnect); the registration + ingest wiring + mapping are real and verified, but the live scrape -> ingest hop is the chico/ChittyConnect follow-up. Co-Authored-By: Claude Opus 4.8 (1M context) --- server/app.ts | 3 + server/routes/vendor-charge.ts | 188 +++++++++++++++++++++++++++++++++ server/storage/system.ts | 72 +++++++++++++ 3 files changed, 263 insertions(+) create mode 100644 server/routes/vendor-charge.ts diff --git a/server/app.ts b/server/app.ts index 5c223d3..75d1283 100644 --- a/server/app.ts +++ b/server/app.ts @@ -40,6 +40,7 @@ import { allocationRoutes } from './accounting/allocations'; import { classificationRoutes } from './routes/classification'; import { emailRoutes } from './routes/email'; import { comptrollerRoutes } from './routes/comptroller'; +import { vendorChargeRoutes } from './routes/vendor-charge'; import { createDb } from './db/connection'; import { SystemStorage } from './storage/system'; @@ -103,6 +104,7 @@ export function createApp() { '/api/integrations', '/api/tasks', '/api/ai-messages', '/api/ai', '/api/summary', '/api/mercury', '/api/github', '/api/charges', '/api/forensics', '/api/portfolio', '/api/import', '/api/reports', '/api/google', '/api/comms', '/api/workflows', '/api/leases', '/api/coa', '/api/classification', '/mcp', + '/api/vendor-charge', ]; app.use('/api/tenants', ...authAndContext); app.use('/api/tenants/*', ...authAndContext); @@ -135,6 +137,7 @@ export function createApp() { app.route('/', classificationRoutes); app.route('/', emailRoutes); app.route('/', comptrollerRoutes); + app.route('/', vendorChargeRoutes); app.route('/', googleRoutes); app.route('/', commsRoutes); app.route('/', workflowRoutes); diff --git a/server/routes/vendor-charge.ts b/server/routes/vendor-charge.ts new file mode 100644 index 0000000..4df1b79 --- /dev/null +++ b/server/routes/vendor-charge.ts @@ -0,0 +1,188 @@ +import { Hono } from 'hono'; +import { z } from 'zod'; +import type { HonoEnv } from '../env'; +import { ledgerLog } from '../lib/ledger-client'; +import { INFRA_ACCOUNT_EXTERNAL_ID } from '../lib/comptroller-sync'; + +// ChittyScrape (scrape.chitty.cc) extracts vendor charges from portals that +// have no API — registered-agent fees, utility bills, mortgage statements. +// This route is the canonical "ChittyScrape feeds the cost flow" wiring: it +// accepts a ChittyScrape result envelope plus the resolved charge amount and +// records it as an idempotent expense transaction in the books. +// +// The live portal scrape itself is credential-gated (portal login via +// ChittyConnect); this ingest does NOT scrape. It is driven by whatever +// dispatches the scrape (ChittyCommand cron) and posts the result here. + +// Vendor category → real chart_of_accounts code. Codes verified against the +// seeded ChittyFinance COA (Neon solitary-rice-14149088). Do NOT invent codes. +// registered-agent : 5050 Legal & Professional Fees +// (Northwest Registered Agent et al. are private vendors selling a +// statutory-agent service — a professional-services fee, not a government +// license, so 5050 rather than 6040 Licenses & Permits.) +// utility-electric : 5100 / utility-gas : 5110 / utility-water : 5120 / +// utility-trash : 5130 / utility-internet : 5140 +// mortgage : 5300 Mortgage Interest (the expense-recognised portion +// of a mortgage charge; principal is a 2500 liability, not an expense). +export const VENDOR_CATEGORY_COA: Record = { + 'registered-agent': '5050', + 'utility-electric': '5100', + 'utility-gas': '5110', + 'utility-water': '5120', + 'utility-trash': '5130', + 'utility-internet': '5140', + utility: '5140', // generic utility fallback (internet/cable) when unspecified + mortgage: '5300', +}; + +// Map a ChittyScrape portalId to a default vendor category when the caller +// does not specify one explicitly. +const PORTAL_DEFAULT_CATEGORY: Record = { + 'nw-registered-agent': 'registered-agent', + 'fl-registered-agent': 'registered-agent', + comed: 'utility-electric', + 'peoples-gas': 'utility-gas', + 'mr-cooper': 'mortgage', +}; + +// ChittyScrape result envelope shape (src/scrapers/base.ts ScrapeResult): +// { success, data?, error?, method:'scrape', portal, scrapedAt } +// We accept that envelope and a small ingest contract carrying the resolved +// charge (amount, vendor, category, period) since the raw scraped `data` shape +// differs per portal and the monetary amount is portal-specific. +const ingestSchema = z.object({ + // The ChittyScrape envelope (required so we honour success/error + portal). + envelope: z.object({ + success: z.boolean(), + portal: z.string().min(1), + scrapedAt: z.string().min(1), + error: z.string().nullish(), + data: z.unknown().optional(), + method: z.literal('scrape').optional(), + }), + // The resolved charge derived from the scrape. + charge: z.object({ + vendor: z.string().min(1), + // amount is REQUIRED and must be > 0 — no defaulting to 0. + amountUsd: z.number().positive(), + period: z.string().min(1), // e.g. '2026' (annual) or '2026-06' (monthly) + category: z.string().optional(), // overrides the portal default + paymentStatus: z.string().optional(), // scraped 'ok' | 'failed' + date: z.string().optional(), // ISO; defaults to envelope.scrapedAt + description: z.string().optional(), + }), + // Account to book against, by external_id. Defaults to the infra account. + accountExternalId: z.string().optional(), +}); + +export const vendorChargeRoutes = new Hono(); + +vendorChargeRoutes.post('/api/vendor-charge/ingest', async (c) => { + const storage = c.get('storage'); + const tenantId = c.get('tenantId'); + + let body: unknown; + try { + body = await c.req.json(); + } catch { + return c.json({ error: 'invalid_json' }, 400); + } + + const parsed = ingestSchema.safeParse(body); + if (!parsed.success) { + return c.json({ error: 'invalid_payload', details: parsed.error.flatten() }, 400); + } + const { envelope, charge } = parsed.data; + + // A failed scrape carries no trustworthy charge — refuse to book it. + if (!envelope.success) { + return c.json( + { error: 'scrape_failed', message: envelope.error ?? 'ChittyScrape reported success=false' }, + 422, + ); + } + + const category = charge.category ?? PORTAL_DEFAULT_CATEGORY[envelope.portal]; + if (!category) { + return c.json( + { + error: 'unknown_category', + message: `No vendor category for portal '${envelope.portal}'. Pass charge.category explicitly.`, + }, + 400, + ); + } + const coaCode = VENDOR_CATEGORY_COA[category]; + if (!coaCode) { + return c.json( + { + error: 'unmapped_category', + message: `No COA mapping for category '${category}'. Known: ${Object.keys(VENDOR_CATEGORY_COA).join(', ')}.`, + }, + 400, + ); + } + + const accountExternalId = parsed.data.accountExternalId ?? INFRA_ACCOUNT_EXTERNAL_ID; + const account = await storage.getAccountByExternalId(accountExternalId, tenantId); + if (!account) { + return c.json( + { + error: 'account_missing', + message: `No account with external_id='${accountExternalId}' for this tenant.`, + }, + 400, + ); + } + + const date = charge.date ? new Date(charge.date) : new Date(envelope.scrapedAt); + if (Number.isNaN(date.getTime())) { + return c.json({ error: 'invalid_date', message: 'charge.date / envelope.scrapedAt is not a valid date' }, 400); + } + + const row = await storage.upsertVendorCharge({ + tenantId, + accountId: account.id, + date, + period: charge.period, + portalId: envelope.portal, + vendor: charge.vendor, + amountUsd: charge.amountUsd, + coaCode, + description: charge.description, + paymentStatus: charge.paymentStatus, + metadata: { category, scrapedAt: envelope.scrapedAt }, + }); + + ledgerLog( + c, + { + entityType: 'audit', + action: 'vendor-charge.ingest', + metadata: { + tenantId, + portal: envelope.portal, + vendor: charge.vendor, + period: charge.period, + category, + coaCode, + amount: row.amount, + externalId: row.externalId, + transactionId: row.id, + }, + }, + c.env, + ); + + return c.json({ + recorded: true, + transaction: { + id: row.id, + externalId: row.externalId, + amount: row.amount, + coaCode: row.coaCode, + type: row.type, + payee: row.payee, + }, + }); +}); diff --git a/server/storage/system.ts b/server/storage/system.ts index f2fa652..1bc6624 100755 --- a/server/storage/system.ts +++ b/server/storage/system.ts @@ -208,6 +208,78 @@ export class SystemStorage { return row; } + // ChittyScrape feeds the cost flow: a vendor charge extracted from a portal + // (registered-agent annual fee, utility bill, mortgage statement) is mirrored + // into the books as an expense transaction. Idempotent by external_id + // `scrape:{portalId}:{period}:{vendor}` + // so re-running a scrape (e.g. the ChittyCommand cron re-dispatching the + // portal) updates the same row rather than duplicating it. + // + // amount is REQUIRED — the scraper output is the source of truth for the fee. + // The route rejects (400) a charge with no amount; we never default to 0, + // because a $0 expense row would be a silent data-integrity lie. + async upsertVendorCharge(input: { + tenantId: string; + accountId: string; + date: Date; + period: string; // billing period the charge belongs to, e.g. '2026' or '2026-06' + portalId: string; // ChittyScrape portal id, e.g. 'nw-registered-agent' + vendor: string; // human vendor name, e.g. 'Northwest Registered Agent' + amountUsd: number; // > 0; the extracted fee + coaCode: string; // real chart_of_accounts code chosen by vendor category + description?: string; + paymentStatus?: string; // scraped 'ok' | 'failed' — recorded, does not gate the expense + metadata?: Record; + }) { + const externalId = `scrape:${input.portalId}:${input.period}:${input.vendor}`; + const amount = input.amountUsd.toFixed(2); + const now = new Date(); + const metadata = { + source: 'chittyscrape', + portalId: input.portalId, + vendor: input.vendor, + period: input.period, + exactAmountUsd: input.amountUsd, + paymentStatus: input.paymentStatus ?? null, + ...input.metadata, + }; + + const [row] = await this.db + .insert(schema.transactions) + .values({ + tenantId: input.tenantId, + accountId: input.accountId, + amount, + currency: 'USD', + type: 'expense', + category: 'vendor_charge', + description: input.description ?? `Vendor charge: ${input.vendor} (${input.period})`, + date: input.date, + payee: input.vendor, + externalId, + coaCode: input.coaCode, + classifiedBy: 'chittyscrape', + classifiedAt: now, + metadata, + }) + .onConflictDoUpdate({ + target: [schema.transactions.tenantId, schema.transactions.externalId], + // Partial arbiter index (WHERE external_id IS NOT NULL) — predicate + // required so Postgres infers the partial unique index. + targetWhere: sql`${schema.transactions.externalId} IS NOT NULL`, + set: { + amount, + coaCode: input.coaCode, + classifiedBy: 'chittyscrape', + classifiedAt: now, + metadata, + updatedAt: now, + }, + }) + .returning(); + return row; + } + async updateTransaction(id: string, tenantId: string, data: Partial) { const [row] = await this.db .update(schema.transactions)