diff --git a/README.md b/README.md index 7ffcced..1225cc6 100644 --- a/README.md +++ b/README.md @@ -69,7 +69,7 @@ The normal CLI path is a plugin-managed runtime installed explicitly with the `i The first managed runtime asset is published for `darwin-arm64`. Other platforms should use an explicit CLI until their runtime asset is published. -The default setup path prepares a fast starter preload. For broader customer-owned history, use `historical_export`; the plugin should not treat the starter preload as a limit on local data access. +The default setup path prepares a 30-day recent local preload for all locations and can continue the remaining supported local history in a background job. Agents should use `onboarding_status` to track that job and tell the user when the full supported local history is ready. For explicit separate customer-owned history exports, use `historical_export`; the plugin should not treat the recent preload as a limit on local data access. The plugin ships one visual design contract at `plugins/density/assets/design.md`. Edit that file when a customer-specific artifact style is needed. diff --git a/plugins/density/mcp-server/server.mjs b/plugins/density/mcp-server/server.mjs index 4f89945..66be3c4 100755 --- a/plugins/density/mcp-server/server.mjs +++ b/plugins/density/mcp-server/server.mjs @@ -15,6 +15,7 @@ import { localDataProfile, localUtilizationQuery, onboardCustomer, + onboardingStatus, repairFastQuestions, resolveDataDir, sensorHealthReport, @@ -51,24 +52,34 @@ const tools = [ }, additionalProperties: false, }), - tool('onboard_customer', 'Prepare a starter local customer dataset with staged setup by default; full preload sync requires explicit fullSync.', { + tool('onboard_customer', 'Prepare local Density data. Recommended full sync fetches 30 days for all locations now and can continue deeper supported history in the background.', { type: 'object', properties: { dataDir: { type: 'string' }, orgId: { type: 'string', description: 'Optional organization id to select before syncing.' }, - days: { type: 'number', minimum: 1, maximum: 14, description: 'Starter metrics preload window. Defaults to 14 days; windows over 7 days use hourly metrics.' }, - fullSync: { type: 'boolean', description: 'Run starter preload metrics/occupancy/export phases. Defaults false.' }, + days: { type: 'number', minimum: 1, maximum: 30, description: 'Recent metrics preload window. Defaults to 30 days; windows over 7 days use hourly metrics.' }, + fullSync: { type: 'boolean', description: 'Run recent metrics/occupancy/export phases. Defaults false.' }, + backgroundDeepSync: { type: 'boolean', description: 'After the recent preload completes, start a background job for deeper supported history. Defaults true for the recommended 30-day full sync.' }, + backgroundDeepSyncDays: { type: 'number', minimum: 1, maximum: 365, description: 'Background deeper-history window. Defaults to 365 days.' }, prewarmQuestions: { type: 'boolean', description: 'After full sync, precompute starter-question answers and chart artifacts when supported. Defaults true.' }, timeoutSeconds: { type: 'number', minimum: 1, maximum: 600, description: 'Per-command timeout for explicit full sync.' }, }, additionalProperties: false, }), - tool('historical_export', 'Export a larger customer-owned local history window to Parquet. Separate from the fast 14-day starter preload.', { + tool('onboarding_status', 'Check Density onboarding progress, including any background deeper-history sync started by onboard_customer.', { + type: 'object', + properties: { + dataDir: { type: 'string' }, + }, + additionalProperties: false, + }), + tool('historical_export', 'Export a larger customer-owned local history window to Parquet. Separate from the fast 30-day recent onboarding preload.', { type: 'object', properties: { dataDir: { type: 'string' }, orgId: { type: 'string', description: 'Optional organization id to select before exporting.' }, days: { type: 'number', minimum: 1, maximum: 365, description: 'Historical local export window. Defaults to 90 days.' }, + until: { type: 'string', description: 'Optional end of the historical export window, e.g. now or 30d. Defaults to now.' }, timeoutSeconds: { type: 'number', minimum: 1, maximum: 3600, description: 'Per-command timeout for historical export. Defaults to 600 seconds.' }, }, additionalProperties: false, @@ -267,6 +278,8 @@ async function callTool(name, args) { return jsonTool(await authLogin(args)); case 'onboard_customer': return jsonTool(await onboardCustomer(args)); + case 'onboarding_status': + return jsonTool(await onboardingStatus(args)); case 'historical_export': return jsonTool(await historicalExport(args)); case 'create_demo_customer': diff --git a/plugins/density/scripts/density-background-deep-sync.mjs b/plugins/density/scripts/density-background-deep-sync.mjs new file mode 100644 index 0000000..7b6ed6c --- /dev/null +++ b/plugins/density/scripts/density-background-deep-sync.mjs @@ -0,0 +1,71 @@ +#!/usr/bin/env node +import { mkdir, readFile, rename, writeFile } from 'node:fs/promises'; +import path from 'node:path'; +import { historicalExport } from './density-core.mjs'; + +const args = process.argv.slice(2); +const valueAfter = (flag) => { + const index = args.indexOf(flag); + return index >= 0 ? args[index + 1] : undefined; +}; + +const statusFile = valueAfter('--status-file'); +const dataDir = valueAfter('--data-dir'); +const orgId = valueAfter('--org'); +const days = Number(valueAfter('--days')); +const recentDays = Number(valueAfter('--recent-days')); + +if (!statusFile || !dataDir || !Number.isInteger(days) || !Number.isInteger(recentDays)) { + throw new Error('Usage: density-background-deep-sync.mjs --data-dir --days --recent-days --status-file [--org ]'); +} + +const nowIso = () => new Date().toISOString(); + +const readStatus = async () => { + try { + return JSON.parse(await readFile(statusFile, 'utf8')); + } catch { + return {}; + } +}; + +const writeStatus = async (patch) => { + await mkdir(path.dirname(statusFile), { recursive: true }); + const current = await readStatus(); + const tempFile = `${statusFile}.${process.pid}.${Date.now()}.tmp`; + await writeFile(tempFile, `${JSON.stringify({ + ...current, + ...patch, + updatedAt: nowIso(), + }, null, 2)}\n`); + await rename(tempFile, statusFile); +}; + +try { + await writeStatus({ + status: 'running', + olderHistoryWindow: { + since: `${days}d`, + until: `${recentDays}d`, + }, + }); + const result = await historicalExport({ + dataDir, + orgId, + days, + until: `${recentDays}d`, + }); + await writeStatus({ + status: result.ok ? 'complete' : 'failed', + completedAt: nowIso(), + result, + }); + process.exitCode = result.ok ? 0 : 1; +} catch (error) { + await writeStatus({ + status: 'failed', + completedAt: nowIso(), + error: error instanceof Error ? error.message : String(error), + }); + process.exitCode = 1; +} diff --git a/plugins/density/scripts/density-core.mjs b/plugins/density/scripts/density-core.mjs index 663bf5e..e2442d1 100644 --- a/plugins/density/scripts/density-core.mjs +++ b/plugins/density/scripts/density-core.mjs @@ -1,5 +1,7 @@ import path from 'node:path'; import os from 'node:os'; +import { spawn } from 'node:child_process'; +import { mkdir, readFile, rename, writeFile } from 'node:fs/promises'; import { checkPluginUpdate, defaultDataDir, @@ -18,14 +20,17 @@ import { which, } from './density-lib.mjs'; -export const DEFAULT_METRICS_DAYS = 14; -export const MAX_METRICS_DAYS = 14; +export const DEFAULT_METRICS_DAYS = 30; +export const MAX_METRICS_DAYS = 30; export const MAX_15M_METRICS_DAYS = 7; export const DEFAULT_HISTORICAL_EXPORT_DAYS = 90; export const MAX_HISTORICAL_EXPORT_DAYS = 365; +export const DEFAULT_BACKGROUND_DEEP_SYNC_DAYS = MAX_HISTORICAL_EXPORT_DAYS; export const resolveDataDir = (value) => value || process.env.DENSITY_CLI_DATA_DIR || defaultDataDir(); const availableBuildingsSupported = (capabilities) => Boolean(capabilities.commands?.availableBuildings || capabilities.availableBuildings); +const onboardingStateDir = (dataDir) => path.join(dataDir, 'onboarding'); +const deepSyncStatusFile = (dataDir) => path.join(onboardingStateDir(dataDir), 'deep-history-sync.json'); const SOURCE_LAYERS = { localCustomerData: 'local_customer_data', @@ -43,6 +48,68 @@ const chartContextCache = new Map(); const oneLine = (value) => String(value ?? '').trim(); const sourceBadgeFor = (sourceLayer) => SOURCE_BADGES[sourceLayer] ?? 'Mixed'; +const nowIso = () => new Date().toISOString(); + +const readJsonFile = async (file) => { + try { + return JSON.parse(await readFile(file, 'utf8')); + } catch (error) { + if (error?.code === 'ENOENT') return undefined; + throw error; + } +}; + +const writeJsonFile = async (file, value) => { + await mkdir(path.dirname(file), { recursive: true }); + const tempFile = `${file}.${process.pid}.${Date.now()}.tmp`; + await writeFile(tempFile, `${JSON.stringify(value, null, 2)}\n`); + await rename(tempFile, file); +}; + +const latestDeepSyncStatus = async (dataDir) => { + const statusFile = deepSyncStatusFile(dataDir); + const status = await readJsonFile(statusFile); + return status ? { ...status, statusFile } : undefined; +}; + +const startBackgroundDeepSync = async ({ dataDir, orgId, days, recentDays }) => { + const statusFile = deepSyncStatusFile(dataDir); + const startedAt = nowIso(); + const job = { + kind: 'density.onboarding.deep-history-sync', + jobId: `deep-history-${Date.now()}`, + status: 'running', + mode: 'historical-export', + scope: { type: 'all_locations' }, + days, + recentDays, + dataDir, + statusFile, + startedAt, + updatedAt: startedAt, + note: 'Background sync uses the Density CLI historical export path, which splits Data Access API observation requests at UTC calendar-month boundaries.', + }; + await writeJsonFile(statusFile, job); + + const script = new URL('./density-background-deep-sync.mjs', import.meta.url).pathname; + const child = spawn(process.execPath, [ + script, + '--data-dir', dataDir, + '--days', String(days), + '--recent-days', String(recentDays), + '--status-file', statusFile, + ...(orgId ? ['--org', orgId] : []), + ], { + detached: true, + stdio: 'ignore', + env: process.env, + }); + child.unref(); + + const withPid = { ...job, pid: child.pid }; + await writeJsonFile(statusFile, withPid); + return withPid; +}; const chartContextKey = (dataDir) => path.resolve(dataDir); @@ -608,9 +675,15 @@ export async function setup(args = {}) { }, cli && status?.code === 0 && (!storage.parquetReady || (capabilities.commands?.questionStarter && !storage.fastQuestionsReady)) && { id: 'onboard_customer', - label: 'Prepare local Density data.', + label: `Fetch ${DEFAULT_METRICS_DAYS} days for all locations, then continue deeper history in the background.`, tool: 'onboard_customer', - args: { dataDir, days: DEFAULT_METRICS_DAYS, fullSync: true }, + args: { + dataDir, + days: DEFAULT_METRICS_DAYS, + fullSync: true, + backgroundDeepSync: true, + backgroundDeepSyncDays: DEFAULT_BACKGROUND_DEEP_SYNC_DAYS, + }, command: `density sync --stream spaces && density sync --stream metrics --all-spaces --since ${DEFAULT_METRICS_DAYS}d --until now --interval ${metricsIntervalForDays(DEFAULT_METRICS_DAYS)} && density export parquet --out ${path.join(dataDir, 'parquet')} --all-orgs`, }, cli && status?.code === 0 && !availableBuildingsSupported(capabilities) && { @@ -788,11 +861,11 @@ export async function repairFastQuestions(args = {}) { storage: await storageReport(dataDir), nextAction: { id: 'onboard_customer', - label: 'Prepare local Density data.', + label: `Fetch ${DEFAULT_METRICS_DAYS} days for all locations, then continue deeper history in the background.`, tool: 'onboard_customer', - args: { dataDir, days: DEFAULT_METRICS_DAYS, fullSync: true }, + args: { dataDir, days: DEFAULT_METRICS_DAYS, fullSync: true, backgroundDeepSync: true }, }, - nextSteps: ['Prepare local Density data.'], + nextSteps: [`Fetch ${DEFAULT_METRICS_DAYS} days for all locations, then continue deeper history in the background.`], userVisiblePrimaryActions: 1, }; } @@ -818,11 +891,11 @@ export async function repairFastQuestions(args = {}) { storage, nextAction: storage.fastQuestionsReady ? undefined : { id: 'onboard_customer', - label: 'Prepare local Density data.', + label: `Fetch ${DEFAULT_METRICS_DAYS} days for all locations, then continue deeper history in the background.`, tool: 'onboard_customer', - args: { dataDir, days: DEFAULT_METRICS_DAYS, fullSync: true }, + args: { dataDir, days: DEFAULT_METRICS_DAYS, fullSync: true, backgroundDeepSync: true }, }, - nextSteps: storage.fastQuestionsReady ? [] : ['Prepare local Density data.'], + nextSteps: storage.fastQuestionsReady ? [] : [`Fetch ${DEFAULT_METRICS_DAYS} days for all locations, then continue deeper history in the background.`], userVisiblePrimaryActions: storage.fastQuestionsReady ? 0 : 1, }; } @@ -845,6 +918,10 @@ export async function onboardCustomer(args = {}) { const dataDir = resolveDataDir(args.dataDir); const days = boundedMetricsDays(args.days); const fullSync = Boolean(args.fullSync); + const backgroundDeepSync = Boolean(args.backgroundDeepSync ?? (fullSync && days === DEFAULT_METRICS_DAYS)); + const backgroundDeepSyncDays = backgroundDeepSync + ? boundedHistoricalExportDays(args.backgroundDeepSyncDays ?? DEFAULT_BACKGROUND_DEEP_SYNC_DAYS) + : undefined; const prewarmQuestions = args.prewarmQuestions !== false; const timeoutSeconds = Number.isFinite(Number(args.timeoutSeconds)) ? Number(args.timeoutSeconds) : 110; const steps = []; @@ -908,12 +985,47 @@ export async function onboardCustomer(args = {}) { storage, nextAction: { id: 'run_full_sync', - label: 'Run explicit full sync when ready.', + label: `Fetch ${DEFAULT_METRICS_DAYS} days for all locations, then continue deeper history in the background.`, tool: 'onboard_customer', - args: { dataDir, orgId: args.orgId, days, fullSync: true }, + args: { + dataDir, + orgId: args.orgId, + days: DEFAULT_METRICS_DAYS, + fullSync: true, + backgroundDeepSync: true, + backgroundDeepSyncDays: DEFAULT_BACKGROUND_DEEP_SYNC_DAYS, + }, command: `density ${metricsCommand.join(' ')}`, }, - nextSteps: ['Run explicit full sync when ready.'], + onboardingOptions: [ + { + id: 'recommended_recent_plus_background', + recommended: true, + label: `Fetch ${DEFAULT_METRICS_DAYS} days for all locations now, then background the remaining supported history.`, + tool: 'onboard_customer', + args: { + dataDir, + orgId: args.orgId, + days: DEFAULT_METRICS_DAYS, + fullSync: true, + backgroundDeepSync: true, + backgroundDeepSyncDays: DEFAULT_BACKGROUND_DEEP_SYNC_DAYS, + }, + }, + { + id: 'recent_only', + label: `Fetch ${DEFAULT_METRICS_DAYS} days for all locations and skip the background history job.`, + tool: 'onboard_customer', + args: { dataDir, orgId: args.orgId, days: DEFAULT_METRICS_DAYS, fullSync: true, backgroundDeepSync: false }, + }, + { + id: 'specific_location', + label: 'Fetch a specific building, floor, or location slice.', + unavailable: true, + reason: 'Scoped onboarding needs a CLI scope resolver so the plugin can sync descendants without guessing space ids.', + }, + ], + nextSteps: [`Fetch ${DEFAULT_METRICS_DAYS} days for all locations, then continue deeper history in the background.`], userVisiblePrimaryActions: 1, }; } @@ -962,11 +1074,29 @@ export async function onboardCustomer(args = {}) { }; } } + const backgroundJob = backgroundDeepSync && storage.parquetReady && storage.fastQuestionsReady + ? await startBackgroundDeepSync({ + dataDir, + orgId: args.orgId, + days: backgroundDeepSyncDays, + recentDays: days, + }) + : undefined; + return { ok: storage.parquetReady && (!prewarmQuestions || storage.fastQuestionsReady), - mode: 'full-sync', + mode: backgroundJob ? 'recent-plus-background' : 'full-sync', dataDir, days, + backgroundDeepSync: backgroundJob + ? { + enabled: true, + days: backgroundDeepSyncDays, + recentDays: days, + status: backgroundJob, + pollingTool: 'onboarding_status', + } + : { enabled: false }, cli: safeCliInfo(cli), steps, storage, @@ -1003,6 +1133,7 @@ export async function historicalExport(args = {}) { const cli = await requireCli(); const dataDir = resolveDataDir(args.dataDir); const days = boundedHistoricalExportDays(args.days); + const until = args.until === undefined ? 'now' : String(args.until); const interval = historicalIntervalForDays(days); const timeoutSeconds = Number.isFinite(Number(args.timeoutSeconds)) ? Number(args.timeoutSeconds) : 600; const timeoutMs = Math.max(1, timeoutSeconds) * 1000; @@ -1030,8 +1161,8 @@ export async function historicalExport(args = {}) { return step; }; - const metricsCommand = ['sync', '--stream', 'metrics', '--all-spaces', '--since', `${days}d`, '--until', 'now', '--interval', interval]; - const occupancyCommand = ['sync', '--stream', 'occupancy', '--all-spaces', '--since', `${days}d`, '--until', 'now', '--interval', '1h']; + const metricsCommand = ['sync', '--stream', 'metrics', '--all-spaces', '--since', `${days}d`, '--until', until, '--interval', interval]; + const occupancyCommand = ['sync', '--stream', 'occupancy', '--all-spaces', '--since', `${days}d`, '--until', until, '--interval', '1h']; const exportCommand = ['export', 'parquet', '--out', path.join(dataDir, 'parquet'), '--all-orgs']; try { @@ -1048,6 +1179,7 @@ export async function historicalExport(args = {}) { sourceBadge: sourceBadgeFor(SOURCE_LAYERS.localCustomerData), dataDir, days, + until, interval, cli: safeCliInfo(cli), steps, @@ -1063,6 +1195,7 @@ export async function historicalExport(args = {}) { sourceBadge: sourceBadgeFor(SOURCE_LAYERS.localCustomerData), dataDir, days, + until, interval, cli: safeCliInfo(cli), steps: error.steps ?? steps, @@ -1078,6 +1211,31 @@ export async function historicalExport(args = {}) { } } +export async function onboardingStatus(args = {}) { + const dataDir = resolveDataDir(args.dataDir); + const backgroundDeepSync = await latestDeepSyncStatus(dataDir); + return { + ok: true, + kind: 'density.onboarding-status', + sourceLayer: SOURCE_LAYERS.localCustomerData, + sourceBadge: sourceBadgeFor(SOURCE_LAYERS.localCustomerData), + dataDir, + backgroundDeepSync: backgroundDeepSync ?? { + status: 'not_started', + statusFile: deepSyncStatusFile(dataDir), + }, + nextAction: backgroundDeepSync?.status === 'running' + ? { + id: 'check_background_deep_sync', + label: 'Check the Density background history sync again later.', + tool: 'onboarding_status', + args: { dataDir }, + } + : undefined, + userVisiblePrimaryActions: backgroundDeepSync?.status === 'running' ? 1 : 0, + }; +} + export async function askChart(args = {}) { const question = String(args.question || '').trim(); if (!question) throw new Error('question is required.'); @@ -1453,9 +1611,9 @@ export async function localDataProfile(args = {}) { ? undefined : { id: 'onboard_customer', - label: 'Prepare local Density data.', + label: `Fetch ${DEFAULT_METRICS_DAYS} days for all locations, then continue deeper history in the background.`, tool: 'onboard_customer', - args: { dataDir, days: DEFAULT_METRICS_DAYS, fullSync: true }, + args: { dataDir, days: DEFAULT_METRICS_DAYS, fullSync: true, backgroundDeepSync: true }, }, userVisiblePrimaryActions: storage.parquetReady && storage.fastQuestionsReady ? 0 : 1, }; diff --git a/plugins/density/scripts/density-onboard-customer.mjs b/plugins/density/scripts/density-onboard-customer.mjs index e564c53..7885a98 100755 --- a/plugins/density/scripts/density-onboard-customer.mjs +++ b/plugins/density/scripts/density-onboard-customer.mjs @@ -6,6 +6,7 @@ const json = args.includes('--json'); const dataDirFlag = args.find((arg) => arg.startsWith('--data-dir=')); const orgFlag = args.find((arg) => arg.startsWith('--org=')); const daysFlag = args.find((arg) => arg.startsWith('--days=')); +const backgroundDaysFlag = args.find((arg) => arg.startsWith('--background-days=')); const timeoutFlag = args.find((arg) => arg.startsWith('--timeout-seconds=')); const payload = await onboardCustomer({ @@ -13,6 +14,8 @@ const payload = await onboardCustomer({ orgId: orgFlag?.slice('--org='.length), days: daysFlag ? Number(daysFlag.slice('--days='.length)) : undefined, fullSync: args.includes('--full-sync'), + backgroundDeepSync: args.includes('--no-background-deep-sync') ? false : undefined, + backgroundDeepSyncDays: backgroundDaysFlag ? Number(backgroundDaysFlag.slice('--background-days='.length)) : undefined, timeoutSeconds: timeoutFlag ? Number(timeoutFlag.slice('--timeout-seconds='.length)) : undefined, }); diff --git a/plugins/density/skills/data-health/SKILL.md b/plugins/density/skills/data-health/SKILL.md index 4e2c0af..a0ab74c 100644 --- a/plugins/density/skills/data-health/SKILL.md +++ b/plugins/density/skills/data-health/SKILL.md @@ -33,6 +33,7 @@ Prefer the plugin MCP tools when available: - `starter_questions` - `repair_fast_questions` - `onboard_customer` +- `onboarding_status` - `historical_export` ## Diagnosis Checklist @@ -53,4 +54,4 @@ Check these before answering an analytical question from local data: When data is not good enough, say exactly what is missing and what evidence showed that. Then give one primary next action: repair metadata, sync metrics, export Parquet, warm starter questions, or narrow the question. -If the issue is that the requested window is broader than the starter preload, recommend `historical_export` rather than implying the local-first product is capped at the preload window. +If the issue is that the requested window is broader than the recent preload, check `onboarding_status` first. If a background deeper-history job is still running, say the local dataset is recent-first and still filling in deeper history. If no job exists, recommend the deeper-history onboarding/export path rather than implying the local-first product is capped at the preload window. diff --git a/plugins/density/skills/density/SKILL.md b/plugins/density/skills/density/SKILL.md index 50d6f45..89ce6d7 100644 --- a/plugins/density/skills/density/SKILL.md +++ b/plugins/density/skills/density/SKILL.md @@ -72,6 +72,7 @@ Prefer the plugin MCP tools when available: - `setup` - `auth_login` - `onboard_customer` +- `onboarding_status` - `historical_export` - `create_demo_customer` - `ask_chart` @@ -103,28 +104,30 @@ A newer version of the Density plugin is available. Say `update @density` and I Only run the returned update command after the user says yes, `update @density`, `update density`, or an equivalent explicit approval. After updating, ask the user to start a new thread so the latest Density skill and tools load. -2. If setup says local data is missing, use `onboard_customer` or the fallback script. This is a starter preload for fast first value, not a cap on customer-owned local history. The default path is staged: it may sync cheap metadata, then returns one primary next action for longer starter metrics/export work instead of hiding a long all-spaces sync. +2. If setup says local data is missing, use `onboard_customer` or the fallback script. Present three onboarding choices: fetch 30 days for all locations now and continue deeper supported history in the background (recommended), fetch 30 days and skip background history, or fetch a specific location slice once the CLI scoped onboarding resolver is available. ```bash node scripts/density-onboard-customer.mjs --json ``` -Use explicit full sync only when the user is ready for longer local work: +Use explicit full sync when the user is ready to fetch the recent local dataset: ```bash -node scripts/density-onboard-customer.mjs --full-sync --days=14 --json +node scripts/density-onboard-customer.mjs --full-sync --days=30 --json ``` -The default starter metrics preload is 14 days. Windows up to 7 days use 15-minute metrics; longer windows use hourly metrics so two-week utilization questions stay practical locally. -Explicit full sync prewarms starter-question answers and SVG/HTML chart artifacts when the CLI supports it. Pass `prewarmQuestions: false` only when the user wants raw sync/export without the fast-answer cache. +The default recent preload is 30 days. Windows up to 7 days use 15-minute metrics; longer windows use hourly metrics so first setup stays practical locally. +Explicit full sync prewarms starter-question answers and SVG/HTML chart artifacts when the CLI supports it. By default, the recommended 30-day full sync starts a background deeper-history job for the full supported local history window. Pass `backgroundDeepSync: false` or `--no-background-deep-sync` only when the user chooses to skip deeper history. Pass `prewarmQuestions: false` only when the user wants raw sync/export without the fast-answer cache. -For broader local history, use `historical_export` instead of stretching onboarding: +Track the background job with `onboarding_status` and tell the user when it completes. The background job uses the CLI historical export path and preserves UTC calendar-month chunking for Data Access API observation requests. + +For a separate broader local history export, use `historical_export`: ```text historical_export ``` -The default historical export window is 90 days and the maximum is 365 days. This is still customer-owned local data; benchmark context and live availability remain separate source layers. +The default historical export window is 90 days and the maximum supported local history window is 365 days. This is still customer-owned local data; benchmark context and live availability remain separate source layers. 3. If the user needs demo data from an existing local customer dataset, create a fresh Parquet-first local data dir: @@ -225,7 +228,7 @@ The first-run product loop is: 1. Set up local customer data. 2. Answer one useful historical utilization question locally and fast. 3. Show what Density benchmark-network context or live feed would add when relevant. -4. Use `historical_export` when the user needs more local history than the starter preload. +4. Use `onboarding_status` to track background deeper-history syncs, and use `historical_export` when the user needs an explicit separate local history export. 5. Use `data-health` or `sensor-health` when trust in the answer is uncertain. ## Good Local Test Questions diff --git a/plugins/density/skills/setup/SKILL.md b/plugins/density/skills/setup/SKILL.md index 2a4b925..cec1127 100644 --- a/plugins/density/skills/setup/SKILL.md +++ b/plugins/density/skills/setup/SKILL.md @@ -30,6 +30,7 @@ Prefer the plugin MCP tools when available: - `install_managed_cli` - `auth_login` - `onboard_customer` +- `onboarding_status` - `historical_export` - `create_demo_customer` - `storage_report` @@ -45,12 +46,12 @@ Fallback scripts live in the plugin root under `scripts/`. 2. If setup says a plugin update is available, tell the user: "A newer version of the Density plugin is available. Say `update @density` and I can install it." Run the returned update command only after the user says yes, `update @density`, `update density`, or an equivalent explicit approval. After updating, ask the user to start a new thread so the latest Density skill and tools load. 3. If setup asks for the managed CLI runtime, use `install_managed_cli`. This is an explicit download/copy action that verifies the manifest checksum before installing into `~/.density-cli/plugin-runtime/`. 4. If auth is missing, use `auth_login` or tell the user the next step is browser auth. -5. If Parquet or fast-question inputs are missing, use `onboard_customer` for the starter preload. +5. If Parquet or fast-question inputs are missing, present the onboarding choices from setup/onboard_customer. Recommend fetching 30 days for all locations now and continuing the remaining supported history in the background. 6. If generic Parquet exists but normalized fast-question metadata is missing and repair is available, use `repair_fast_questions`. 7. Confirm lifecycle readiness is advertised. If setup reports that building lifecycle/go-live readiness is missing, update the CLI before trusting building-level analysis artifacts. 8. Use `available_buildings` when the user asks which buildings are available, live, queryable, mapped, or eligible for wayfinding. 9. Use `storage_report` when the user asks what is local, stale, oversized, or suspicious. -10. Use `historical_export` when the user wants broader customer-owned local history beyond the starter preload. +10. Use `onboarding_status` to check a background deeper-history job and tell the user when the full supported local history is ready. Use `historical_export` when the user explicitly asks for a separate broader customer-owned local history export. Normal setup should not run `npm install` or build the CLI from source. Use `DENSITY_CLI_REPO` plus `DENSITY_CLI_BUILD_FROM_SOURCE=1` only for explicit development work. @@ -68,10 +69,14 @@ Good local analytics stores include canonical Parquet tables plus normalized fas Treat `parquetReady` as necessary but not sufficient for utilization. For fast historical questions, also check `fastQuestionsReady` and starter-cache usefulness when available. -## Sync Defaults +## Onboarding Choices -The default starter metrics preload is 14 days. Windows up to 7 days may use 15-minute metrics; longer windows may use hourly metrics to keep two-week answers practical. +When setup reaches local data preparation, present these choices: -Prefer staged setup unless the user explicitly wants a longer full sync. +- Recommended: fetch 30 days for all locations now, then run the remaining supported history in the background. +- Recent only: fetch 30 days for all locations and skip the background history job. +- Specific location: fetch a named building, floor, or location slice once the CLI exposes a scoped onboarding resolver. -For larger local history, use `historical_export`. Do not describe the starter preload limit as a limit on customer access to their own data. +Windows up to 7 days may use 15-minute metrics; longer windows use hourly metrics to keep setup practical. Background deeper-history sync uses the CLI historical export path, which splits Data Access API observation requests at UTC calendar-month boundaries. + +Do not describe the recent preload as a limit on customer access to their own data. Until the background job completes, answers should disclose that local history is recent-first and still filling in deeper history. diff --git a/plugins/density/test/density-core.test.mjs b/plugins/density/test/density-core.test.mjs index 13fbbe0..7a9f734 100644 --- a/plugins/density/test/density-core.test.mjs +++ b/plugins/density/test/density-core.test.mjs @@ -24,10 +24,12 @@ import { localUtilizationQuery, metricsIntervalForDays, onboardCustomer, + onboardingStatus, repairFastQuestions, sensorHealthReport, starterQuestions, setup, + DEFAULT_BACKGROUND_DEEP_SYNC_DAYS, DEFAULT_METRICS_DAYS, } from '../scripts/density-core.mjs'; import { checkPluginUpdate, managedCliPlatform, resolveDensityCli, storageReport, which } from '../scripts/density-lib.mjs'; @@ -484,6 +486,16 @@ const readFakeLog = async (file) => { } }; +const waitFor = async (predicate, { timeoutMs = 3000, intervalMs = 50 } = {}) => { + const startedAt = Date.now(); + while (Date.now() - startedAt < timeoutMs) { + const value = await predicate(); + if (value) return value; + await new Promise((resolve) => setTimeout(resolve, intervalMs)); + } + return undefined; +}; + const sha256File = async (file) => createHash('sha256') .update(await readFile(file)) .digest('hex'); @@ -1347,8 +1359,15 @@ test('default onboarding is staged and does not start sync commands', async () = assert.equal(result.mode, 'staged'); assert.equal(result.days, DEFAULT_METRICS_DAYS); assert.equal(result.nextAction.id, 'run_full_sync'); - assert.match(result.nextAction.command, /--since 14d/); + assert.equal(result.nextAction.args.days, 30); + assert.equal(result.nextAction.args.backgroundDeepSync, true); + assert.equal(result.nextAction.args.backgroundDeepSyncDays, DEFAULT_BACKGROUND_DEEP_SYNC_DAYS); + assert.match(result.nextAction.command, /--since 30d/); assert.match(result.nextAction.command, /--interval 1h/); + assert.equal(result.onboardingOptions[0].id, 'recommended_recent_plus_background'); + assert.equal(result.onboardingOptions[0].recommended, true); + assert.equal(result.onboardingOptions[1].id, 'recent_only'); + assert.equal(result.onboardingOptions[2].id, 'specific_location'); assert.equal(calls.some((args) => args[0] === 'sync'), false); assert.equal(calls.some((args) => args[0] === 'sync' && args.includes('metrics')), false); }); @@ -1401,6 +1420,70 @@ test('full onboarding uses hourly metrics for two-week windows', async () => { }); }); +test('recommended full onboarding syncs 30 days and starts background deeper history', async () => { + await withTempEnv(async (tempDir) => { + const fakeCli = path.join(tempDir, 'density.mjs'); + const logFile = path.join(tempDir, 'calls.log'); + const dataDir = path.join(tempDir, 'data'); + await writeFakeCli(fakeCli); + process.env.DENSITY_CLI_BIN = fakeCli; + process.env.FAKE_CLI_LOG = logFile; + + const result = await onboardCustomer({ + dataDir, + fullSync: true, + backgroundDeepSyncDays: 60, + }); + const foregroundCalls = await readFakeLog(logFile); + const metricsCall = foregroundCalls.find((args) => args[0] === 'sync' && args.includes('metrics')); + + assert.equal(result.ok, true); + assert.equal(result.mode, 'recent-plus-background'); + assert.equal(result.days, DEFAULT_METRICS_DAYS); + assert.equal(result.backgroundDeepSync.enabled, true); + assert.equal(result.backgroundDeepSync.days, 60); + assert.equal(result.backgroundDeepSync.recentDays, DEFAULT_METRICS_DAYS); + assert.equal(result.backgroundDeepSync.pollingTool, 'onboarding_status'); + assert.equal(metricsCall[metricsCall.indexOf('--since') + 1], '30d'); + + const completed = await waitFor(async () => { + const status = await onboardingStatus({ dataDir }); + return status.backgroundDeepSync.status === 'complete' ? status : undefined; + }); + assert.ok(completed); + assert.equal(completed.backgroundDeepSync.result.days, 60); + assert.equal(completed.backgroundDeepSync.result.until, '30d'); + + const calls = await readFakeLog(logFile); + assert.ok(calls.some((args) => args[0] === 'sync' && args.includes('metrics') && args[args.indexOf('--since') + 1] === '60d' && args[args.indexOf('--until') + 1] === '30d')); + assert.ok(calls.some((args) => args[0] === 'sync' && args.includes('occupancy') && args[args.indexOf('--since') + 1] === '60d' && args[args.indexOf('--until') + 1] === '30d')); + }); +}); + +test('recent-only onboarding skips the background deeper-history job', async () => { + await withTempEnv(async (tempDir) => { + const fakeCli = path.join(tempDir, 'density.mjs'); + const logFile = path.join(tempDir, 'calls.log'); + const dataDir = path.join(tempDir, 'data'); + await writeFakeCli(fakeCli); + process.env.DENSITY_CLI_BIN = fakeCli; + process.env.FAKE_CLI_LOG = logFile; + + const result = await onboardCustomer({ + dataDir, + fullSync: true, + backgroundDeepSync: false, + }); + const status = await onboardingStatus({ dataDir }); + + assert.equal(result.ok, true); + assert.equal(result.mode, 'full-sync'); + assert.equal(result.days, DEFAULT_METRICS_DAYS); + assert.deepEqual(result.backgroundDeepSync, { enabled: false }); + assert.equal(status.backgroundDeepSync.status, 'not_started'); + }); +}); + test('full onboarding prewarms starter questions when supported', async () => { await withTempEnv(async (tempDir) => { const fakeCli = path.join(tempDir, 'density.mjs'); @@ -1436,8 +1519,8 @@ test('onboarding rejects invalid metrics window before sync', async () => { process.env.FAKE_CLI_LOG = logFile; await assert.rejects( - onboardCustomer({ dataDir: path.join(tempDir, 'data'), days: 15 }), - /between 1 and 14/ + onboardCustomer({ dataDir: path.join(tempDir, 'data'), days: 31 }), + /between 1 and 30/ ); assert.deepEqual(await readFakeLog(logFile), []); }); @@ -1483,6 +1566,7 @@ test('metrics preload interval chooses high resolution only for short windows', assert.equal(metricsIntervalForDays(7), '15m'); assert.equal(metricsIntervalForDays(8), '1h'); assert.equal(metricsIntervalForDays(14), '1h'); + assert.equal(metricsIntervalForDays(30), '1h'); }); test('generic demo customer windows remain bounded separately from metrics windows', () => { diff --git a/plugins/density/test/plugin-packaging.test.mjs b/plugins/density/test/plugin-packaging.test.mjs index 4e5ff80..ebb56a5 100644 --- a/plugins/density/test/plugin-packaging.test.mjs +++ b/plugins/density/test/plugin-packaging.test.mjs @@ -29,6 +29,7 @@ const EXPECTED_MCP_TOOLS = [ 'install_managed_cli', 'auth_login', 'onboard_customer', + 'onboarding_status', 'historical_export', 'create_demo_customer', 'ask_chart',