diff --git a/README.md b/README.md
index 7ffcced..1225cc6 100644
--- a/README.md
+++ b/README.md
@@ -69,7 +69,7 @@ The normal CLI path is a plugin-managed runtime installed explicitly with the `i
The first managed runtime asset is published for `darwin-arm64`. Other platforms should use an explicit CLI until their runtime asset is published.
-The default setup path prepares a fast starter preload. For broader customer-owned history, use `historical_export`; the plugin should not treat the starter preload as a limit on local data access.
+The default setup path prepares a 30-day recent local preload for all locations and can continue the remaining supported local history in a background job. Agents should use `onboarding_status` to track that job and tell the user when the full supported local history is ready. For explicit separate customer-owned history exports, use `historical_export`; the plugin should not treat the recent preload as a limit on local data access.
The plugin ships one visual design contract at `plugins/density/assets/design.md`. Edit that file when a customer-specific artifact style is needed.
diff --git a/plugins/density/mcp-server/server.mjs b/plugins/density/mcp-server/server.mjs
index 4f89945..66be3c4 100755
--- a/plugins/density/mcp-server/server.mjs
+++ b/plugins/density/mcp-server/server.mjs
@@ -15,6 +15,7 @@ import {
localDataProfile,
localUtilizationQuery,
onboardCustomer,
+ onboardingStatus,
repairFastQuestions,
resolveDataDir,
sensorHealthReport,
@@ -51,24 +52,34 @@ const tools = [
},
additionalProperties: false,
}),
- tool('onboard_customer', 'Prepare a starter local customer dataset with staged setup by default; full preload sync requires explicit fullSync.', {
+ tool('onboard_customer', 'Prepare local Density data. Recommended full sync fetches 30 days for all locations now and can continue deeper supported history in the background.', {
type: 'object',
properties: {
dataDir: { type: 'string' },
orgId: { type: 'string', description: 'Optional organization id to select before syncing.' },
- days: { type: 'number', minimum: 1, maximum: 14, description: 'Starter metrics preload window. Defaults to 14 days; windows over 7 days use hourly metrics.' },
- fullSync: { type: 'boolean', description: 'Run starter preload metrics/occupancy/export phases. Defaults false.' },
+ days: { type: 'number', minimum: 1, maximum: 30, description: 'Recent metrics preload window. Defaults to 30 days; windows over 7 days use hourly metrics.' },
+ fullSync: { type: 'boolean', description: 'Run recent metrics/occupancy/export phases. Defaults false.' },
+ backgroundDeepSync: { type: 'boolean', description: 'After the recent preload completes, start a background job for deeper supported history. Defaults true for the recommended 30-day full sync.' },
+ backgroundDeepSyncDays: { type: 'number', minimum: 1, maximum: 365, description: 'Background deeper-history window. Defaults to 365 days.' },
prewarmQuestions: { type: 'boolean', description: 'After full sync, precompute starter-question answers and chart artifacts when supported. Defaults true.' },
timeoutSeconds: { type: 'number', minimum: 1, maximum: 600, description: 'Per-command timeout for explicit full sync.' },
},
additionalProperties: false,
}),
- tool('historical_export', 'Export a larger customer-owned local history window to Parquet. Separate from the fast 14-day starter preload.', {
+ tool('onboarding_status', 'Check Density onboarding progress, including any background deeper-history sync started by onboard_customer.', {
+ type: 'object',
+ properties: {
+ dataDir: { type: 'string' },
+ },
+ additionalProperties: false,
+ }),
+ tool('historical_export', 'Export a larger customer-owned local history window to Parquet. Separate from the fast 30-day recent onboarding preload.', {
type: 'object',
properties: {
dataDir: { type: 'string' },
orgId: { type: 'string', description: 'Optional organization id to select before exporting.' },
days: { type: 'number', minimum: 1, maximum: 365, description: 'Historical local export window. Defaults to 90 days.' },
+ until: { type: 'string', description: 'Optional end of the historical export window, e.g. now or 30d. Defaults to now.' },
timeoutSeconds: { type: 'number', minimum: 1, maximum: 3600, description: 'Per-command timeout for historical export. Defaults to 600 seconds.' },
},
additionalProperties: false,
@@ -267,6 +278,8 @@ async function callTool(name, args) {
return jsonTool(await authLogin(args));
case 'onboard_customer':
return jsonTool(await onboardCustomer(args));
+ case 'onboarding_status':
+ return jsonTool(await onboardingStatus(args));
case 'historical_export':
return jsonTool(await historicalExport(args));
case 'create_demo_customer':
diff --git a/plugins/density/scripts/density-background-deep-sync.mjs b/plugins/density/scripts/density-background-deep-sync.mjs
new file mode 100644
index 0000000..7b6ed6c
--- /dev/null
+++ b/plugins/density/scripts/density-background-deep-sync.mjs
@@ -0,0 +1,71 @@
+#!/usr/bin/env node
+import { mkdir, readFile, rename, writeFile } from 'node:fs/promises';
+import path from 'node:path';
+import { historicalExport } from './density-core.mjs';
+
+const args = process.argv.slice(2);
+const valueAfter = (flag) => {
+ const index = args.indexOf(flag);
+ return index >= 0 ? args[index + 1] : undefined;
+};
+
+const statusFile = valueAfter('--status-file');
+const dataDir = valueAfter('--data-dir');
+const orgId = valueAfter('--org');
+const days = Number(valueAfter('--days'));
+const recentDays = Number(valueAfter('--recent-days'));
+
+if (!statusFile || !dataDir || !Number.isInteger(days) || !Number.isInteger(recentDays)) {
+ throw new Error('Usage: density-background-deep-sync.mjs --data-dir
--days --recent-days --status-file [--org ]');
+}
+
+const nowIso = () => new Date().toISOString();
+
+const readStatus = async () => {
+ try {
+ return JSON.parse(await readFile(statusFile, 'utf8'));
+ } catch {
+ return {};
+ }
+};
+
+const writeStatus = async (patch) => {
+ await mkdir(path.dirname(statusFile), { recursive: true });
+ const current = await readStatus();
+ const tempFile = `${statusFile}.${process.pid}.${Date.now()}.tmp`;
+ await writeFile(tempFile, `${JSON.stringify({
+ ...current,
+ ...patch,
+ updatedAt: nowIso(),
+ }, null, 2)}\n`);
+ await rename(tempFile, statusFile);
+};
+
+try {
+ await writeStatus({
+ status: 'running',
+ olderHistoryWindow: {
+ since: `${days}d`,
+ until: `${recentDays}d`,
+ },
+ });
+ const result = await historicalExport({
+ dataDir,
+ orgId,
+ days,
+ until: `${recentDays}d`,
+ });
+ await writeStatus({
+ status: result.ok ? 'complete' : 'failed',
+ completedAt: nowIso(),
+ result,
+ });
+ process.exitCode = result.ok ? 0 : 1;
+} catch (error) {
+ await writeStatus({
+ status: 'failed',
+ completedAt: nowIso(),
+ error: error instanceof Error ? error.message : String(error),
+ });
+ process.exitCode = 1;
+}
diff --git a/plugins/density/scripts/density-core.mjs b/plugins/density/scripts/density-core.mjs
index 663bf5e..e2442d1 100644
--- a/plugins/density/scripts/density-core.mjs
+++ b/plugins/density/scripts/density-core.mjs
@@ -1,5 +1,7 @@
import path from 'node:path';
import os from 'node:os';
+import { spawn } from 'node:child_process';
+import { mkdir, readFile, rename, writeFile } from 'node:fs/promises';
import {
checkPluginUpdate,
defaultDataDir,
@@ -18,14 +20,17 @@ import {
which,
} from './density-lib.mjs';
-export const DEFAULT_METRICS_DAYS = 14;
-export const MAX_METRICS_DAYS = 14;
+export const DEFAULT_METRICS_DAYS = 30;
+export const MAX_METRICS_DAYS = 30;
export const MAX_15M_METRICS_DAYS = 7;
export const DEFAULT_HISTORICAL_EXPORT_DAYS = 90;
export const MAX_HISTORICAL_EXPORT_DAYS = 365;
+export const DEFAULT_BACKGROUND_DEEP_SYNC_DAYS = MAX_HISTORICAL_EXPORT_DAYS;
export const resolveDataDir = (value) => value || process.env.DENSITY_CLI_DATA_DIR || defaultDataDir();
const availableBuildingsSupported = (capabilities) => Boolean(capabilities.commands?.availableBuildings || capabilities.availableBuildings);
+const onboardingStateDir = (dataDir) => path.join(dataDir, 'onboarding');
+const deepSyncStatusFile = (dataDir) => path.join(onboardingStateDir(dataDir), 'deep-history-sync.json');
const SOURCE_LAYERS = {
localCustomerData: 'local_customer_data',
@@ -43,6 +48,68 @@ const chartContextCache = new Map();
const oneLine = (value) => String(value ?? '').trim();
const sourceBadgeFor = (sourceLayer) => SOURCE_BADGES[sourceLayer] ?? 'Mixed';
+const nowIso = () => new Date().toISOString();
+
+const readJsonFile = async (file) => {
+ try {
+ return JSON.parse(await readFile(file, 'utf8'));
+ } catch (error) {
+ if (error?.code === 'ENOENT') return undefined;
+ throw error;
+ }
+};
+
+const writeJsonFile = async (file, value) => {
+ await mkdir(path.dirname(file), { recursive: true });
+ const tempFile = `${file}.${process.pid}.${Date.now()}.tmp`;
+ await writeFile(tempFile, `${JSON.stringify(value, null, 2)}\n`);
+ await rename(tempFile, file);
+};
+
+const latestDeepSyncStatus = async (dataDir) => {
+ const statusFile = deepSyncStatusFile(dataDir);
+ const status = await readJsonFile(statusFile);
+ return status ? { ...status, statusFile } : undefined;
+};
+
+const startBackgroundDeepSync = async ({ dataDir, orgId, days, recentDays }) => {
+ const statusFile = deepSyncStatusFile(dataDir);
+ const startedAt = nowIso();
+ const job = {
+ kind: 'density.onboarding.deep-history-sync',
+ jobId: `deep-history-${Date.now()}`,
+ status: 'running',
+ mode: 'historical-export',
+ scope: { type: 'all_locations' },
+ days,
+ recentDays,
+ dataDir,
+ statusFile,
+ startedAt,
+ updatedAt: startedAt,
+ note: 'Background sync uses the Density CLI historical export path, which splits Data Access API observation requests at UTC calendar-month boundaries.',
+ };
+ await writeJsonFile(statusFile, job);
+
+ const script = new URL('./density-background-deep-sync.mjs', import.meta.url).pathname;
+ const child = spawn(process.execPath, [
+ script,
+ '--data-dir', dataDir,
+ '--days', String(days),
+ '--recent-days', String(recentDays),
+ '--status-file', statusFile,
+ ...(orgId ? ['--org', orgId] : []),
+ ], {
+ detached: true,
+ stdio: 'ignore',
+ env: process.env,
+ });
+ child.unref();
+
+ const withPid = { ...job, pid: child.pid };
+ await writeJsonFile(statusFile, withPid);
+ return withPid;
+};
const chartContextKey = (dataDir) => path.resolve(dataDir);
@@ -608,9 +675,15 @@ export async function setup(args = {}) {
},
cli && status?.code === 0 && (!storage.parquetReady || (capabilities.commands?.questionStarter && !storage.fastQuestionsReady)) && {
id: 'onboard_customer',
- label: 'Prepare local Density data.',
+ label: `Fetch ${DEFAULT_METRICS_DAYS} days for all locations, then continue deeper history in the background.`,
tool: 'onboard_customer',
- args: { dataDir, days: DEFAULT_METRICS_DAYS, fullSync: true },
+ args: {
+ dataDir,
+ days: DEFAULT_METRICS_DAYS,
+ fullSync: true,
+ backgroundDeepSync: true,
+ backgroundDeepSyncDays: DEFAULT_BACKGROUND_DEEP_SYNC_DAYS,
+ },
command: `density sync --stream spaces && density sync --stream metrics --all-spaces --since ${DEFAULT_METRICS_DAYS}d --until now --interval ${metricsIntervalForDays(DEFAULT_METRICS_DAYS)} && density export parquet --out ${path.join(dataDir, 'parquet')} --all-orgs`,
},
cli && status?.code === 0 && !availableBuildingsSupported(capabilities) && {
@@ -788,11 +861,11 @@ export async function repairFastQuestions(args = {}) {
storage: await storageReport(dataDir),
nextAction: {
id: 'onboard_customer',
- label: 'Prepare local Density data.',
+ label: `Fetch ${DEFAULT_METRICS_DAYS} days for all locations, then continue deeper history in the background.`,
tool: 'onboard_customer',
- args: { dataDir, days: DEFAULT_METRICS_DAYS, fullSync: true },
+ args: { dataDir, days: DEFAULT_METRICS_DAYS, fullSync: true, backgroundDeepSync: true },
},
- nextSteps: ['Prepare local Density data.'],
+ nextSteps: [`Fetch ${DEFAULT_METRICS_DAYS} days for all locations, then continue deeper history in the background.`],
userVisiblePrimaryActions: 1,
};
}
@@ -818,11 +891,11 @@ export async function repairFastQuestions(args = {}) {
storage,
nextAction: storage.fastQuestionsReady ? undefined : {
id: 'onboard_customer',
- label: 'Prepare local Density data.',
+ label: `Fetch ${DEFAULT_METRICS_DAYS} days for all locations, then continue deeper history in the background.`,
tool: 'onboard_customer',
- args: { dataDir, days: DEFAULT_METRICS_DAYS, fullSync: true },
+ args: { dataDir, days: DEFAULT_METRICS_DAYS, fullSync: true, backgroundDeepSync: true },
},
- nextSteps: storage.fastQuestionsReady ? [] : ['Prepare local Density data.'],
+ nextSteps: storage.fastQuestionsReady ? [] : [`Fetch ${DEFAULT_METRICS_DAYS} days for all locations, then continue deeper history in the background.`],
userVisiblePrimaryActions: storage.fastQuestionsReady ? 0 : 1,
};
}
@@ -845,6 +918,10 @@ export async function onboardCustomer(args = {}) {
const dataDir = resolveDataDir(args.dataDir);
const days = boundedMetricsDays(args.days);
const fullSync = Boolean(args.fullSync);
+ const backgroundDeepSync = Boolean(args.backgroundDeepSync ?? (fullSync && days === DEFAULT_METRICS_DAYS));
+ const backgroundDeepSyncDays = backgroundDeepSync
+ ? boundedHistoricalExportDays(args.backgroundDeepSyncDays ?? DEFAULT_BACKGROUND_DEEP_SYNC_DAYS)
+ : undefined;
const prewarmQuestions = args.prewarmQuestions !== false;
const timeoutSeconds = Number.isFinite(Number(args.timeoutSeconds)) ? Number(args.timeoutSeconds) : 110;
const steps = [];
@@ -908,12 +985,47 @@ export async function onboardCustomer(args = {}) {
storage,
nextAction: {
id: 'run_full_sync',
- label: 'Run explicit full sync when ready.',
+ label: `Fetch ${DEFAULT_METRICS_DAYS} days for all locations, then continue deeper history in the background.`,
tool: 'onboard_customer',
- args: { dataDir, orgId: args.orgId, days, fullSync: true },
+ args: {
+ dataDir,
+ orgId: args.orgId,
+ days: DEFAULT_METRICS_DAYS,
+ fullSync: true,
+ backgroundDeepSync: true,
+ backgroundDeepSyncDays: DEFAULT_BACKGROUND_DEEP_SYNC_DAYS,
+ },
command: `density ${metricsCommand.join(' ')}`,
},
- nextSteps: ['Run explicit full sync when ready.'],
+ onboardingOptions: [
+ {
+ id: 'recommended_recent_plus_background',
+ recommended: true,
+ label: `Fetch ${DEFAULT_METRICS_DAYS} days for all locations now, then background the remaining supported history.`,
+ tool: 'onboard_customer',
+ args: {
+ dataDir,
+ orgId: args.orgId,
+ days: DEFAULT_METRICS_DAYS,
+ fullSync: true,
+ backgroundDeepSync: true,
+ backgroundDeepSyncDays: DEFAULT_BACKGROUND_DEEP_SYNC_DAYS,
+ },
+ },
+ {
+ id: 'recent_only',
+ label: `Fetch ${DEFAULT_METRICS_DAYS} days for all locations and skip the background history job.`,
+ tool: 'onboard_customer',
+ args: { dataDir, orgId: args.orgId, days: DEFAULT_METRICS_DAYS, fullSync: true, backgroundDeepSync: false },
+ },
+ {
+ id: 'specific_location',
+ label: 'Fetch a specific building, floor, or location slice.',
+ unavailable: true,
+ reason: 'Scoped onboarding needs a CLI scope resolver so the plugin can sync descendants without guessing space ids.',
+ },
+ ],
+ nextSteps: [`Fetch ${DEFAULT_METRICS_DAYS} days for all locations, then continue deeper history in the background.`],
userVisiblePrimaryActions: 1,
};
}
@@ -962,11 +1074,29 @@ export async function onboardCustomer(args = {}) {
};
}
}
+ const backgroundJob = backgroundDeepSync && storage.parquetReady && storage.fastQuestionsReady
+ ? await startBackgroundDeepSync({
+ dataDir,
+ orgId: args.orgId,
+ days: backgroundDeepSyncDays,
+ recentDays: days,
+ })
+ : undefined;
+
return {
ok: storage.parquetReady && (!prewarmQuestions || storage.fastQuestionsReady),
- mode: 'full-sync',
+ mode: backgroundJob ? 'recent-plus-background' : 'full-sync',
dataDir,
days,
+ backgroundDeepSync: backgroundJob
+ ? {
+ enabled: true,
+ days: backgroundDeepSyncDays,
+ recentDays: days,
+ status: backgroundJob,
+ pollingTool: 'onboarding_status',
+ }
+ : { enabled: false },
cli: safeCliInfo(cli),
steps,
storage,
@@ -1003,6 +1133,7 @@ export async function historicalExport(args = {}) {
const cli = await requireCli();
const dataDir = resolveDataDir(args.dataDir);
const days = boundedHistoricalExportDays(args.days);
+ const until = args.until === undefined ? 'now' : String(args.until);
const interval = historicalIntervalForDays(days);
const timeoutSeconds = Number.isFinite(Number(args.timeoutSeconds)) ? Number(args.timeoutSeconds) : 600;
const timeoutMs = Math.max(1, timeoutSeconds) * 1000;
@@ -1030,8 +1161,8 @@ export async function historicalExport(args = {}) {
return step;
};
- const metricsCommand = ['sync', '--stream', 'metrics', '--all-spaces', '--since', `${days}d`, '--until', 'now', '--interval', interval];
- const occupancyCommand = ['sync', '--stream', 'occupancy', '--all-spaces', '--since', `${days}d`, '--until', 'now', '--interval', '1h'];
+ const metricsCommand = ['sync', '--stream', 'metrics', '--all-spaces', '--since', `${days}d`, '--until', until, '--interval', interval];
+ const occupancyCommand = ['sync', '--stream', 'occupancy', '--all-spaces', '--since', `${days}d`, '--until', until, '--interval', '1h'];
const exportCommand = ['export', 'parquet', '--out', path.join(dataDir, 'parquet'), '--all-orgs'];
try {
@@ -1048,6 +1179,7 @@ export async function historicalExport(args = {}) {
sourceBadge: sourceBadgeFor(SOURCE_LAYERS.localCustomerData),
dataDir,
days,
+ until,
interval,
cli: safeCliInfo(cli),
steps,
@@ -1063,6 +1195,7 @@ export async function historicalExport(args = {}) {
sourceBadge: sourceBadgeFor(SOURCE_LAYERS.localCustomerData),
dataDir,
days,
+ until,
interval,
cli: safeCliInfo(cli),
steps: error.steps ?? steps,
@@ -1078,6 +1211,31 @@ export async function historicalExport(args = {}) {
}
}
+export async function onboardingStatus(args = {}) {
+ const dataDir = resolveDataDir(args.dataDir);
+ const backgroundDeepSync = await latestDeepSyncStatus(dataDir);
+ return {
+ ok: true,
+ kind: 'density.onboarding-status',
+ sourceLayer: SOURCE_LAYERS.localCustomerData,
+ sourceBadge: sourceBadgeFor(SOURCE_LAYERS.localCustomerData),
+ dataDir,
+ backgroundDeepSync: backgroundDeepSync ?? {
+ status: 'not_started',
+ statusFile: deepSyncStatusFile(dataDir),
+ },
+ nextAction: backgroundDeepSync?.status === 'running'
+ ? {
+ id: 'check_background_deep_sync',
+ label: 'Check the Density background history sync again later.',
+ tool: 'onboarding_status',
+ args: { dataDir },
+ }
+ : undefined,
+ userVisiblePrimaryActions: backgroundDeepSync?.status === 'running' ? 1 : 0,
+ };
+}
+
export async function askChart(args = {}) {
const question = String(args.question || '').trim();
if (!question) throw new Error('question is required.');
@@ -1453,9 +1611,9 @@ export async function localDataProfile(args = {}) {
? undefined
: {
id: 'onboard_customer',
- label: 'Prepare local Density data.',
+ label: `Fetch ${DEFAULT_METRICS_DAYS} days for all locations, then continue deeper history in the background.`,
tool: 'onboard_customer',
- args: { dataDir, days: DEFAULT_METRICS_DAYS, fullSync: true },
+ args: { dataDir, days: DEFAULT_METRICS_DAYS, fullSync: true, backgroundDeepSync: true },
},
userVisiblePrimaryActions: storage.parquetReady && storage.fastQuestionsReady ? 0 : 1,
};
diff --git a/plugins/density/scripts/density-onboard-customer.mjs b/plugins/density/scripts/density-onboard-customer.mjs
index e564c53..7885a98 100755
--- a/plugins/density/scripts/density-onboard-customer.mjs
+++ b/plugins/density/scripts/density-onboard-customer.mjs
@@ -6,6 +6,7 @@ const json = args.includes('--json');
const dataDirFlag = args.find((arg) => arg.startsWith('--data-dir='));
const orgFlag = args.find((arg) => arg.startsWith('--org='));
const daysFlag = args.find((arg) => arg.startsWith('--days='));
+const backgroundDaysFlag = args.find((arg) => arg.startsWith('--background-days='));
const timeoutFlag = args.find((arg) => arg.startsWith('--timeout-seconds='));
const payload = await onboardCustomer({
@@ -13,6 +14,8 @@ const payload = await onboardCustomer({
orgId: orgFlag?.slice('--org='.length),
days: daysFlag ? Number(daysFlag.slice('--days='.length)) : undefined,
fullSync: args.includes('--full-sync'),
+ backgroundDeepSync: args.includes('--no-background-deep-sync') ? false : undefined,
+ backgroundDeepSyncDays: backgroundDaysFlag ? Number(backgroundDaysFlag.slice('--background-days='.length)) : undefined,
timeoutSeconds: timeoutFlag ? Number(timeoutFlag.slice('--timeout-seconds='.length)) : undefined,
});
diff --git a/plugins/density/skills/data-health/SKILL.md b/plugins/density/skills/data-health/SKILL.md
index 4e2c0af..a0ab74c 100644
--- a/plugins/density/skills/data-health/SKILL.md
+++ b/plugins/density/skills/data-health/SKILL.md
@@ -33,6 +33,7 @@ Prefer the plugin MCP tools when available:
- `starter_questions`
- `repair_fast_questions`
- `onboard_customer`
+- `onboarding_status`
- `historical_export`
## Diagnosis Checklist
@@ -53,4 +54,4 @@ Check these before answering an analytical question from local data:
When data is not good enough, say exactly what is missing and what evidence showed that. Then give one primary next action: repair metadata, sync metrics, export Parquet, warm starter questions, or narrow the question.
-If the issue is that the requested window is broader than the starter preload, recommend `historical_export` rather than implying the local-first product is capped at the preload window.
+If the issue is that the requested window is broader than the recent preload, check `onboarding_status` first. If a background deeper-history job is still running, say the local dataset is recent-first and still filling in deeper history. If no job exists, recommend the deeper-history onboarding/export path rather than implying the local-first product is capped at the preload window.
diff --git a/plugins/density/skills/density/SKILL.md b/plugins/density/skills/density/SKILL.md
index 50d6f45..89ce6d7 100644
--- a/plugins/density/skills/density/SKILL.md
+++ b/plugins/density/skills/density/SKILL.md
@@ -72,6 +72,7 @@ Prefer the plugin MCP tools when available:
- `setup`
- `auth_login`
- `onboard_customer`
+- `onboarding_status`
- `historical_export`
- `create_demo_customer`
- `ask_chart`
@@ -103,28 +104,30 @@ A newer version of the Density plugin is available. Say `update @density` and I
Only run the returned update command after the user says yes, `update @density`, `update density`, or an equivalent explicit approval. After updating, ask the user to start a new thread so the latest Density skill and tools load.
-2. If setup says local data is missing, use `onboard_customer` or the fallback script. This is a starter preload for fast first value, not a cap on customer-owned local history. The default path is staged: it may sync cheap metadata, then returns one primary next action for longer starter metrics/export work instead of hiding a long all-spaces sync.
+2. If setup says local data is missing, use `onboard_customer` or the fallback script. Present three onboarding choices: fetch 30 days for all locations now and continue deeper supported history in the background (recommended), fetch 30 days and skip background history, or fetch a specific location slice once the CLI scoped onboarding resolver is available.
```bash
node scripts/density-onboard-customer.mjs --json
```
-Use explicit full sync only when the user is ready for longer local work:
+Use explicit full sync when the user is ready to fetch the recent local dataset:
```bash
-node scripts/density-onboard-customer.mjs --full-sync --days=14 --json
+node scripts/density-onboard-customer.mjs --full-sync --days=30 --json
```
-The default starter metrics preload is 14 days. Windows up to 7 days use 15-minute metrics; longer windows use hourly metrics so two-week utilization questions stay practical locally.
-Explicit full sync prewarms starter-question answers and SVG/HTML chart artifacts when the CLI supports it. Pass `prewarmQuestions: false` only when the user wants raw sync/export without the fast-answer cache.
+The default recent preload is 30 days. Windows up to 7 days use 15-minute metrics; longer windows use hourly metrics so first setup stays practical locally.
+Explicit full sync prewarms starter-question answers and SVG/HTML chart artifacts when the CLI supports it. By default, the recommended 30-day full sync starts a background deeper-history job for the full supported local history window. Pass `backgroundDeepSync: false` or `--no-background-deep-sync` only when the user chooses to skip deeper history. Pass `prewarmQuestions: false` only when the user wants raw sync/export without the fast-answer cache.
-For broader local history, use `historical_export` instead of stretching onboarding:
+Track the background job with `onboarding_status` and tell the user when it completes. The background job uses the CLI historical export path and preserves UTC calendar-month chunking for Data Access API observation requests.
+
+For a separate broader local history export, use `historical_export`:
```text
historical_export
```
-The default historical export window is 90 days and the maximum is 365 days. This is still customer-owned local data; benchmark context and live availability remain separate source layers.
+The default historical export window is 90 days and the maximum supported local history window is 365 days. This is still customer-owned local data; benchmark context and live availability remain separate source layers.
3. If the user needs demo data from an existing local customer dataset, create a fresh Parquet-first local data dir:
@@ -225,7 +228,7 @@ The first-run product loop is:
1. Set up local customer data.
2. Answer one useful historical utilization question locally and fast.
3. Show what Density benchmark-network context or live feed would add when relevant.
-4. Use `historical_export` when the user needs more local history than the starter preload.
+4. Use `onboarding_status` to track background deeper-history syncs, and use `historical_export` when the user needs an explicit separate local history export.
5. Use `data-health` or `sensor-health` when trust in the answer is uncertain.
## Good Local Test Questions
diff --git a/plugins/density/skills/setup/SKILL.md b/plugins/density/skills/setup/SKILL.md
index 2a4b925..cec1127 100644
--- a/plugins/density/skills/setup/SKILL.md
+++ b/plugins/density/skills/setup/SKILL.md
@@ -30,6 +30,7 @@ Prefer the plugin MCP tools when available:
- `install_managed_cli`
- `auth_login`
- `onboard_customer`
+- `onboarding_status`
- `historical_export`
- `create_demo_customer`
- `storage_report`
@@ -45,12 +46,12 @@ Fallback scripts live in the plugin root under `scripts/`.
2. If setup says a plugin update is available, tell the user: "A newer version of the Density plugin is available. Say `update @density` and I can install it." Run the returned update command only after the user says yes, `update @density`, `update density`, or an equivalent explicit approval. After updating, ask the user to start a new thread so the latest Density skill and tools load.
3. If setup asks for the managed CLI runtime, use `install_managed_cli`. This is an explicit download/copy action that verifies the manifest checksum before installing into `~/.density-cli/plugin-runtime/`.
4. If auth is missing, use `auth_login` or tell the user the next step is browser auth.
-5. If Parquet or fast-question inputs are missing, use `onboard_customer` for the starter preload.
+5. If Parquet or fast-question inputs are missing, present the onboarding choices from setup/onboard_customer. Recommend fetching 30 days for all locations now and continuing the remaining supported history in the background.
6. If generic Parquet exists but normalized fast-question metadata is missing and repair is available, use `repair_fast_questions`.
7. Confirm lifecycle readiness is advertised. If setup reports that building lifecycle/go-live readiness is missing, update the CLI before trusting building-level analysis artifacts.
8. Use `available_buildings` when the user asks which buildings are available, live, queryable, mapped, or eligible for wayfinding.
9. Use `storage_report` when the user asks what is local, stale, oversized, or suspicious.
-10. Use `historical_export` when the user wants broader customer-owned local history beyond the starter preload.
+10. Use `onboarding_status` to check a background deeper-history job and tell the user when the full supported local history is ready. Use `historical_export` when the user explicitly asks for a separate broader customer-owned local history export.
Normal setup should not run `npm install` or build the CLI from source. Use `DENSITY_CLI_REPO` plus `DENSITY_CLI_BUILD_FROM_SOURCE=1` only for explicit development work.
@@ -68,10 +69,14 @@ Good local analytics stores include canonical Parquet tables plus normalized fas
Treat `parquetReady` as necessary but not sufficient for utilization. For fast historical questions, also check `fastQuestionsReady` and starter-cache usefulness when available.
-## Sync Defaults
+## Onboarding Choices
-The default starter metrics preload is 14 days. Windows up to 7 days may use 15-minute metrics; longer windows may use hourly metrics to keep two-week answers practical.
+When setup reaches local data preparation, present these choices:
-Prefer staged setup unless the user explicitly wants a longer full sync.
+- Recommended: fetch 30 days for all locations now, then run the remaining supported history in the background.
+- Recent only: fetch 30 days for all locations and skip the background history job.
+- Specific location: fetch a named building, floor, or location slice once the CLI exposes a scoped onboarding resolver.
-For larger local history, use `historical_export`. Do not describe the starter preload limit as a limit on customer access to their own data.
+Windows up to 7 days may use 15-minute metrics; longer windows use hourly metrics to keep setup practical. Background deeper-history sync uses the CLI historical export path, which splits Data Access API observation requests at UTC calendar-month boundaries.
+
+Do not describe the recent preload as a limit on customer access to their own data. Until the background job completes, answers should disclose that local history is recent-first and still filling in deeper history.
diff --git a/plugins/density/test/density-core.test.mjs b/plugins/density/test/density-core.test.mjs
index 13fbbe0..7a9f734 100644
--- a/plugins/density/test/density-core.test.mjs
+++ b/plugins/density/test/density-core.test.mjs
@@ -24,10 +24,12 @@ import {
localUtilizationQuery,
metricsIntervalForDays,
onboardCustomer,
+ onboardingStatus,
repairFastQuestions,
sensorHealthReport,
starterQuestions,
setup,
+ DEFAULT_BACKGROUND_DEEP_SYNC_DAYS,
DEFAULT_METRICS_DAYS,
} from '../scripts/density-core.mjs';
import { checkPluginUpdate, managedCliPlatform, resolveDensityCli, storageReport, which } from '../scripts/density-lib.mjs';
@@ -484,6 +486,16 @@ const readFakeLog = async (file) => {
}
};
+const waitFor = async (predicate, { timeoutMs = 3000, intervalMs = 50 } = {}) => {
+ const startedAt = Date.now();
+ while (Date.now() - startedAt < timeoutMs) {
+ const value = await predicate();
+ if (value) return value;
+ await new Promise((resolve) => setTimeout(resolve, intervalMs));
+ }
+ return undefined;
+};
+
const sha256File = async (file) => createHash('sha256')
.update(await readFile(file))
.digest('hex');
@@ -1347,8 +1359,15 @@ test('default onboarding is staged and does not start sync commands', async () =
assert.equal(result.mode, 'staged');
assert.equal(result.days, DEFAULT_METRICS_DAYS);
assert.equal(result.nextAction.id, 'run_full_sync');
- assert.match(result.nextAction.command, /--since 14d/);
+ assert.equal(result.nextAction.args.days, 30);
+ assert.equal(result.nextAction.args.backgroundDeepSync, true);
+ assert.equal(result.nextAction.args.backgroundDeepSyncDays, DEFAULT_BACKGROUND_DEEP_SYNC_DAYS);
+ assert.match(result.nextAction.command, /--since 30d/);
assert.match(result.nextAction.command, /--interval 1h/);
+ assert.equal(result.onboardingOptions[0].id, 'recommended_recent_plus_background');
+ assert.equal(result.onboardingOptions[0].recommended, true);
+ assert.equal(result.onboardingOptions[1].id, 'recent_only');
+ assert.equal(result.onboardingOptions[2].id, 'specific_location');
assert.equal(calls.some((args) => args[0] === 'sync'), false);
assert.equal(calls.some((args) => args[0] === 'sync' && args.includes('metrics')), false);
});
@@ -1401,6 +1420,70 @@ test('full onboarding uses hourly metrics for two-week windows', async () => {
});
});
+test('recommended full onboarding syncs 30 days and starts background deeper history', async () => {
+ await withTempEnv(async (tempDir) => {
+ const fakeCli = path.join(tempDir, 'density.mjs');
+ const logFile = path.join(tempDir, 'calls.log');
+ const dataDir = path.join(tempDir, 'data');
+ await writeFakeCli(fakeCli);
+ process.env.DENSITY_CLI_BIN = fakeCli;
+ process.env.FAKE_CLI_LOG = logFile;
+
+ const result = await onboardCustomer({
+ dataDir,
+ fullSync: true,
+ backgroundDeepSyncDays: 60,
+ });
+ const foregroundCalls = await readFakeLog(logFile);
+ const metricsCall = foregroundCalls.find((args) => args[0] === 'sync' && args.includes('metrics'));
+
+ assert.equal(result.ok, true);
+ assert.equal(result.mode, 'recent-plus-background');
+ assert.equal(result.days, DEFAULT_METRICS_DAYS);
+ assert.equal(result.backgroundDeepSync.enabled, true);
+ assert.equal(result.backgroundDeepSync.days, 60);
+ assert.equal(result.backgroundDeepSync.recentDays, DEFAULT_METRICS_DAYS);
+ assert.equal(result.backgroundDeepSync.pollingTool, 'onboarding_status');
+ assert.equal(metricsCall[metricsCall.indexOf('--since') + 1], '30d');
+
+ const completed = await waitFor(async () => {
+ const status = await onboardingStatus({ dataDir });
+ return status.backgroundDeepSync.status === 'complete' ? status : undefined;
+ });
+ assert.ok(completed);
+ assert.equal(completed.backgroundDeepSync.result.days, 60);
+ assert.equal(completed.backgroundDeepSync.result.until, '30d');
+
+ const calls = await readFakeLog(logFile);
+ assert.ok(calls.some((args) => args[0] === 'sync' && args.includes('metrics') && args[args.indexOf('--since') + 1] === '60d' && args[args.indexOf('--until') + 1] === '30d'));
+ assert.ok(calls.some((args) => args[0] === 'sync' && args.includes('occupancy') && args[args.indexOf('--since') + 1] === '60d' && args[args.indexOf('--until') + 1] === '30d'));
+ });
+});
+
+test('recent-only onboarding skips the background deeper-history job', async () => {
+ await withTempEnv(async (tempDir) => {
+ const fakeCli = path.join(tempDir, 'density.mjs');
+ const logFile = path.join(tempDir, 'calls.log');
+ const dataDir = path.join(tempDir, 'data');
+ await writeFakeCli(fakeCli);
+ process.env.DENSITY_CLI_BIN = fakeCli;
+ process.env.FAKE_CLI_LOG = logFile;
+
+ const result = await onboardCustomer({
+ dataDir,
+ fullSync: true,
+ backgroundDeepSync: false,
+ });
+ const status = await onboardingStatus({ dataDir });
+
+ assert.equal(result.ok, true);
+ assert.equal(result.mode, 'full-sync');
+ assert.equal(result.days, DEFAULT_METRICS_DAYS);
+ assert.deepEqual(result.backgroundDeepSync, { enabled: false });
+ assert.equal(status.backgroundDeepSync.status, 'not_started');
+ });
+});
+
test('full onboarding prewarms starter questions when supported', async () => {
await withTempEnv(async (tempDir) => {
const fakeCli = path.join(tempDir, 'density.mjs');
@@ -1436,8 +1519,8 @@ test('onboarding rejects invalid metrics window before sync', async () => {
process.env.FAKE_CLI_LOG = logFile;
await assert.rejects(
- onboardCustomer({ dataDir: path.join(tempDir, 'data'), days: 15 }),
- /between 1 and 14/
+ onboardCustomer({ dataDir: path.join(tempDir, 'data'), days: 31 }),
+ /between 1 and 30/
);
assert.deepEqual(await readFakeLog(logFile), []);
});
@@ -1483,6 +1566,7 @@ test('metrics preload interval chooses high resolution only for short windows',
assert.equal(metricsIntervalForDays(7), '15m');
assert.equal(metricsIntervalForDays(8), '1h');
assert.equal(metricsIntervalForDays(14), '1h');
+ assert.equal(metricsIntervalForDays(30), '1h');
});
test('generic demo customer windows remain bounded separately from metrics windows', () => {
diff --git a/plugins/density/test/plugin-packaging.test.mjs b/plugins/density/test/plugin-packaging.test.mjs
index 4e5ff80..ebb56a5 100644
--- a/plugins/density/test/plugin-packaging.test.mjs
+++ b/plugins/density/test/plugin-packaging.test.mjs
@@ -29,6 +29,7 @@ const EXPECTED_MCP_TOOLS = [
'install_managed_cli',
'auth_login',
'onboard_customer',
+ 'onboarding_status',
'historical_export',
'create_demo_customer',
'ask_chart',