test: Add CI for npm-installed CLI

.github/workflows/boxel-cli-publish.yml

@@ -66,6 +66,11 @@ jobs:
         || (inputs.confirm == '' && github.ref == 'refs/heads/main')
       )
     runs-on: ubuntu-latest
+    outputs:
+      # Whether this run actually published a new version to npm, so the
+      # downstream verify job can skip when the push path no-op'd (no PR,
+      # or no npm-affecting bump) instead of re-testing a stale `unstable`.
+      published: ${{ steps.publish.outputs.published }}
     steps:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
         with:
@@ -277,6 +282,7 @@ jobs:
           '
 
       - name: Publish to npm
+        id: publish
         if: github.event_name == 'workflow_dispatch' || (steps.pr.outputs.skip != 'true' && steps.release.outputs.npmBump != 'none')
         working-directory: packages/boxel-cli
         env:
@@ -313,6 +319,7 @@ jobs:
           # Use --no-git-checks because the workflow's commit + tag is already
           # pushed; pnpm's pre-publish git-state check would otherwise complain.
           pnpm publish --tag unstable --access public --provenance --no-git-checks
+          echo "published=true" >> "$GITHUB_OUTPUT"
 
       - name: Create GitHub Release (prerelease)
         if: github.event_name == 'push' && steps.pr.outputs.skip != 'true' && steps.release.outputs.npmBump != 'none' && steps.commit.outputs.tag != ''
@@ -511,3 +518,48 @@ jobs:
               --title "@cardstack/boxel-cli v${VERSION}" \
               --notes "Promoted from unstable. Published to npm under dist-tag \`latest\`: ${NPM_URL}"
           fi
+
+  # ---------------------------------------------------------------------------
+  # Post-publish verification: install the just-published artifact from the
+  # registry (as an end user would, with npm's hoisted node_modules) and run
+  # the server-less CLI smoke — `boxel parse` over card-code fixtures plus the
+  # help/version/arg-validation checks. This is the exact layout where `boxel
+  # parse`'s glint type-check silently resolves nothing; the in-process tests
+  # never installed the package, so they couldn't catch it.
+  #
+  # Deliberately server-less: the full realm-touching suite already ran
+  # pre-merge against the identical bits (the `pnpm pack` tarball in ci.yaml's
+  # "Boxel CLI Tests"), so standing up Matrix + Postgres again on every publish
+  # would re-test the same behavior. What's genuinely new post-publish is the
+  # registry artifact + packaging + propagation — which parse (workspace mode,
+  # no realm) exercises directly.
+  verify-unstable:
+    name: Verify published unstable install
+    needs: unstable
+    if: needs.unstable.outputs.published == 'true'
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          ref: main
+      - uses: ./.github/actions/init
+      - name: Smoke-test the published `unstable` artifact from npm
+        working-directory: packages/boxel-cli
+        run: NODE_NO_WARNINGS=1 node scripts/run-cli-suite.ts --source published --version unstable -- pnpm run test:cli:smoke
+
+  verify-stable:
+    name: Verify published stable install
+    needs: stable
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          ref: main
+      - uses: ./.github/actions/init
+      - name: Smoke-test the published `latest` artifact from npm
+        working-directory: packages/boxel-cli
+        run: NODE_NO_WARNINGS=1 node scripts/run-cli-suite.ts --source published --version latest -- pnpm run test:cli:smoke

.github/workflows/ci.yaml

@@ -980,8 +980,17 @@ jobs:
           # would race the tests ahead of the base realm finishing its
           # initial index.
           timeout 600 bash -c 'until curl -sk -o /dev/null -w "%{http_code}" https://localhost:4200/ | grep -qx 200 && curl -sk -o /dev/null -w "%{http_code}" -H "Accept: application/vnd.api+json" https://localhost:4201/base/_readiness-check | grep -qx 200; do sleep 2; done'
-      - name: Run integration tests
-        run: pnpm test:integration
+      # Run the integration suite against the CLI as an end user installs
+      # it: `pnpm pack` the build from the step above, `npm install` the
+      # tarball into a throwaway dir outside the monorepo (npm hoisting,
+      # not pnpm's nested layout), and drive that binary via BOXEL_CLI_BIN.
+      # This is the layout where `boxel parse`'s glint type-check silently
+      # resolves nothing — which the in-process, function-call tests could
+      # never exercise because they import command functions from src. The
+      # `test:cli:tarball` script execs `pnpm test:integration` under the
+      # installed binary, so the test-PG provisioning is unchanged.
+      - name: Run integration tests against the packed npm install
+        run: pnpm test:cli:tarball
         working-directory: packages/boxel-cli
       - name: Print server logs
         if: ${{ !cancelled() }}

packages/boxel-cli/.eslintignore

@@ -6,3 +6,10 @@ bundled-test-harness/
 bundled-realms/
 *.tgz
 *.log
+
+# Card-code fixtures for `boxel parse`. These are intentionally card
+# authoring code (inline <template>, field decorators) and one file with
+# a deliberate type error — meant to be type-checked by glint via the
+# CLI, not linted as CLI source. eslint's TS parser chokes on the
+# template tags and the repo's no-decorators rule fires on `@field`.
+tests/fixtures/parse/

packages/boxel-cli/package.json

@@ -94,6 +94,8 @@
     "test:unit": "vitest run --exclude 'tests/integration/**'",
     "test:unit-exclude-smoke": "vitest run --exclude 'tests/integration/**' --exclude 'tests/smoke.test.ts'",
     "test:integration": "./tests/scripts/run-integration-with-test-pg.sh",
+    "test:cli:tarball": "NODE_NO_WARNINGS=1 node scripts/run-cli-suite.ts --source tarball -- pnpm test:integration",
+    "test:cli:smoke": "vitest run tests/integration/parse.test.ts tests/smoke.test.ts",
     "test:watch": "vitest",
     "version:patch": "npm version patch",
     "version:minor": "npm version minor",

packages/boxel-cli/scripts/run-cli-suite.ts

@@ -0,0 +1,197 @@
+/**
+ * Run the boxel-cli test suite against the CLI *as an end user installs
+ * it* — not against `src/`. This is the piece the in-process,
+ * function-call tests could never cover: a `boxel parse` failure that
+ * only surfaces under npm's hoisted `node_modules` layout can ship
+ * despite a fully green suite.
+ *
+ * It installs the CLI into a throwaway directory *outside* the monorepo,
+ * points `BOXEL_CLI_BIN` at the installed JS entry, and execs a test
+ * command that inherits that env. `tests/helpers/run-boxel.ts` reads
+ * `BOXEL_CLI_BIN`, so every `runBoxel(...)` call in the suite drives the
+ * installed binary. The suite itself is identical across contexts — only
+ * the thing executed changes.
+ *
+ *   node scripts/run-cli-suite.ts --source tarball -- <test command…>
+ *   node scripts/run-cli-suite.ts --source published --version 0.5.0-unstable.4 -- <test command…>
+ *
+ * Sources:
+ *   --source tarball    `pnpm pack` the current build, `npm install` the
+ *                        tarball. pnpm (not npm) packs because it rewrites
+ *                        `catalog:` / `workspace:*` specifiers to real
+ *                        versions exactly as `pnpm publish` would; npm
+ *                        would leave them literal and the install would
+ *                        fail. The tarball's own deps then install under
+ *                        npm's hoisting — the layout that breaks parse.
+ *                        Requires a prior `pnpm build` so dist/ and
+ *                        bundled-* exist to pack.
+ *   --source published  `npm install @cardstack/boxel-cli@<version>` from
+ *                        the registry, polling for propagation. Verifies
+ *                        the actual shipped artifact post-release.
+ *
+ * Everything after `--` is the test command run with `BOXEL_CLI_BIN` set
+ * (e.g. `pnpm test:integration`, or `vitest run tests/integration/parse.test.ts`).
+ */
+import { execFileSync, spawnSync } from 'node:child_process';
+import {
+  existsSync,
+  mkdtempSync,
+  readdirSync,
+  rmSync,
+  writeFileSync,
+} from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join, resolve } from 'node:path';
+
+const PKG_ROOT = resolve(import.meta.dirname, '..');
+const PKG_NAME = '@cardstack/boxel-cli';
+
+interface Args {
+  source: 'tarball' | 'published';
+  version: string;
+  testCommand: string[];
+}
+
+function parseArgs(argv: string[]): Args {
+  let source: string | undefined;
+  let version = 'latest';
+  let testCommand: string[] = [];
+  for (let i = 0; i < argv.length; i++) {
+    let arg = argv[i];
+    if (arg === '--') {
+      testCommand = argv.slice(i + 1);
+      break;
+    } else if (arg === '--source') {
+      source = argv[++i];
+    } else if (arg === '--version') {
+      version = argv[++i];
+    } else {
+      throw new Error(`Unknown argument: ${arg}`);
+    }
+  }
+  if (source !== 'tarball' && source !== 'published') {
+    throw new Error(
+      `--source must be 'tarball' or 'published' (got ${source})`,
+    );
+  }
+  if (testCommand.length === 0) {
+    throw new Error('No test command given. Pass it after `--`.');
+  }
+  return { source, version, testCommand };
+}
+
+/**
+ * `pnpm pack` the current package into `destDir` and return the tarball
+ * path. pnpm resolves `catalog:` / `workspace:*` specifiers in the
+ * packed package.json, matching what `pnpm publish` ships.
+ */
+function packTarball(destDir: string): string {
+  execFileSync('pnpm', ['pack', '--pack-destination', destDir], {
+    cwd: PKG_ROOT,
+    stdio: 'inherit',
+  });
+  let tgz = readdirSync(destDir).find((f) => f.endsWith('.tgz'));
+  if (!tgz) {
+    throw new Error(`pnpm pack produced no .tgz in ${destDir}`);
+  }
+  return join(destDir, tgz);
+}
+
+/**
+ * Poll `npm view` until `version` (a concrete version or a dist-tag) is
+ * resolvable, absorbing post-publish registry propagation delay.
+ */
+function waitForPublishedVersion(version: string): void {
+  let deadline = Date.now() + 180_000;
+  let attempt = 0;
+  for (;;) {
+    let result = spawnSync(
+      'npm',
+      ['view', `${PKG_NAME}@${version}`, 'version'],
+      { encoding: 'utf8' },
+    );
+    if (result.status === 0 && result.stdout.trim()) {
+      console.log(`Resolved ${PKG_NAME}@${version} → ${result.stdout.trim()}`);
+      return;
+    }
+    if (Date.now() > deadline) {
+      throw new Error(
+        `${PKG_NAME}@${version} not resolvable after 180s. Last npm error:\n${result.stderr}`,
+      );
+    }
+    let delay = Math.min(15_000, 2_000 * ++attempt);
+    console.log(
+      `Waiting for ${PKG_NAME}@${version} to propagate (${attempt})…`,
+    );
+    execFileSync('sleep', [String(delay / 1000)]);
+  }
+}
+
+/**
+ * Create a clean install dir outside the monorepo and `npm install` the
+ * given spec (a tarball path or a registry spec). npm — not pnpm — so
+ * the CLI's dependencies land in the hoisted layout a real user gets.
+ */
+function npmInstall(spec: string): { installDir: string; entry: string } {
+  let installDir = mkdtempSync(join(tmpdir(), 'boxel-cli-suite-'));
+  // A minimal package.json so `npm install` treats this as a project
+  // root and hoists deps into `installDir/node_modules`.
+  writeFileSync(
+    join(installDir, 'package.json'),
+    JSON.stringify(
+      { name: 'boxel-cli-suite-host', version: '0.0.0', private: true },
+      null,
+      2,
+    ) + '\n',
+  );
+  execFileSync(
+    'npm',
+    ['install', spec, '--no-audit', '--no-fund', '--loglevel', 'error'],
+    { cwd: installDir, stdio: 'inherit' },
+  );
+  let entry = join(installDir, 'node_modules', PKG_NAME, 'dist', 'index.js');
+  if (!existsSync(entry)) {
+    throw new Error(`Installed CLI entry not found at ${entry}`);
+  }
+  return { installDir, entry };
+}
+
+function main(): void {
+  let { source, version, testCommand } = parseArgs(process.argv.slice(2));
+
+  let workDir = mkdtempSync(join(tmpdir(), 'boxel-cli-pack-'));
+  let cleanupDirs = [workDir];
+  try {
+    let spec: string;
+    if (source === 'tarball') {
+      spec = packTarball(workDir);
+    } else {
+      waitForPublishedVersion(version);
+      spec = `${PKG_NAME}@${version}`;
+    }
+
+    let { installDir, entry } = npmInstall(spec);
+    cleanupDirs.push(installDir);
+    console.log(
+      `\nBOXEL_CLI_BIN=${entry}\nRunning: ${testCommand.join(' ')}\n`,
+    );
+
+    let [cmd, ...cmdArgs] = testCommand;
+    let result = spawnSync(cmd, cmdArgs, {
+      cwd: PKG_ROOT,
+      stdio: 'inherit',
+      env: { ...process.env, BOXEL_CLI_BIN: entry },
+    });
+    process.exitCode = result.status ?? 1;
+  } finally {
+    for (let dir of cleanupDirs) {
+      try {
+        rmSync(dir, { recursive: true, force: true });
+      } catch {
+        // best-effort cleanup
+      }
+    }
+  }
+}
+
+main();