From 1bdeb2fdaacb2b125ccd0192c73958089958522e Mon Sep 17 00:00:00 2001 From: Joshua Pozos <25085383+JoshuaPozos@users.noreply.github.com> Date: Sun, 10 May 2026 14:40:56 -0600 Subject: [PATCH 1/8] refactor: promote schema-inference + locale-expander to src/shared/ MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit These two modules were named with a `sanity-` prefix in v0.6.0 because the only caller at the time was `pull-sanity`. Their logic is fully source-agnostic — `inferSchemas` walks Document samples by content type and recognizes both Sanity-shaped (`_type: 'reference'`/`'image'`) and Contentful Link-sys values, and `detectLocales` / `expandDocumentsByLocale` only inspect field-value shape. Promoting them to `src/shared/` (sibling of `src/sanity/`) unblocks `pull-wordpress` reusing them without an awkward import path like `../sanity/sanity-schema-inference.js` from WordPress code. Renames (no behavior change): src/sanity/sanity-schema-inference.ts → src/shared/schema-inference.ts src/sanity/sanity-locale-expander.ts → src/shared/locale-expander.ts + their .test.ts companions Import path updates in `pull-sanity.ts` and `index.ts`. The public type alias `SanityInferenceWarning` (etc.) is removed in favor of the unprefixed names — the `inferSchemas` export is now top-level. 601 tests pass (same count as v0.6.0 — no regressions). 5 examples (`example-from-scratch`, `-sanity-migration`, `-wp-migration`, `-pull-contentful`, `-sanity-pull`) all produce identical counters. --- src/index.ts | 10 ++++++---- src/sanity/pull-sanity.ts | 6 +++--- .../locale-expander.test.ts} | 7 +++++-- .../locale-expander.ts} | 9 +++++++-- .../schema-inference.test.ts} | 11 ++++++++--- .../schema-inference.ts} | 12 +++++++++--- 6 files changed, 38 insertions(+), 17 deletions(-) rename src/{sanity/sanity-locale-expander.test.ts => shared/locale-expander.test.ts} (94%) rename src/{sanity/sanity-locale-expander.ts => shared/locale-expander.ts} (91%) rename src/{sanity/sanity-schema-inference.test.ts => shared/schema-inference.test.ts} (93%) rename src/{sanity/sanity-schema-inference.ts => shared/schema-inference.ts} (95%) diff --git a/src/index.ts b/src/index.ts index b463f2b..22d808c 100644 --- a/src/index.ts +++ b/src/index.ts @@ -110,6 +110,12 @@ export type { WXRReadOptions, WXRSite, WXRResult } from './wordpress/wxr-reader. export { analyzeWXR, generateScaffold, scaffoldFromWXR } from './wordpress/wxr-scaffold.js'; export type { ScaffoldAnalysis, AnalyzedContentType, AnalyzedField, ScaffoldOptions, ScaffoldOutput } from './wordpress/wxr-scaffold.js'; +// ── Shared (source-agnostic pull plumbing) ────────────────────── +export { inferSchemas } from './shared/schema-inference.js'; +export type { InferenceWarning, InferenceResult, InferenceOptions, DocumentSamplesByType } from './shared/schema-inference.js'; +export { detectLocales, expandDocumentsByLocale } from './shared/locale-expander.js'; +export type { LocaleDetectionResult, ExpandOptions as LocaleExpandOptions } from './shared/locale-expander.js'; + // ── Sanity ─────────────────────────────────────────────────────── export { readSanity, parseSanity, parseSanityString, portableTextToHTML, isSanityNDJSON } from './sanity/sanity-reader.js'; export type { SanityReadOptions, SanityImageAsset, SanityResult } from './sanity/sanity-reader.js'; @@ -117,11 +123,7 @@ export { portableTextToRichText, isPortableText } from './sanity/portable-text-t export type { RichTextWarning, RichTextResult } from './sanity/portable-text-to-richtext.js'; export { pullSanity } from './sanity/pull-sanity.js'; export type { PullSanityOptions, PullSanityResult } from './sanity/pull-sanity.js'; -export { inferSchemas as inferSanitySchemas } from './sanity/sanity-schema-inference.js'; -export type { InferenceWarning as SanityInferenceWarning, InferenceResult as SanityInferenceResult, InferenceOptions as SanityInferenceOptions } from './sanity/sanity-schema-inference.js'; export { mapRefsToContentful, mapDocumentRefs } from './sanity/sanity-ref-mapper.js'; export type { RefMapResult } from './sanity/sanity-ref-mapper.js'; export { buildAssetUrlMap, mapImageRefsToLinks, mapDocumentAssets, toAssetMetadata } from './sanity/sanity-asset-mapper.js'; export type { AssetMapResult, PullSanityAssetMeta } from './sanity/sanity-asset-mapper.js'; -export { detectLocales, expandDocumentsByLocale } from './sanity/sanity-locale-expander.js'; -export type { LocaleDetectionResult, ExpandOptions as LocaleExpandOptions } from './sanity/sanity-locale-expander.js'; diff --git a/src/sanity/pull-sanity.ts b/src/sanity/pull-sanity.ts index 5496135..60383e9 100644 --- a/src/sanity/pull-sanity.ts +++ b/src/sanity/pull-sanity.ts @@ -18,11 +18,11 @@ import { join } from 'node:path'; import { parseSanity, parseSanityString } from './sanity-reader.js'; import type { SanityImageAsset } from './sanity-reader.js'; -import { inferSchemas } from './sanity-schema-inference.js'; -import type { InferenceWarning } from './sanity-schema-inference.js'; +import { inferSchemas } from '../shared/schema-inference.js'; +import type { InferenceWarning } from '../shared/schema-inference.js'; import { mapDocumentRefs } from './sanity-ref-mapper.js'; import { mapDocumentAssets, toAssetMetadata } from './sanity-asset-mapper.js'; -import { detectLocales, expandDocumentsByLocale } from './sanity-locale-expander.js'; +import { detectLocales, expandDocumentsByLocale } from '../shared/locale-expander.js'; import type { ContentTypeDefinition, Document } from '../types.js'; // ── Public API ─────────────────────────────────────────────────── diff --git a/src/sanity/sanity-locale-expander.test.ts b/src/shared/locale-expander.test.ts similarity index 94% rename from src/sanity/sanity-locale-expander.test.ts rename to src/shared/locale-expander.test.ts index 7c43a8a..acda2ad 100644 --- a/src/sanity/sanity-locale-expander.test.ts +++ b/src/shared/locale-expander.test.ts @@ -1,5 +1,8 @@ /** - * Tests for src/sanity/sanity-locale-expander.js — v0.6.0 M7. + * Tests for src/shared/locale-expander.js — v0.6.0 M7 (originally + * sanity-only; promoted to shared/ in v0.6.1 alongside the WordPress + * adapter). The detector is source-agnostic — it only inspects field- + * value shape, not the surrounding document's content type. * * Locale detection rules + the expand-by-locale fan-out behaviour. * Covers: positive detection (en, es, pt-BR), negative detection (Link @@ -11,7 +14,7 @@ import { describe, it } from 'node:test'; import assert from 'node:assert/strict'; -import { detectLocales, expandDocumentsByLocale } from './sanity-locale-expander.js'; +import { detectLocales, expandDocumentsByLocale } from './locale-expander.js'; import type { Document } from '../types.js'; function doc(id: string, ct: string, data: Record): Document { diff --git a/src/sanity/sanity-locale-expander.ts b/src/shared/locale-expander.ts similarity index 91% rename from src/sanity/sanity-locale-expander.ts rename to src/shared/locale-expander.ts index d5965de..c2fc2c8 100644 --- a/src/sanity/sanity-locale-expander.ts +++ b/src/shared/locale-expander.ts @@ -1,11 +1,16 @@ /** - * Content Model Simulator — Sanity Locale Detector + Expander (v0.6.0 M7) + * Content Model Simulator — Locale Detector + Expander (v0.6.0 M7 → shared in v0.6.1) * - * Detects locale-shaped values in normalized Sanity documents + * Detects locale-shaped values in normalized source documents * (`{en: '…', es: '…'}` — a plain object whose keys all match a locale * code pattern) and rewrites the corpus so each `Document` carries * exactly one locale tag, mirroring what `cms-sim simulate` expects. * + * Source-agnostic: only inspects the *value* shape, never the + * surrounding document's content type or origin. Works equally well + * for Sanity NDJSON exports and WordPress WXR-derived documents (when + * a plugin like Polylang stores translations on the same field). + * * Without this pass the schema-inference step classifies localized * fields as plain `Object` and the simulator can't merge entries across * locale variants. With it, each Sanity doc fans out to N docs (one per diff --git a/src/sanity/sanity-schema-inference.test.ts b/src/shared/schema-inference.test.ts similarity index 93% rename from src/sanity/sanity-schema-inference.test.ts rename to src/shared/schema-inference.test.ts index 858b654..0929b65 100644 --- a/src/sanity/sanity-schema-inference.test.ts +++ b/src/shared/schema-inference.test.ts @@ -1,5 +1,10 @@ /** - * Tests for src/sanity/sanity-schema-inference.js — v0.6.0 M2. + * Tests for src/shared/schema-inference.js — v0.6.0 M2 (originally Sanity-only; + * promoted to shared/ in v0.6.1 so WordPress and future source adapters can + * reuse it. The detection logic is intentionally generic — it recognizes + * both Sanity-shaped (`_type: 'reference'`, `_type: 'image'`) and + * Contentful-shaped (`{sys: {type: 'Link', linkType: 'Entry'/'Asset'}}`) + * values without caring which source produced them). * * Field-type inference is what differentiates a useful pull from a * placeholder. These tests cover the per-shape mapping (string sizes, @@ -12,8 +17,8 @@ import { describe, it } from 'node:test'; import assert from 'node:assert/strict'; -import { inferSchemas } from './sanity-schema-inference.js'; -import type { DocumentSamplesByType } from './sanity-schema-inference.js'; +import { inferSchemas } from './schema-inference.js'; +import type { DocumentSamplesByType } from './schema-inference.js'; function group(typeId: string, docs: Array>): DocumentSamplesByType { return new Map([[typeId, docs]]); diff --git a/src/sanity/sanity-schema-inference.ts b/src/shared/schema-inference.ts similarity index 95% rename from src/sanity/sanity-schema-inference.ts rename to src/shared/schema-inference.ts index 8996a09..bdf750e 100644 --- a/src/sanity/sanity-schema-inference.ts +++ b/src/shared/schema-inference.ts @@ -1,11 +1,17 @@ /** - * Content Model Simulator — Sanity Schema Inference (v0.6.0 M2) + * Content Model Simulator — Schema Inference (v0.6.0 M2 → shared in v0.6.1) * - * Given a set of documents grouped by Sanity `_type`, infer a + * Given a set of documents grouped by content type, infer a * Contentful-shaped `ContentTypeDefinition` for each: walks each * field across all sampled docs, classifies the value shape, and * maps it to a Contentful field type (Symbol / Text / Integer / - * Number / Boolean / Date / Object / Link / Array<...>). + * Number / Boolean / Date / Object / RichText / Link / Array<...>). + * + * Source-agnostic: detection recognizes both Sanity-shaped values + * (`{_type: 'reference'}`, `{_type: 'image'}`) and Contentful-shaped + * sys-Link values (`{sys: {type: 'Link', linkType: 'Entry'/'Asset'}}`), + * so it works for both `pull-sanity` and `pull-wordpress` after the + * source-specific mappers have rewritten refs/assets. * * Zero-dependency. Read-only over the documents. * From 665fb63e464dd048a2f84c0823015d7a4d840c71 Mon Sep 17 00:00:00 2001 From: Joshua Pozos <25085383+JoshuaPozos@users.noreply.github.com> Date: Sun, 10 May 2026 14:47:43 -0600 Subject: [PATCH 2/8] =?UTF-8?q?feat:=20pull-wordpress=20adapter=20?= =?UTF-8?q?=E2=80=94=20WXR=20XML=20=E2=86=92=20Contentful-shape=20output?= =?UTF-8?q?=20(skeleton)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds `cms-sim pull-wordpress` and the public `pullWordpress()` API. Reads a WordPress WXR XML export (offline, from wp-admin's Tools → Export → All content) and writes the same on-disk shape as `cms-sim pull` for Contentful spaces, so the rest of the pipeline (simulate / to-import / diff) consumes the result unchanged. Output mirrors pull-contentful + pull-sanity exactly: schemas/.js one .js per surviving WP content type data/entries.ndjson one JSON per line contentful-space.json + WP site metadata (title / url / language) Reuses the existing `parseWXR` (already shipped in v0.5.0) plus the shared `inferSchemas` / `detectLocales` / `expandDocumentsByLocale` modules promoted to `src/shared/` in the previous commit. Stable id prefixes per content type so refs (next commit) have predictable targets: - post / page → wp_ e.g. wp_42 - author → wp_author_ e.g. wp_author_jdoe - category → wp_category_ e.g. wp_category_tech - tag → wp_tag_ e.g. wp_tag_javascript Default skip set drops content WordPress emits but Contentful has no equivalent for: nav_menu_item, wp_navigation, wp_template, wp_template_part, wp_global_styles, wp_block, oembed_cache, customize_changeset, custom_css, revision, and (for now) attachment. Attachments will be reshaped into assets/assets.json in commit 4 — `--include-attachments` opts back into emitting them as entries until then. Status: skeleton + output contract. The following land in follow-up commits: - ref rewriting: post.categories/tags (string slugs) → Array, post.author (login string) → Link Entry + linkContentType validations - asset linking: attachment posts → assets.json + featured_image / inline images → Link Asset - Gutenberg / Classic HTML → Contentful RichText (uses existing htmlToRichText from transform/rich-text.ts) - Polylang locale detection + fan-out +10 unit tests (611 total). Pre-commit smoke against the five `file:`-linked examples regression-free (5/25, 6/20, 5/18, 23/69, 7/21). npm audit: 0 vulnerabilities. --- CHANGELOG.md | 9 + src/cli.ts | 160 ++++++++++++++++ src/index.ts | 2 + src/wordpress/pull-wordpress.test.ts | 221 ++++++++++++++++++++++ src/wordpress/pull-wordpress.ts | 264 +++++++++++++++++++++++++++ 5 files changed, 656 insertions(+) create mode 100644 src/wordpress/pull-wordpress.test.ts create mode 100644 src/wordpress/pull-wordpress.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index fa9411d..c64ff68 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,15 @@ All notable changes to `content-model-simulator` are documented here. +## [Unreleased] + +### Added +- **`cms-sim pull-wordpress`** — new CLI subcommand and public `pullWordpress()` API that convert a WordPress WXR XML export (from wp-admin → Tools → Export → All content) into the same on-disk shape as `cms-sim pull` for Contentful spaces: `contentful-space.json` + `schemas/.js` + `data/entries.ndjson`. Reuses `parseWXR` for parsing and the shared `inferSchemas` / `detectLocales` / `expandDocumentsByLocale` from `src/shared/`. Flags: `--input` (required), `--output` (default `./wordpress-export`), `--default-locale`, `--force-locale`, `--include-attachments`, `--skip-types=`, `--source`, `--verbose`. Stable id prefixes per content type: `wp_` for posts/pages, `wp_author_`, `wp_category_`, `wp_tag_` — so cross-document refs (commit 3) have predictable targets. Default skip set drops `nav_menu_item` / `wp_navigation` / `wp_template` / `wp_template_part` / `wp_global_styles` / `wp_block` / `oembed_cache` / `customize_changeset` / `custom_css` / `revision` / `attachment` (the last opts back in via `--include-attachments` and will be reshaped into `assets/assets.json` in a follow-up commit). **Status:** skeleton + output contract — ref → Link Entry rewriting, asset linking, Gutenberg/Classic HTML → RichText, and Polylang locale detection land in follow-up commits. +- **10 unit tests** for `pullWordpress` covering: three pieces written, `contentful-space.json` shape with WP site metadata + `sourceFormat: 'wordpress-wxr'`, `--force-locale` precedence, one schema per surviving content type sorted, Contentful schema shape with `id` / `name` / `fields[]`, prefixed-id assignment across post / page / author / category / tag, default skip set (attachments + nav_menu_item filtered), `--include-attachments` opt-in, in-memory return shape, missing-input error. + +### Changed +- **`inferSchemas` and locale detection promoted to `src/shared/`** (originally named `sanity-schema-inference` / `sanity-locale-expander` in v0.6.0 because Sanity was the only caller). Both modules are source-agnostic — `inferSchemas` recognizes both Sanity-shaped (`{_type: 'reference'}`, `{_type: 'image'}`) and Contentful-shaped (`{sys: Link Entry/Asset}`) values, and the locale detector inspects only field-value shape. Moved to `src/shared/schema-inference.{ts,test.ts}` and `src/shared/locale-expander.{ts,test.ts}`. Public API export `inferSanitySchemas` is replaced by the unprefixed `inferSchemas` (breaking, but pre-1.0.0 minor — see SemVer caveat in README). Imports in `pull-sanity.ts` updated. No runtime/behavior change; 601 tests still pass. + ## [0.6.0] — 2026-05-10 ### Added diff --git a/src/cli.ts b/src/cli.ts index 477b024..8dd4eab 100755 --- a/src/cli.ts +++ b/src/cli.ts @@ -184,6 +184,7 @@ ${c.cyan}COMMANDS:${c.reset} cms-sim from-migrations [opts] ${c.dim}Convert contentful-migration scripts → cms-sim schemas${c.reset} cms-sim pull [options] ${c.dim}Download content model from Contentful${c.reset} cms-sim pull-sanity [options] ${c.dim}Convert a Sanity NDJSON export to the same shape as ${c.bold}pull${c.reset}${c.dim} (offline)${c.reset} + cms-sim pull-wordpress [opts] ${c.dim}Convert a WordPress WXR export to the same shape as ${c.bold}pull${c.reset}${c.dim} (offline)${c.reset} cms-sim diff --old=A --new=B ${c.dim}Compare two schema directories${c.reset} cms-sim validate [options] ${c.dim}Validate schemas + data (no HTML output)${c.reset} @@ -1061,6 +1062,158 @@ async function pullSanityMain(argv: string[]): Promise { console.log(''); } +// ── pull-wordpress sub-command ─────────────────────────────────── + +function showPullWordpressHelp(): void { + console.log(` +${c.bold}Content Model Simulator — Pull WordPress${c.reset} +Convert a WordPress WXR XML export into the same on-disk shape that +${c.bold}cms-sim pull${c.reset} produces (Contentful-flavored). The result drops into +${c.bold}cms-sim simulate${c.reset}, ${c.bold}cms-sim to-import${c.reset}, and ${c.bold}cms-sim diff${c.reset} unchanged. + +${c.cyan}USAGE:${c.reset} + cms-sim pull-wordpress --input= [options] + +${c.cyan}REQUIRED:${c.reset} + --input= Path to the WordPress WXR XML export + (Tools → Export → All content in wp-admin) + +${c.cyan}OPTIONS:${c.reset} + --output= Output directory (default: ./wordpress-export) + --default-locale= Override the language detected from the WXR tag + --force-locale= Force-tag every entry with this locale (ignore WXR) + --include-attachments Emit attachment posts as entries (default: filtered; + commit 4 will reshape them into assets/assets.json) + --skip-types= Additional post_types to skip (merged with defaults: + nav_menu_item, wp_navigation, wp_template, …) + --source=