From 4acc694e153b4a534d65761e5b277a327d0bbddf Mon Sep 17 00:00:00 2001 From: fkakatie Date: Wed, 8 Jul 2026 16:51:28 -0600 Subject: [PATCH] feat: remove import scripts --- tools/import/.gitignore | 6 - tools/import/README.md | 92 -- tools/import/analyze-blocks.js | 194 ---- tools/import/card-list-manifest.tsv | 184 ---- tools/import/fix-http-images.js | 41 - tools/import/list-endpoints.js | 82 -- tools/import/merge-pr-redirects.js | 41 - tools/import/reconcile.js | 107 --- tools/import/rewrite-links.js | 94 -- tools/import/run-import.sh | 37 - tools/import/transform-page.js | 1040 ---------------------- tools/import/transform-press-releases.js | 229 ----- 12 files changed, 2147 deletions(-) delete mode 100644 tools/import/.gitignore delete mode 100644 tools/import/README.md delete mode 100644 tools/import/analyze-blocks.js delete mode 100644 tools/import/card-list-manifest.tsv delete mode 100644 tools/import/fix-http-images.js delete mode 100644 tools/import/list-endpoints.js delete mode 100644 tools/import/merge-pr-redirects.js delete mode 100644 tools/import/reconcile.js delete mode 100644 tools/import/rewrite-links.js delete mode 100755 tools/import/run-import.sh delete mode 100644 tools/import/transform-page.js delete mode 100644 tools/import/transform-press-releases.js diff --git a/tools/import/.gitignore b/tools/import/.gitignore deleted file mode 100644 index f929a2c..0000000 --- a/tools/import/.gitignore +++ /dev/null @@ -1,6 +0,0 @@ -# generated by the import tooling — not source -protected-files.json -reconcile-report.csv -links-not-in-redirects.tsv -.pr-map-en.csv -.pr-map-fr.csv diff --git a/tools/import/README.md b/tools/import/README.md deleted file mode 100644 index 5d8b985..0000000 --- a/tools/import/README.md +++ /dev/null @@ -1,92 +0,0 @@ -# Progress Rail import pipeline - -Scripts that convert the legacy AEM site (a static crawl) into Edge Delivery -(EDS) document-authoring HTML in the content repo. All scripts are plain Node -and require `cheerio` (`npm install --save-dev cheerio`). - -## Order of operations - -### 0. Clean the crawl (run on the downloaded site, not in this repo) - -These two live with the downloaded site; they produce the `cleaned/` tree the -transformer reads: - -1. `strip-nav-footer.js ` — remove the repeated global nav - (`header.mega--nav`) and footer from every page. -2. `fix-image-src.js ` — swap lazy-load `data-src`/`data-srcset` to - real `src`/`srcset` and upgrade those image URLs to `https`. - -### 1. Transform pages → EDS blocks - -``` -node transform-page.js -``` - -- `` cleaned site root (contains `en.html`, `fr.html`, `en/`, `fr/`) -- `` the EDS content repo to write into - -Converts all block-based pages (homepages → `/index.html`), skips -individual press releases, and writes/merges old→new redirects into -`/redirects.json`. This is the main file; it also exports -`{ transformFile, kebabPath, runBatch }`. - -### 2. Transform press releases - -``` -node transform-press-releases.js [--map mapping.csv] -``` - -Press releases use a simpler title/date/body format and `YYYY-slug` filenames. - -### 3. Rewrite links - -``` -node rewrite-links.js -``` - -Makes `www.progressrail.com` links host-relative and swaps them to their -redirect `Destination`. Paths with no redirect are written to -`.aem/links-not-in-redirects.tsv` for review. - -### 4. Fix any remaining http image sources - -``` -node fix-http-images.js -``` - -Upgrades `src`/`srcset` from `http://` to `https://` (e.g. hand-edited pages -the transformer didn't generate). - -## Analysis / utilities - -- `analyze-blocks.js [--out report.md]` — reports which AEM - components map to which EDS block, and flags anything still unmapped. -- `list-endpoints.js [--curl ]` — builds the - `getDegSubList` endpoint URL for every paginated list (the static crawl only - contains the first page of items). See `.aem/list-pagination-audit.csv`. - -## Preserving human edits made in DA - -Once pages are published, authors may edit them in DA. Re-running the transform -would overwrite those edits, so the flow becomes: - -1. Pull the current DA content into `.aem/` (already present here). -2. `node reconcile.js --apply` — compares the transform output - against `.aem/`, ignoring DA's whitespace reformatting, and classifies each - page as `identical`, `transform` (structure-only change — safe to take), - `author-edit` (text changed — a human edited it) or `conflict` (both - changed). Author-edit + conflict pages are written to - `.aem/protected-files.json` and (with `--apply`) copied back into the working - tree. See `.aem/reconcile-report.csv`. -3. `node transform-page.js --skip-protected` — runs - the transform but never overwrites anything in `protected-files.json`. - -The protected list accumulates across runs, so protection sticks. `conflict` -pages (a human edit AND a transform change) keep the human version and should be -reconciled by hand. - -## Known follow-ups - -- Paginated lists: only the first page of items is in the crawl; fetch the full - set via the endpoints from `list-endpoints.js`. -- `deg-title` headings and a few minor components are not yet mapped. diff --git a/tools/import/analyze-blocks.js b/tools/import/analyze-blocks.js deleted file mode 100644 index f4c22ce..0000000 --- a/tools/import/analyze-blocks.js +++ /dev/null @@ -1,194 +0,0 @@ -#!/usr/bin/env node -/** - * analyze-blocks.js - * - * Scans the cleaned Progress Rail HTML pages, detects the AEM authoring - * components on each page, and maps them to the target Edge Delivery (EDS) - * blocks that the homepage transformation established. - * - * Purpose: discover which blocks the rest of the site needs before we build a - * full transformer. It reports, per page and globally: - * - which known blocks each page uses, - * - which component signatures are NOT yet mapped to a block (so we know what - * still has to be designed). - * - * The known mappings were reverse-engineered from en/index.html: - * - * SOURCE COMPONENT (cleaned DOM) -> EDS BLOCK - * ----------------------------------------------------------------- - * div.teaser.teaser--hero -> hero - * div.teaser.teaser--full-width -> banner - * div.teaser.teaser--checkerboard (±--right) -> columns (one row each) - * div.list.list--content -> cards - * div.secondary-navigation -> jump-nav - * div.text / div.cmp-text (rte) -> (default content) - * div.title -> (default heading) - * div.button -> (default link) - * + <meta name="description"> -> metadata (every page) - * div.teaser.teaser--expired -> (skipped: expired/hidden) - * - * Everything else is reported as UNMAPPED. - * - * Usage: - * node analyze-blocks.js <cleanedDir> [--out report.md] - * - * Requires: cheerio - */ - -const fs = require("fs"); -const path = require("path"); -const cheerio = require("cheerio"); - -const SRC = path.resolve(process.argv[2] || "."); -const outIdx = process.argv.indexOf("--out"); -const OUT = outIdx > -1 ? path.resolve(process.argv[outIdx + 1]) : null; - -// Map a component's (type, variant) to a known block, or null if unmapped. -// variant is the most specific `type--variant` modifier (expired excluded). -function mapToBlock(type, variants) { - const v = new Set(variants); - if (type === "teaser") { - if (v.has("expired")) return "(skip: expired teaser)"; - if (v.has("hero")) return "hero"; - if (v.has("full-width")) return "banner"; - if (v.has("checkerboard")) return "columns"; - if (v.has("tile")) return "cards (horizontal)"; - if (v.has("banner")) return "default + section-metadata (dark)"; - return "cards (horizontal)"; // bare/unknown teaser - } - if (type === "list") { - if (v.has("content")) return "cards"; - if (v.has("links")) return "list (links)"; - if (v.has("detailed")) return "list (detailed)"; - if (v.has("simple-product")) return "list (product)"; - return null; - } - if (type === "carousel") return "carousel"; - if (type === "accordion") return "accordion"; - if (type === "tabs") return "tabs"; - if (type === "navigation") return "(skip: redirect)"; - if (type === "secondary-navigation" || type === "secondaryNavigation") return "jump-nav"; - if (type === "text" || type === "title" || type === "button") return "(default content)"; - // Structural wrappers we intentionally ignore. - if (["container", "section-container", "responsivegrid", "experienceFragment"].includes(type)) - return "(structural)"; - return null; -} - -// Component type -> CSS selector for its root instances in the cleaned DOM. -// Element-agnostic: component roots may be <div>, <section>, <nav>, etc. -const COMPONENT_SELECTORS = { - teaser: ".teaser", - list: ".list", - "secondary-navigation": ".secondary-navigation", - title: ".title", - text: ".text", - button: ".button", - tabs: ".tabs", - accordion: ".accordion", - carousel: ".carousel", - embed: ".embed", - separator: ".separator", - download: ".download", - navigation: ".navigation", - breadcrumb: ".breadcrumb", - search: ".search", -}; - -function variantsOf($el, type) { - const cls = ($el.attr("class") || "").split(/\s+/); - const prefix = type + "--"; - return cls.filter((c) => c.startsWith(prefix)).map((c) => c.slice(prefix.length)); -} - -function analyzePage(html) { - const $ = cheerio.load(html); - const found = []; // {type, variants, block} - const seen = new Set(); - - for (const [type, sel] of Object.entries(COMPONENT_SELECTORS)) { - $(sel).each((_, el) => { - const $el = $(el); - // Skip nested duplicates (a .teaser inside a .teaser, etc.) - if ($el.parents(sel).length) return; - const variants = variantsOf($el, type); - const block = mapToBlock(type, variants); - found.push({ type, variants, block }); - }); - } - // metadata is implicit on every page that has a description. - const hasMeta = $('meta[name="description"]').length || $("title").length; - return { found, hasMeta }; -} - -function findHtml(dir, acc = []) { - for (const e of fs.readdirSync(dir, { withFileTypes: true })) { - const full = path.join(dir, e.name); - if (e.isDirectory()) findHtml(full, acc); - else if (e.isFile() && /\.html?$/i.test(e.name)) acc.push(full); - } - return acc; -} - -function sigKey(type, variants) { - const v = variants.filter((x) => x !== "expired").sort(); - return v.length ? `${type}.${type}--${v.join("+" + type + "--")}` : type; -} - -function main() { - const files = findHtml(SRC).sort(); - const global = new Map(); // signature -> {count, pages:Set, block} - const perPage = []; - - for (const file of files) { - const { found, hasMeta } = analyzePage(fs.readFileSync(file, "utf8")); - const blocks = new Set(); - if (hasMeta) blocks.add("metadata"); - - for (const { type, variants, block } of found) { - const key = sigKey(type, variants); - if (!global.has(key)) global.set(key, { count: 0, pages: new Set(), block }); - const g = global.get(key); - g.count++; - g.pages.add(file); - if (block && !block.startsWith("(")) blocks.add(block); - } - perPage.push({ file: path.relative(SRC, file), blocks: [...blocks].sort() }); - } - - // Build report. - const rows = [...global.entries()].sort((a, b) => b[1].count - a[1].count); - const mapped = rows.filter(([, g]) => g.block && !g.block.startsWith("(")); - const skipped = rows.filter(([, g]) => g.block && g.block.startsWith("(")); - const unmapped = rows.filter(([, g]) => !g.block); - - let md = `# Block discovery report\n\n`; - md += `Scanned **${files.length}** pages in \`${SRC}\`.\n\n`; - - md += `## Component signatures MAPPED to a block\n\n`; - md += `| signature | -> block | instances | pages |\n|---|---|--:|--:|\n`; - for (const [k, g] of mapped) md += `| \`${k}\` | ${g.block} | ${g.count} | ${g.pages.size} |\n`; - - md += `\n## Signatures NOT yet mapped (need a block design)\n\n`; - md += `| signature | instances | pages |\n|---|--:|--:|\n`; - for (const [k, g] of unmapped) md += `| \`${k}\` | ${g.count} | ${g.pages.size} |\n`; - - md += `\n## Intentionally skipped / structural\n\n`; - md += `| signature | note | instances |\n|---|---|--:|\n`; - for (const [k, g] of skipped) md += `| \`${k}\` | ${g.block} | ${g.count} |\n`; - - const out = md; - if (OUT) fs.writeFileSync(OUT, out); - // Console summary. - console.log(`Scanned ${files.length} pages.`); - console.log(`Mapped signatures: ${mapped.length}`); - console.log(`UNMAPPED signatures: ${unmapped.length}`); - for (const [k, g] of unmapped) { - const example = path.relative(SRC, [...g.pages][0]); - console.log(` - ${k} (${g.count} instances, ${g.pages.size} pages)`); - console.log(` example: ${example}`); - } - if (OUT) console.log(`\nFull report -> ${OUT}`); -} - -main(); diff --git a/tools/import/card-list-manifest.tsv b/tools/import/card-list-manifest.tsv deleted file mode 100644 index e977052..0000000 --- a/tools/import/card-list-manifest.tsv +++ /dev/null @@ -1,184 +0,0 @@ -page scheme showImage showDescription sort -/company/about-us/emd100/1920s root=parent true true desc -/company/about-us/emd100/1930s root=parent true true desc -/company/about-us/emd100/1940s root=parent true true asc -/company/about-us/emd100/1950s root=parent true true desc -/company/about-us/emd100/1960s root=parent true true desc -/company/about-us/emd100/1970s root=parent true true desc -/company/about-us/emd100/1980s root=parent true true desc -/company/about-us/emd100/1990s root=parent true true desc -/company/about-us/emd100/2000s root=parent true true desc -/company/about-us/emd100/2010s root=parent true true desc -/company/about-us/emd100/2020s root=parent true true desc -/company/about-us/emd100 (self) true false asc -/company/codeof-conduct root=parent true true asc -/company/community-outreach/home-activities root=/segments true false desc -/company/leadership/larry-haskell root=/company true true asc -/company/leadership/leah-green root=/company true true asc -/company/leadership/sam-doerr root=/segments/infrastructure true true asc -/company/locations/africa root=/segments true true asc -/company/locations/asia root=/segments true true asc -/company/locations/australia root=/segments true true asc -/company/locations/europe/inspection-informations-system-gmb-h root=/segments/infrastructure/signaling true true asc -/company/locations/europe root=/segments true true asc -/company/locations/middle-east-africa root=/segments true true asc -/company/locations/north-america root=/segments true true asc -/company/locations/south-america root=/segments true true asc -/company/locations root=/segments true true desc -/company/news/press-releases (self) false true desc -/company/news root=/company/news/press-releases false true desc -/company/valentines root=/segments true false desc -/segments/freight-car/freight-car-services root=parent true false desc -/segments/freight-car/freight-parts/wheel-axle-bearing-logistics root=/segments/freight-car true false asc -/segments/freight-car/freight-parts root=parent true false asc -/segments/freight-car (self) true true asc -/segments/infrastructure/fasteners/egg-direct-fixation-fastener root=/segments/infrastructure true true asc -/segments/infrastructure/fasteners/high-resilient-special-trackwork-df root=/segments/infrastructure true true asc -/segments/infrastructure/fasteners root=parent true true asc -/segments/infrastructure/rail-services root=parent true true asc -/segments/infrastructure/signaling/asset-protection root=parent true true asc -/segments/infrastructure/signaling/services root=parent true true asc -/segments/infrastructure/signaling/structures root=parent true true asc -/segments/infrastructure/signaling/systems root=/segments/rail-technology/predictive-condition-monitoring-systems true true asc -/segments/infrastructure/signaling/systems root=parent true true asc -/segments/infrastructure/signaling (self) true true asc -/segments/infrastructure/trackwork root=parent true true asc -/segments/infrastructure (self) true true asc -/segments/locomotive/engines/cat-rail-engines root=parent true true asc -/segments/locomotive/engines/engine-product-support root=parent true false asc -/segments/locomotive/engines/marine-stationary-engines root=parent true false asc -/segments/locomotive/engines/natural-gas-solutions root=parent true false asc -/segments/locomotive/engines (self) true false asc -/segments/locomotive/locomotive-components FLAG(curated) true false asc -/segments/locomotive/locomotive-electronics root=/segments/rail-technology true false asc -/segments/locomotive/locomotive-services/contract-maintenance-customer-performance root=/company/locations true true asc -/segments/locomotive/locomotive-services/engine-overhaul root=/segments/locomotive true true asc -/segments/locomotive/locomotive-services/locomotive-remanufacturing root=/segments/locomotive true true asc -/segments/locomotive/locomotive-services root=parent true true asc -/segments/locomotive/locomotives/freight-locomotives/gt38-ac root=parent true true asc -/segments/locomotive/locomotives/freight-locomotives/gt38-acl root=parent true true asc -/segments/locomotive/locomotives/freight-locomotives/gt42-ac root=parent true true asc -/segments/locomotive/locomotives/freight-locomotives/gt46-ac root=parent true true asc -/segments/locomotive/locomotives/freight-locomotives/gt46-ac-1520mm root=parent true true asc -/segments/locomotive/locomotives/freight-locomotives/gt46-c-a-ce-gen-ii root=parent true true asc -/segments/locomotive/locomotives/freight-locomotives/gt46-c-a-ce-gen-iii root=parent true true asc -/segments/locomotive/locomotives/freight-locomotives/jt42-a-ce root=parent true true asc -/segments/locomotive/locomotives/freight-locomotives/sd70-acs root=parent true true asc -/segments/locomotive/locomotives/freight-locomotives/sd70-a-ce-45 root=parent true true asc -/segments/locomotive/locomotives/freight-locomotives/sd70-a-ce-bb root=parent true true asc -/segments/locomotive/locomotives/freight-locomotives/sd70-a-ce-l-ci root=parent true true asc -/segments/locomotive/locomotives/freight-locomotives/sd70-a-ce-t4 root=parent true true asc -/segments/locomotive/locomotives/freight-locomotives/sd70-a-ce root=parent true true asc -/segments/locomotive/locomotives/freight-locomotives/sd80-a-ce root=parent true true asc -/segments/locomotive/locomotives/freight-locomotives (self) true false asc -/segments/locomotive/locomotives/freight-locomotives root=/segments/locomotive true true asc -/segments/locomotive/locomotives/passenger-locomotives/repowered root=/segments/locomotive/locomotives/freight-locomotives true true asc -/segments/locomotive/locomotives/passenger-locomotives root=/segments/locomotive/locomotives/freight-locomotives true true asc -/segments/locomotive/locomotives/repowered-locomotives/emd20-b root=parent true true asc -/segments/locomotive/locomotives/repowered-locomotives/emd24-b root=parent true true asc -/segments/locomotive/locomotives/repowered-locomotives/emd30-c root=parent true true asc -/segments/locomotive/locomotives/repowered-locomotives/emd710-eco root=parent true true asc -/segments/locomotive/locomotives/repowered-locomotives/gt26-cu-3 root=/segments/locomotive/locomotives/freight-locomotives true true asc -/segments/locomotive/locomotives/repowered-locomotives (self) true false asc -/segments/locomotive/locomotives/repowered-locomotives root=/segments/locomotive true false asc -/segments/locomotive/locomotives root=parent true true asc -/segments/locomotive (self) true true asc -/segments/rail-technology/nitro/nitro-yard root=/segments/rail-technology true true asc -/segments/rail-technology/nitro root=parent true true asc -/segments/rail-technology/pr-uptime-suite root=parent true true asc -/segments/rail-technology/power-view-suite/connect root=/segments/rail-technology true false asc -/segments/rail-technology/power-view-suite/event-recorder root=/segments/rail-technology true false asc -/segments/rail-technology/power-view-suite/fuel-monitoring root=/segments/rail-technology true false asc -/segments/rail-technology/power-view-suite root=parent true false asc -/segments/rail-technology/predictive-condition-monitoring-systems/argus root=parent true false asc -/segments/rail-technology/predictive-condition-monitoring-systems/battery-condition-monitoring root=parent true false asc -/segments/rail-technology/predictive-condition-monitoring-systems/oscar root=parent true false asc -/segments/rail-technology/predictive-condition-monitoring-systems root=parent true true asc -/segments/rail-technology/talos root=parent true true asc -/segments/rail-technology (self) true false desc -/segments (self) true false desc -/services/contract-maintenance-customer-performance FLAG(no-items) true true asc -/services/parts FLAG(no-items) true false asc -/services (self) true true asc -/company/codeof-conduct FLAG(no-items) true true asc -/company/community-outreach/home-activities FLAG(no-items) true false desc -/company/leadership/john-newman FLAG(no-items) true true asc -/company/leadership/larry-haskell FLAG(no-items) true true asc -/company/leadership/leah-green FLAG(no-items) true true asc -/company/leadership/sam-doerr FLAG(no-items) true true asc -/company/locations/africa FLAG(no-items) true true asc -/company/locations/asia FLAG(no-items) true true asc -/company/locations/australia FLAG(no-items) true true asc -/company/locations/europe/inspection-informations-system-gmb-h root=/segments/infrastructure/signaling true true asc -/company/locations/europe FLAG(no-items) true true asc -/company/locations/middle-east-africa FLAG(no-items) true true asc -/company/locations/north-america FLAG(no-items) true true asc -/company/locations/south-america FLAG(no-items) true true asc -/company/locations FLAG(no-items) true true desc -/company/news/press-releases FLAG(no-items) false true desc -/company/news FLAG(no-items) false true desc -/segments/freight-car/freight-car-services FLAG(no-items) true false desc -/segments/freight-car/freight-parts/wheel-axle-bearing-logistics FLAG(no-items) true false asc -/segments/freight-car/freight-parts FLAG(no-items) true false asc -/segments/freight-car FLAG(no-items) true true asc -/segments/infrastructure/fasteners FLAG(no-items) true true asc -/segments/infrastructure/rail-services FLAG(no-items) true true asc -/segments/infrastructure/signaling/asset-protection FLAG(no-items) true true asc -/segments/infrastructure/signaling/services FLAG(no-items) true true asc -/segments/infrastructure/signaling/structures FLAG(no-items) true true asc -/segments/infrastructure/signaling/systems FLAG(no-items) true true asc -/segments/infrastructure/signaling/systems FLAG(no-items) true true asc -/segments/infrastructure/signaling FLAG(no-items) true true asc -/segments/infrastructure/trackwork FLAG(no-items) true true asc -/segments/infrastructure FLAG(no-items) true true asc -/segments/locomotive/engines/cat-rail-engines FLAG(no-items) true true asc -/segments/locomotive/engines/engine-product-support FLAG(no-items) true false asc -/segments/locomotive/engines/marine-stationary-engines FLAG(no-items) true false asc -/segments/locomotive/engines/natural-gas-solutions FLAG(no-items) true false asc -/segments/locomotive/engines FLAG(no-items) true false asc -/segments/locomotive/locomotive-components FLAG(no-items) true false asc -/segments/locomotive/locomotive-electronics FLAG(no-items) true false asc -/segments/locomotive/locomotive-services FLAG(no-items) true true asc -/segments/locomotive/locomotives/freight-locomotives/gt38-ac FLAG(no-items) true true asc -/segments/locomotive/locomotives/freight-locomotives/gt38-acl FLAG(no-items) true true asc -/segments/locomotive/locomotives/freight-locomotives/gt42-ac FLAG(no-items) true true asc -/segments/locomotive/locomotives/freight-locomotives/gt46-ac FLAG(no-items) true true asc -/segments/locomotive/locomotives/freight-locomotives/gt46-ac-1520mm FLAG(no-items) true true asc -/segments/locomotive/locomotives/freight-locomotives/gt46-c-a-ce-gen-ii FLAG(no-items) true true asc -/segments/locomotive/locomotives/freight-locomotives/gt46-c-a-ce-gen-iii FLAG(no-items) true true asc -/segments/locomotive/locomotives/freight-locomotives/sd70-acs FLAG(no-items) true true asc -/segments/locomotive/locomotives/freight-locomotives/sd70-a-ce-45 FLAG(no-items) true true asc -/segments/locomotive/locomotives/freight-locomotives/sd70-a-ce-bb FLAG(no-items) true true asc -/segments/locomotive/locomotives/freight-locomotives/sd70-a-ce-l-ci FLAG(no-items) true true asc -/segments/locomotive/locomotives/freight-locomotives/sd70-a-ce-t4 FLAG(no-items) true true asc -/segments/locomotive/locomotives/freight-locomotives/sd70-a-ce FLAG(no-items) true true asc -/segments/locomotive/locomotives/freight-locomotives/sd80-a-ce FLAG(no-items) true true asc -/segments/locomotive/locomotives/freight-locomotives FLAG(no-items) true false asc -/segments/locomotive/locomotives/freight-locomotives FLAG(no-items) true true asc -/segments/locomotive/locomotives/passenger-locomotives/f125 FLAG(no-items) true true asc -/segments/locomotive/locomotives/passenger-locomotives/repowered FLAG(no-items) true true asc -/segments/locomotive/locomotives/passenger-locomotives FLAG(no-items) true true asc -/segments/locomotive/locomotives/repowered-locomotives/emd20-b FLAG(no-items) true true asc -/segments/locomotive/locomotives/repowered-locomotives/emd24-b FLAG(no-items) true true asc -/segments/locomotive/locomotives/repowered-locomotives/emd30-c FLAG(no-items) true true asc -/segments/locomotive/locomotives/repowered-locomotives/emd710-eco FLAG(no-items) true true asc -/segments/locomotive/locomotives/repowered-locomotives/gt26-cu-3 FLAG(no-items) true true asc -/segments/locomotive/locomotives/repowered-locomotives FLAG(no-items) true false asc -/segments/locomotive/locomotives/repowered-locomotives FLAG(no-items) true false asc -/segments/locomotive/locomotives FLAG(no-items) true true asc -/segments/locomotive FLAG(no-items) true true asc -/segments/rail-technology/nitro/nitro-yard FLAG(no-items) true true asc -/segments/rail-technology/nitro FLAG(no-items) true true asc -/segments/rail-technology/pr-uptime-suite FLAG(no-items) true true asc -/segments/rail-technology/power-view-suite/connect FLAG(no-items) true false asc -/segments/rail-technology/power-view-suite/event-recorder FLAG(no-items) true false asc -/segments/rail-technology/power-view-suite/fuel-monitoring FLAG(no-items) true false asc -/segments/rail-technology/power-view-suite FLAG(no-items) true false asc -/segments/rail-technology/predictive-condition-monitoring-systems/argus FLAG(no-items) true false asc -/segments/rail-technology/predictive-condition-monitoring-systems/battery-condition-monitoring FLAG(no-items) true false asc -/segments/rail-technology/predictive-condition-monitoring-systems/oscar FLAG(no-items) true false asc -/segments/rail-technology/predictive-condition-monitoring-systems FLAG(no-items) true true asc -/segments/rail-technology/talos FLAG(no-items) true true asc -/segments/rail-technology FLAG(no-items) true false desc -/segments FLAG(no-items) true false desc -/services FLAG(no-items) true true asc \ No newline at end of file diff --git a/tools/import/fix-http-images.js b/tools/import/fix-http-images.js deleted file mode 100644 index 27ff4c3..0000000 --- a/tools/import/fix-http-images.js +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env node -/** - * fix-http-images.js - * - * Upgrades image sources from http:// to https:// across the repo. Scoped to - * src="..." and srcset="..." attribute values only, so external href links - * (which may not support https) are left untouched. - * - * Usage: node fix-http-images.js <repoRoot> - */ -const fs = require("fs"); -const path = require("path"); - -const REPO = path.resolve(process.argv[2]); - -function walk(dir, acc = []) { - for (const e of fs.readdirSync(dir, { withFileTypes: true })) { - if (e.name === ".aem" || e.name.startsWith(".")) continue; - const f = path.join(dir, e.name); - if (e.isDirectory()) walk(f, acc); - else if (/\.html?$/i.test(e.name)) acc.push(f); - } - return acc; -} - -let filesChanged = 0, occurrences = 0; -// Match any http/https scheme (any case, e.g. stray "httpS://") in an image -// attribute and force it to canonical lowercase https://. -const RE = /\b(src|srcset)="https?:\/\//gi; - -for (const file of walk(REPO)) { - const orig = fs.readFileSync(file, "utf8"); - let n = 0; - const out = orig.replace(RE, (m, attr) => { - const fixed = `${attr}="https://`; - if (m === fixed) return m; // already canonical, leave as-is - n++; return fixed; - }); - if (n > 0) { fs.writeFileSync(file, out); filesChanged++; occurrences += n; } -} -console.log(`image sources normalized to https://: ${occurrences} in ${filesChanged} file(s)`); diff --git a/tools/import/list-endpoints.js b/tools/import/list-endpoints.js deleted file mode 100644 index 71e7a3f..0000000 --- a/tools/import/list-endpoints.js +++ /dev/null @@ -1,82 +0,0 @@ -#!/usr/bin/env node -/** - * list-endpoints.js - * - * Scans cleaned Progress Rail pages and, for every AEM List component, prints - * the "getDegSubList" endpoint that returns the COMPLETE set of list items - * (the static HTML only contains the first page; the rest load dynamically). - * - * The endpoint is built from the list's data-list-resourcetype attribute: - * https://<host><PATH>.getDegSubListv10.html - * ?listResourcePath=<urlencoded PATH>&isLoadMore=0&paginateAfter=0&itemPerPage=<N> - * - * Output is TSV: <page>\t<variant>\t<resourcePath>\t<url> - * Pair it with --curl to emit a runnable curl script that saves each response - * to <outDir>/<sha-of-path>.json so a transformer can read them back. - * - * Usage: - * node list-endpoints.js <cleanedDir> [--host https://www.progressrail.com] - * [--per 99] [--curl <outDir>] - */ - -const fs = require("fs"); -const path = require("path"); -const crypto = require("crypto"); -const cheerio = require("cheerio"); - -const SRC = path.resolve(process.argv[2] || "."); -const arg = (k, d) => { const i = process.argv.indexOf(k); return i > -1 ? process.argv[i + 1] : d; }; -const HOST = arg("--host", "https://www.progressrail.com"); -const PER = arg("--per", "99"); -const curlIdx = process.argv.indexOf("--curl"); -const CURL_DIR = curlIdx > -1 ? process.argv[curlIdx + 1] : null; - -function endpointFor(resourcePath) { - return ( - HOST + resourcePath + ".getDegSubListv10.html" + - "?listResourcePath=" + encodeURIComponent(resourcePath) + - "&isLoadMore=0&paginateAfter=0&itemPerPage=" + PER - ); -} -// Stable filename for a resource path (so the transformer can find the JSON). -const cacheName = (rp) => crypto.createHash("sha1").update(rp).digest("hex").slice(0, 16) + ".json"; - -function findHtml(dir, acc = []) { - for (const e of fs.readdirSync(dir, { withFileTypes: true })) { - const full = path.join(dir, e.name); - if (e.isDirectory()) findHtml(full, acc); - else if (e.isFile() && /\.html?$/i.test(e.name)) acc.push(full); - } - return acc; -} - -const rows = []; -for (const file of findHtml(SRC).sort()) { - const $ = cheerio.load(fs.readFileSync(file, "utf8"), { decodeEntities: false }); - $(".list").each((_, L) => { - const $L = $(L); - const rp = $L.find("[data-list-resourcetype]").attr("data-list-resourcetype") - || $L.attr("data-list-resourcetype"); - if (!rp) return; - const variant = ($L.attr("class") || "").split(/\s+/) - .filter((c) => c.startsWith("list--")).join(",") || "list"; - rows.push({ page: path.relative(SRC, file), variant, rp }); - }); -} - -// De-dupe by resource path (same list can appear via multiple selectors). -const seen = new Set(); -const uniq = rows.filter((r) => (seen.has(r.rp) ? false : seen.add(r.rp))); - -if (CURL_DIR) { - const lines = ["#!/bin/sh", `mkdir -p '${CURL_DIR}'`]; - for (const r of uniq) { - lines.push(`curl -s '${endpointFor(r.rp)}' -o '${path.join(CURL_DIR, cacheName(r.rp))}' # ${r.variant} ${r.page}`); - } - process.stdout.write(lines.join("\n") + "\n"); -} else { - for (const r of uniq) { - process.stdout.write(`${r.page}\t${r.variant}\t${r.rp}\t${endpointFor(r.rp)}\n`); - } - process.stderr.write(`\n${uniq.length} unique list endpoint(s) across ${rows.length} list instance(s).\n`); -} diff --git a/tools/import/merge-pr-redirects.js b/tools/import/merge-pr-redirects.js deleted file mode 100644 index 6c5e310..0000000 --- a/tools/import/merge-pr-redirects.js +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env node -/** - * merge-pr-redirects.js - * - * Builds the press-release old->new redirects from transform-press-releases.js - * --map output(s) and merges them into <contentRoot>/redirects.json. - * - * node merge-pr-redirects.js <contentRoot> <map-en.csv> [<map-fr.csv> ...] - * - * The language is taken from the map filename (…-fr… => fr, else en). - */ -const fs = require("fs"); -const path = require("path"); - -const CONTENT = path.resolve(process.argv[2]); -const maps = process.argv.slice(3); - -function load(f) { - return fs.readFileSync(f, "utf8").split("\n").slice(1).filter(Boolean) - .map((l) => l.match(/"((?:[^"]|"")*)"/g).map((s) => s.slice(1, -1).replace(/""/g, '"'))); -} - -const reds = []; -for (const m of maps) { - const lang = /(^|[^a-z])fr([^a-z]|$)/i.test(path.basename(m)) ? "fr" : "en"; - for (const [src, nw] of load(m)) { - reds.push({ - Source: `/${lang}/Company/News/PressReleases/${src}`, - Destination: `/${lang}/company/news/press-releases/${nw.replace(/\.html?$/i, "")}`, - }); - } -} - -const rp = path.join(CONTENT, "redirects.json"); -const sheet = JSON.parse(fs.readFileSync(rp, "utf8")); -const seen = new Set(reds.map((r) => r.Source)); -const merged = reds.concat((sheet.data || []).filter((r) => !seen.has(r.Source))) - .sort((a, b) => a.Source.localeCompare(b.Source)); -sheet.data = merged; sheet.total = merged.length; sheet.limit = merged.length; sheet.offset = 0; -fs.writeFileSync(rp, JSON.stringify(sheet)); -console.log(`press-release redirects: ${reds.length} merged, ${merged.length} total in redirects.json`); diff --git a/tools/import/reconcile.js b/tools/import/reconcile.js deleted file mode 100644 index 118fedd..0000000 --- a/tools/import/reconcile.js +++ /dev/null @@ -1,107 +0,0 @@ -#!/usr/bin/env node -/** - * reconcile.js - * - * Protects human edits made in DA from being clobbered when the transform is - * re-run. Compares the freshly transformed output (en/, fr/) against the - * version pulled from DA (.aem/) and classifies every page, ignoring DA's - * whitespace reformatting: - * - * identical - same content - * transform - only the block STRUCTURE changed (a transform improvement); - * safe to take the new output - * author-edit - the TEXT/content changed but structure didn't -> a human - * edited it in DA; preserve the DA version - * conflict - both the author and the transform changed it -> preserve the - * DA version and flag for manual review - * - * The set of author-edit + conflict pages is written to - * .aem/protected-files.json (UNION with any prior list, so protection sticks). - * Run the transform with --skip-protected and it will never overwrite them. - * - * With --apply, those pages are copied from .aem back into en//fr/ so the - * working tree holds the human edits. - * - * Usage: - * node reconcile.js <repoRoot> [--apply] - */ - -const fs = require("fs"); -const path = require("path"); -const cheerio = require("cheerio"); - -const REPO = path.resolve(process.argv[2]); -const APPLY = process.argv.includes("--apply"); -const AEM = path.join(REPO, ".aem"); - -function walk(dir, base, acc = []) { - if (!fs.existsSync(dir)) return acc; - for (const e of fs.readdirSync(dir, { withFileTypes: true })) { - const f = path.join(dir, e.name); - if (e.isDirectory()) walk(f, base, acc); - else if (/\.html$/.test(e.name)) acc.push(path.relative(base, f)); - } - return acc; -} - -const BLOCK_RE = /^(hero|banner|cards|columns|carousel|accordion|tabs|jump-nav|list|metadata|section-metadata)\b/; - -// Semantic signature: block-structure sequence + normalised text. -// Whitespace-insensitive, so DA's serializer reformatting is not seen as a change. -function signature(html) { - const $ = cheerio.load(html, { decodeEntities: false }); - const main = $("main"); - const struct = []; - main.find("div[class]").each((_, d) => { - const c = ($(d).attr("class") || "").trim(); - if (BLOCK_RE.test(c)) struct.push(c); - }); - return { struct: struct.join("|"), text: main.text().replace(/\s+/g, " ").trim() }; -} - -// Detect text edits robustly: compare the multiset of "words" so a reordered -// block doesn't read as a content change. -const wordbag = (t) => t.toLowerCase().match(/[a-z0-9à-ÿ]+/gi) || []; -function textChanged(a, b) { - const A = wordbag(a).sort().join(" "), B = wordbag(b).sort().join(" "); - return A !== B; -} - -const protectedPath = path.join(__dirname, "protected-files.json"); // tooling state, kept out of content -const prior = fs.existsSync(protectedPath) ? new Set(JSON.parse(fs.readFileSync(protectedPath, "utf8"))) : new Set(); - -const files = [...walk(path.join(REPO, "en"), REPO), ...walk(path.join(REPO, "fr"), REPO)].sort(); -const report = [["file", "classification"]]; -const tally = {}; -const nowProtected = new Set(prior); -let applied = 0; - -for (const rel of files) { - const aemFile = path.join(AEM, rel); - if (!fs.existsSync(aemFile)) { report.push([rel, "new"]); tally.new = (tally.new || 0) + 1; continue; } - const N = signature(fs.readFileSync(path.join(REPO, rel), "utf8")); - const A = signature(fs.readFileSync(aemFile, "utf8")); - const txt = textChanged(A.text, N.text); - const struct = A.struct !== N.struct; - - let cls = (!txt && !struct) ? "identical" - : !txt ? "transform" - : struct ? "conflict" : "author-edit"; - - if (cls === "author-edit" || cls === "conflict") { - nowProtected.add(rel); - if (APPLY) { fs.writeFileSync(path.join(REPO, rel), fs.readFileSync(aemFile, "utf8")); applied++; } - } - report.push([rel, cls]); - tally[cls] = (tally[cls] || 0) + 1; -} - -fs.writeFileSync(protectedPath, JSON.stringify([...nowProtected].sort(), null, 2)); -fs.writeFileSync(path.join(__dirname, "reconcile-report.csv"), - report.map((r) => r.map((c) => `"${c}"`).join(",")).join("\n")); - -console.log(`reconciled ${files.length} files`); -Object.entries(tally).sort((a, b) => b[1] - a[1]).forEach(([k, v]) => console.log(` ${String(v).padStart(4)} ${k}`)); -console.log(`protected list: ${nowProtected.size} files (${path.relative(process.cwd(), protectedPath)})`); -if (APPLY) console.log(` applied ${applied} DA version(s) to the working tree`); -else console.log(` report-only; rerun with --apply to copy DA edits into en//fr/`); diff --git a/tools/import/rewrite-links.js b/tools/import/rewrite-links.js deleted file mode 100644 index 5f85a9b..0000000 --- a/tools/import/rewrite-links.js +++ /dev/null @@ -1,94 +0,0 @@ -#!/usr/bin/env node -/** - * rewrite-links.js - * - * Walks the content/aemsites/progressrail folder and fixes every absolute link - * that points at the Progress Rail host: - * - * 1. Strips the host so the link is relative to the host: - * href="https://www.progressrail.com/en/Company.html" -> /en/Company.html - * 2. If that host-relative path is a Source in redirects.json, rewrites it to - * the matching Destination: - * /en/Company.html -> /en/company - * (any #fragment or ?query is preserved). - * - * Links whose path is NOT found in redirects.json are still made host-relative, - * and are written to a review file so they can be checked / given redirects. - * - * Usage: - * node rewrite-links.js <repoRoot> [--review <file>] [--dry] - */ - -const fs = require("fs"); -const path = require("path"); - -const REPO = path.resolve(process.argv[2]); -const reviewIdx = process.argv.indexOf("--review"); -const REVIEW = reviewIdx > -1 ? path.resolve(process.argv[reviewIdx + 1]) - : path.join(__dirname, "links-not-in-redirects.tsv"); // beside the script, not in content -const DRY = process.argv.includes("--dry"); - -const redirects = JSON.parse(fs.readFileSync(path.join(REPO, "redirects.json"), "utf8")); -const map = new Map((redirects.data || []).map((r) => [r.Source, r.Destination])); - -function walk(dir, acc = []) { - for (const e of fs.readdirSync(dir, { withFileTypes: true })) { - if (e.name === ".aem" || e.name.startsWith(".")) continue; - const f = path.join(dir, e.name); - if (e.isDirectory()) walk(f, acc); - else if (/\.html?$/i.test(e.name)) acc.push(f); - } - return acc; -} - -// Any href; we then decide if it's a host-absolute or root-relative internal link. -const RE = /href="([^"]*)"/gi; -const HOST = /^https?:\/\/(?:www\.)?progressrail\.com/i; - -const notFound = new Map(); // basePath -> { count, files:Set } -let filesChanged = 0, rewritten = 0, hostRelOnly = 0; - -for (const file of walk(REPO)) { - const orig = fs.readFileSync(file, "utf8"); - let changed = false; - const record = (basePath) => { - if (!notFound.has(basePath)) notFound.set(basePath, { count: 0, files: new Set() }); - const e = notFound.get(basePath); e.count++; e.files.add(path.relative(REPO, file)); - }; - const out = orig.replace(RE, (m, value) => { - let rest, isHost = false; - if (HOST.test(value)) { rest = value.replace(HOST, ""); isHost = true; } - else if (value.startsWith("/")) { rest = value; } // root-relative internal link - else return m; // external / mailto / #anchor / other - const cut = rest.search(/[#?]/); - let basePath = cut >= 0 ? rest.slice(0, cut) : rest; - const suffix = cut >= 0 ? rest.slice(cut) : ""; - if (basePath === "") basePath = "/"; - - if (map.has(basePath)) { changed = true; rewritten++; return `href="${map.get(basePath)}${suffix}"`; } - if (isHost) { // strip host even if no redirect - changed = true; hostRelOnly++; record(basePath); - return `href="${basePath}${suffix}"`; - } - // root-relative not in redirects: leave as-is, but flag genuine internal page links - if (/^\/(?:en|fr)\/.+\.html?$/i.test(basePath)) record(basePath); - return m; - }); - if (changed && !DRY) { fs.writeFileSync(file, out); } - if (changed) filesChanged++; -} - -// Write the review list (paths not covered by a redirect). -const rows = [["host_relative_path", "occurrences", "example_file"]]; -[...notFound.entries()] - .sort((a, b) => b[1].count - a[1].count || a[0].localeCompare(b[0])) - .forEach(([p, e]) => rows.push([p, String(e.count), [...e.files][0]])); -if (!DRY) { - fs.mkdirSync(path.dirname(REVIEW), { recursive: true }); - fs.writeFileSync(REVIEW, rows.map((r) => r.join("\t")).join("\n")); -} - -console.log(`${DRY ? "[dry-run] " : ""}files touched: ${filesChanged}`); -console.log(`links rewritten via redirect: ${rewritten}`); -console.log(`links made host-relative but NOT in redirects: ${hostRelOnly} (${notFound.size} distinct paths)`); -console.log(`review file: ${REVIEW}`); diff --git a/tools/import/run-import.sh b/tools/import/run-import.sh deleted file mode 100755 index 7e2fcfc..0000000 --- a/tools/import/run-import.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/sh -# Full import pipeline, in the required order. Run as: -# tools/import/run-import.sh <sourceRoot> <contentRoot> -# -# <sourceRoot> cleaned crawl (contains en.html, fr.html, en/, fr/) -# <contentRoot> EDS content repo (e.g. .../content/aemsites/progressrail) -# -# IMPORTANT: rewrite-links must run AFTER the transforms, because the transform -# emits links in their raw source form (/en/Company.html). This wrapper keeps -# that ordering so a transform is never left without the link-rewrite after it. -set -e - -SRC="$1"; CONTENT="$2" -DIR="$(cd "$(dirname "$0")" && pwd)" -if [ -z "$SRC" ] || [ -z "$CONTENT" ]; then - echo "usage: run-import.sh <sourceRoot> <contentRoot>"; exit 1 -fi - -echo "1/5 block pages (skipping protected/author-edited)…" -node "$DIR/transform-page.js" "$SRC" "$CONTENT" --skip-protected - -echo "2/5 press releases…" -node "$DIR/transform-press-releases.js" "$SRC/en/Company/News/PressReleases" \ - "$CONTENT/en/company/news/press-releases" --map "$DIR/.pr-map-en.csv" -node "$DIR/transform-press-releases.js" "$SRC/fr/Company/News/PressReleases" \ - "$CONTENT/fr/company/news/press-releases" --map "$DIR/.pr-map-fr.csv" - -echo "3/5 merge press-release redirects…" -node "$DIR/merge-pr-redirects.js" "$CONTENT" "$DIR/.pr-map-en.csv" "$DIR/.pr-map-fr.csv" - -echo "4/5 rewrite links (host-absolute + root-relative -> redirect destinations)…" -node "$DIR/rewrite-links.js" "$CONTENT" - -echo "5/5 upgrade http image sources to https…" -node "$DIR/fix-http-images.js" "$CONTENT" - -echo "import pipeline complete." diff --git a/tools/import/transform-page.js b/tools/import/transform-page.js deleted file mode 100644 index f95acc0..0000000 --- a/tools/import/transform-page.js +++ /dev/null @@ -1,1040 +0,0 @@ -#!/usr/bin/env node -/** - * transform-page.js (prototype, validated on Fasteners.html) - * - * Transforms a cleaned Progress Rail page into the Edge Delivery (EDS) block - * format, walking the top-level AEM components in document order. - * - * Block mappings (extended from the homepage reverse-engineering): - * teaser--hero -> hero - * teaser--full-width -> banner - * teaser--checkerboard -> columns - * teaser--tile -> cards (horizontal) <-- NEW - * list--content -> cards - * secondary-navigation -> jump-nav - * title / texteditor -> default headings & paragraphs - * teaser--expired -> skipped - * <title> + description -> metadata - * - * Consecutive components that map to the same card/column block are grouped - * into a single block (so a run of tiles becomes one "cards (horizontal)"). - * - * Output paths are ALWAYS lowercased / kebab-cased (AEM paths have capitals, - * underscores and camelCase; EDS wants lowercase-hyphen paths). E.g. - * en/Segments/Infrastructure/Fasteners.html - * -> en/segments/infrastructure/fasteners.html - * en/Company/Community_Outreach/ChristmasForKids.html - * -> en/company/community-outreach/christmas-for-kids.html - * - * Usage: - * node transform-page.js <sourceRoot> <contentRoot> - * <sourceRoot> cleaned site root (contains en.html, fr.html, en/, fr/), - * e.g. .../downloads/www.progressrail.com/cleaned - * <contentRoot> EDS content repo to write into, - * e.g. .../content/aemsites/progressrail - * Also exports { transformFile, kebabPath, runBatch } for programmatic use. - * Requires cheerio. - */ - -const fs = require("fs"); -const path = require("path"); -const cheerio = require("cheerio"); - -// Lowercase + kebab a single path segment (handles _, spaces, camelCase, acronyms). -function kebabSeg(s) { - const ext = /\.html?$/i.test(s) ? s.match(/\.html?$/i)[0].toLowerCase() : ""; - return ( - s - .replace(/\.html?$/i, "") - .replace(/[_\s]+/g, "-") - .replace(/([a-z0-9])([A-Z])/g, "$1-$2") // camelCase boundary - .replace(/([A-Z]+)([A-Z][a-z])/g, "$1-$2") // ACRONYMWord boundary - .toLowerCase() - .replace(/-+/g, "-") - .replace(/^-|-$/g, "") + ext - ); -} -const kebabPath = (rel) => rel.split("/").map(kebabSeg).join("/"); - -// Current document — (re)assigned per file by transformFile(). -let $; - -const https = (u) => (u || "").replace(/^http:\/\//, "https://"); - -// Match DA's serialization to keep diffs clean: literal non-breaking space, and -// & for ampersands inside attribute values (cheerio emits   / &). -function daSerialize(html) { - // Drop every non-breaking space (entity or literal char) -> regular space. - html = html.replace(/ | | | /gi, " "); - // Normalize the URL scheme to lowercase (source has stray "httpS://"). - html = html.replace(/\bhttp(s)?:\/\//gi, (_m, s) => (s ? "https" : "http") + "://"); - // Use & for ampersands inside attribute values (e.g. alt="News & Events"). - return html.replace(/="[^"]*"/g, (m) => m.replace(/&/g, "&")); -} - -// Build an EDS <picture> from a single image URL. -function pic(src, alt = "") { - src = https(src); - return ( - `<picture><source srcset="${src}">` + - `<source srcset="${src}" media="(min-width: 600px)">` + - `<img src="${src}" alt="${alt}" loading="lazy"></picture>` - ); -} - -// Text of a link/button with any material-icons glyph text removed. -function linkText($a) { - const c = $a.clone(); - c.find(".material-icons").remove(); - return c.text().replace(/\s+/g, " ").trim(); -} - -function firstImg($el) { - const img = $el.find("img").first(); - return { src: img.attr("src") || "", alt: img.attr("alt") || "" }; -} - -// Descriptive body paragraph of a teaser (skip dates / hidden fields). -// All meaningful body paragraphs of a teaser (role line, bio, etc.), in order, -// skipping datelines and empties. (Leadership teasers have several paragraphs.) -function teaserParas($t) { - const scope = $t.find(".teaser__text-wrap").first(); - const where = scope.length ? scope : $t; - const out = [], seen = new Set(); - where.find("p").each((_, p) => { - const t = $(p).text().replace(/\s+/g, " ").trim(); - if (!t) return; - if (/^\d{4}-\d{2}-\d{2}$/.test(t)) return; // iso date - if (/^[A-Z][a-z]+ \d{1,2}, \d{4}$/.test(t)) return; // Month D, YYYY - if (seen.has(t)) return; - seen.add(t); - out.push(t); - }); - return out; -} -const parasHtml = (paras) => paras.map((p) => `<p>${p}</p>`).join(""); - -function teaserCTA($t) { - const a = $t.find("a.button, a.teaserDefaultButtonText, a").first(); - if (!a.length) return ""; - return { href: https(a.attr("href")), text: linkText(a) }; -} - -// ---- block builders (match the homepage output conventions) ---- - -function heroBlock($t) { - const h = $t.find(".teaserHeading").first().text().trim(); - const { src, alt } = firstImg($t); - const cta = teaserCTA($t); - const ctaHtml = cta && cta.text - ? `<p><a href="${cta.href}"><strong>${cta.text}</strong></a></p>` : ""; - // two-cell model: text (heading + body + CTA) then image (matches current DA hero) - return `<div class="hero"><div><div><h1>${h}</h1>${parasHtml(teaserParas($t))}${ctaHtml}</div><div>${pic(src, alt)}</div></div></div>`; -} - -function bannerBlock($t) { - const h = $t.find(".teaserHeading").first().text().trim(); - const { src, alt } = firstImg($t); - const cta = teaserCTA($t); - const ctaHtml = cta && cta.text - ? `<p><a href="${cta.href}"><strong>${cta.text}</strong></a></p>` : ""; - // teaser--full-width is a hero (DA reclassified banner -> hero); keep h2 + body - return `<div class="hero"><div><div><h2>${h}</h2>${parasHtml(teaserParas($t))}${ctaHtml}</div><div>${pic(src, alt)}</div></div></div>`; -} - -// One tile -> one card row (image cell + text cell). -function tileRow($t) { - const h = $t.find(".teaserHeading").first().text().trim(); - const { src, alt } = firstImg($t); - const cta = teaserCTA($t); - const parts = [`<h3>${h}</h3>`, parasHtml(teaserParas($t))]; - if (cta && cta.text) parts.push(`<p><a href="${cta.href}">${cta.text}</a></p>`); - return `<div><div>${pic(src, alt)}</div><div>${parts.join("")}</div></div>`; -} - -function cardsHorizontal(tiles) { - return `<div class="cards horizontal">${tiles.map(tileRow).join("")}</div>`; -} - -// list--content -> cards -function listCardsBlock($list) { - const rows = []; - $list.find("li.list__item").each((_, li) => { - const $li = $(li); - const name = $li.find(".list__name").text().trim(); - const desc = $li.find(".list__item-description, .list__description").first().text().trim() - || $li.find("p").first().text().trim(); - const { src, alt } = firstImg($li); - const a = $li.find("a.list__item-content, a").first(); - const href = https(a.attr("href")); - if (!name && !src && !href) return; // skip empty placeholder items - const text = $li.find(".list__item-cta, .list__readmore").first().text().trim() || "Learn More"; - const parts = [`<h3>${name}</h3>`]; - if (desc) parts.push(`<p>${desc}</p>`); - if (href) parts.push(`<p><a href="${href}">${text}</a></p>`); - rows.push(`<div><div>${pic(src, alt)}</div><div>${parts.join("")}</div></div>`); - }); - return `<div class="cards">${rows.join("")}</div>`; -} - -// Lowercase in-page anchor hrefs (#Foo -> #foo) to match the lowercased section ids. -const anchorHref = (href) => /^#/.test(href || "") ? href.toLowerCase() : href; - -function jumpNavBlock($nav) { - const links = []; - $nav.find(".desktop-view-secondary-nav nav a, nav a").each((_, a) => { - const $a = $(a); - links.push(`<li><a href="${anchorHref(https($a.attr("href")))}">${$a.text().trim()}</a></li>`); - if (links.length >= 8) return false; - }); - // de-dupe (mobile + desktop repeat the same links) - const uniq = [...new Map(links.map((l) => [l, l])).keys()]; - const btn = $nav.find("a.button").first(); - const btnHtml = btn.length - ? `<p><a href="${anchorHref(https(btn.attr("href")))}"><strong>${btn.text().trim()}</strong></a></p>` : ""; - return `<div class="jump-nav"><div><div><ul>${uniq.join("")}</ul>${btnHtml}</div></div></div>`; -} - -// Clean an rte/.cmp-text box: <b>-><strong>, <i>-><em>, drop empty <p>, normalise. -function cleanRte($box) { - if (!$box || !$box.length) return ""; - $box.find("b").each((_, b) => (b.tagName = "strong")); - $box.find("i").each((_, i) => (i.tagName = "em")); - $box.find("p").each((_, p) => { - const $p = $(p); - if ($p.text().replace(/ /g, " ").trim() === "" && !$p.find("img").length) $p.remove(); - }); - return ($box.html() || "") - .replace(/ | /g, " ") - .replace(/>\s+</g, "><") - .replace(/[ \t\r\n]+/g, " ") - .trim(); -} - -// accordion -> two-column block: summary (left) / details (right) per item -function accordionBlock($acc) { - const rows = []; - $acc.find(".accordion__item, .cmp-accordion__item").each((_, it) => { - const $it = $(it); - const summary = $it.find(".accordion__heading, .cmp-accordion__title") - .first().clone().children().remove().end().text().replace(/\s+/g, " ").trim(); - const $body = $it.find(".accordion__body, .cmp-accordion__panel").first(); - const details = cleanRte($body.find(".cmp-text").first()) || cleanRte($body); - rows.push(`<div><div>${summary}</div><div>${details}</div></div>`); - }); - return `<div class="accordion">${rows.join("")}</div>`; -} - -// Extract the fields of a single list item. -function listItem($it) { - const name = $it.find(".list__name").first().text().replace(/\s+/g, " ").trim() - || $it.find("a").first().text().replace(/\s+/g, " ").trim(); - const desc = $it.find(".list__item-description, .list__description").first() - .text().replace(/\s+/g, " ").trim(); - const href = https($it.find("a[href]").first().attr("href") || ""); - const img = $it.find("img").attr("src") || ""; - return { name, desc, href, img }; -} - -// list--links/-simple/-detailed/-simple-product -> <div class="list <variant>"> -// Cards model (image left / text right) when any item has an image; otherwise a -// single cell holding the list of links. -function listBlock($list, variant) { - const items = $list.find("li.list__item, .list__item").toArray() - .map((it) => listItem($(it))) - .filter((x) => x.name || x.href || x.img); // drop empty items - const hasImages = items.some((x) => x.img); - - if (hasImages) { - const rows = items.map((x) => { - const parts = [`<h3>${x.name}</h3>`]; - if (x.desc) parts.push(`<p>${x.desc}</p>`); - if (x.href) parts.push(`<p><a href="${x.href}">Learn More</a></p>`); - return `<div><div>${x.img ? pic(x.img, x.name) : ""}</div><div>${parts.join("")}</div></div>`; - }); - return `<div class="list ${variant}">${rows.join("")}</div>`; - } - const lis = items.map((x) => - x.href ? `<li><a href="${x.href}">${x.name}</a></li>` : `<li>${x.name}</li>` - ); - return `<div class="list ${variant}"><div><div><ul>${lis.join("")}</ul></div></div></div>`; -} - -// tabs -> two-column block: tab label (left) / panel content (right) per tab -function tabsBlock($tabs) { - const labels = $tabs.find(".cmp-tabs__tab, .tabs__tab, [role=tab]"); - const panels = $tabs.find(".cmp-tabs__tabpanel, .tabs__tabpanel, [role=tabpanel]"); - const rows = []; - labels.each((i, t) => { - const label = $(t).text().replace(/\s+/g, " ").trim(); - const $panel = $(panels[i]); - const content = cleanRte($panel.find(".cmp-text").first()) || cleanRte($panel); - rows.push(`<div><div>${label}</div><div>${content}</div></div>`); - }); - return `<div class="tabs">${rows.join("")}</div>`; -} - -// default content from a title / texteditor component -function defaultContent($el, type) { - if (type === "title") { - const hd = $el.find("h1,h2,h3,h4,h5,h6").first(); - if (!hd.length) return ""; - const txt = hd.text().trim(); - if (!txt) return ""; - return `<${hd[0].tagName}>${txt}</${hd[0].tagName}>`; - } - // texteditor / text -> its rte paragraphs - const box = $el.find(".cmp-text").first(); - if (!box.length) return ""; - box.find("b").each((_, b) => (b.tagName = "strong")); - box.find("i").each((_, i) => (i.tagName = "em")); - box.find("p").each((_, p) => { - const $p = $(p); - if ($p.text().replace(/ /g, " ").trim() === "" && !$p.find("img").length) $p.remove(); - }); - let html = (box.html() || "").replace(/ | /g, " ").replace(/>\s+</g, "><").replace(/[ \t\r\n]+/g, " ").trim(); - return html; -} - -// "progress-rail" tag blob (Progress Rail^Category^Facet^Value | ...) -> one -// metadata row per facet, multi-values comma-joined. Entity-encoded values -// (e.g. ">30 mT") are kept as-is, which is valid inside <p> text. -function progressRailFacetRows() { - const raw = $('meta[name="progress-rail"]').attr("content") || ""; - if (!raw) return ""; - const order = [], vals = {}; - for (const entry of raw.split("|")) { - const parts = entry.split("^").map((x) => x.trim()).filter(Boolean); - if (parts.length < 4) continue; - const facet = parts[2], value = parts.slice(3).join("^"); - if (!vals[facet]) { vals[facet] = []; order.push(facet); } - if (!vals[facet].includes(value)) vals[facet].push(value); - } - return order.map((f) => - `<div><div><p>${f}</p></div><div><p>${vals[f].join(", ")}</p></div></div>`).join(""); -} - -function metadataBlock() { - // Prefer <title> over og:title, and drop the "ProgressRail | " site-name prefix. - let title = ($("title").text() || $('meta[property="og:title"]').attr("content") || "").trim(); - const bar = title.indexOf("|"); - if (bar > -1) title = title.slice(bar + 1).trim(); - // Prefer og:description; fall back to the (often longer) name=description. - const desc = ($('meta[property="og:description"]').attr("content") - || $('meta[name="description"]').attr("content") || "").trim(); - // og:image -> Image metadata (the curated social/preview image). - const ogImage = ($('meta[property="og:image"]').attr("content") || "").trim(); - const imageRow = ogImage - ? `<div><div><p>Image</p></div><div>${pic(ogImage, title)}</div></div>` : ""; - return ( - `<div class="metadata">` + - `<div><div><p>Title</p></div><div><p>${title}</p></div></div>` + - `<div><div><p>Description</p></div><div><p>${desc}</p></div></div>` + - imageRow + - progressRailFacetRows() + - `</div>` - ); -} - -// text cell shared by banner / columns / carousel (heading + paragraphs + CTA) -function bannerCell(h, paras, cta) { - const parts = [`<h2>${h}</h2>`, parasHtml(paras)]; - if (cta && cta.text) parts.push(`<p><a href="${cta.href}"><strong>${cta.text}</strong></a></p>`); - return parts.join(""); -} - -// teaser--checkerboard run -> columns block. -// Source DOM is always text-then-image; teaser--right flips the image to the -// LEFT (text right). EDS renders cells left-to-right, so order them accordingly. -function columnsRow($t) { - const h = $t.find(".teaserHeading").first().text().trim(); - const cta = teaserCTA($t); - const { src, alt } = firstImg($t); - const textCell = `<div>${bannerCell(h, teaserParas($t), cta)}</div>`; - const imgCell = `<div>${pic(src, alt)}</div>`; - const right = ($t.attr("class") || "").split(/\s+/).includes("teaser--right"); - return right ? `<div>${imgCell}${textCell}</div>` : `<div>${textCell}${imgCell}</div>`; -} -const columnsBlock = (teasers) => `<div class="columns">${teasers.map(columnsRow).join("")}</div>`; - -// carousel--promo -> carousel block; each slide laid out like a banner row -function carouselBlock($car) { - const rows = []; - $car.find(".cmp-carousel__item, .carousel__item").each((_, it) => { - const $it = $(it); - const h = $it.find(".teaserHeading, h1, h2, h3").first().text().trim(); - const cta = teaserCTA($it); - const { src, alt } = firstImg($it); - rows.push(`<div><div>${bannerCell(h, teaserParas($it), cta)}</div><div>${pic(src, alt)}</div></div>`); - }); - return `<div class="carousel">${rows.join("")}</div>`; -} - -// teaser--banner -> columns block (heading+text | CTA) for a dedicated dark section -function bannerSectionContent($t) { - const h = $t.find(".teaserHeading").first().text().trim(); - const seen = new Set(); - const ps = []; - $t.find(".teaser__text-wrap p, .teaser-blog-content").each((_, p) => { - const t = $(p).text().replace(/\s+/g, " ").trim(); - if (t && !/^\d{4}-\d{2}-\d{2}$/.test(t) && !seen.has(t)) { seen.add(t); ps.push(t); } - }); - const cta = teaserCTA($t); - const textCol = [`<h2>${h}</h2>`, ...ps.map((p) => `<p>${p}</p>`)].join(""); - if (cta && cta.text) { - return `<div class="columns"><div><div>${textCol}</div><div><p><a href="${cta.href}">${cta.text}</a></p></div></div></div>`; - } - return textCol; -} - -// multimedia / media-youtube slider -> default content: -// image slide -> a picture; video slide -> a plain YouTube link (EDS embeds it). -// Caption text of a multimedia slide (class is sometimes misspelled "mutlimedia"). -function slideCaption($s) { - return $s.find("[class*=imedia__description]").first().text().replace(/\s+/g, " ").trim(); -} - -// Collect a multimedia component's slides (images with captions, or videos) in -// document order, then render: more than one slide -> a carousel (slides) block -// (image or YouTube link per slide); a single image -> picture + caption; a -// single video -> a plain YouTube link. -function mediaContent($mm, posterVideos) { - const slides = $mm.find(".multimedia__slide"); - const list = slides.length ? slides.toArray() : [$mm[0]]; - const items = []; - const seen = new Set(); - for (const s of list) { - const $s = $(s); - let ytid = $s.find("[data-ytvideoid]").attr("data-ytvideoid") - || $s.find("[data-videoid]").attr("data-videoid"); - if (!ytid) { - const thumb = $s.find("img").map((i, im) => $(im).attr("src")).get() - .find((x) => /(?:youtube\.com\/vi|ytimg\.com\/vi)\//.test(x || "")); - const m = thumb && thumb.match(/\/vi\/([^/]+)\//); - if (m) ytid = m[1]; - } - if (ytid) { - if (!seen.has("v:" + ytid)) { - seen.add("v:" + ytid); - // a custom poster is stored as data-imageid (not the youtube auto-thumb) - let poster = ($s.find("[data-imageid]").attr("data-imageid") || "").trim(); - if (!/^(?:https?:)?\/\//.test(poster) || /youtube|ytimg/.test(poster)) poster = ""; - items.push({ video: ytid, poster, cap: slideCaption($s) }); - } - continue; - } - const src = $s.find("img").map((i, im) => $(im).attr("src")).get() - .find((x) => x && !/[?&](cc-s|fmt=)/.test(x) && !/youtube|ytimg/.test(x)); - if (src && !seen.has("i:" + src)) { - seen.add("i:" + src); - items.push({ src, alt: $s.find("img").first().attr("alt") || "", cap: slideCaption($s) }); - } - } - const render = (it, inCarousel) => { - if (it.video) { - // poster: a custom image if set, else (in a carousel) the YouTube thumbnail - // (maxres), which gives slides something to show while the embed loads. - const poster = it.poster || ((inCarousel || posterVideos) ? `https://i.ytimg.com/vi/${it.video}/maxresdefault.jpg` : ""); - return `${poster ? pic(poster) : ""}<p><a href="https://www.youtube.com/watch?v=${it.video}">https://www.youtube.com/watch?v=${it.video}</a></p>${it.cap ? `<p>${it.cap}</p>` : ""}`; - } - return `${pic(it.src, it.alt)}${it.cap ? `<p>${it.cap}</p>` : ""}`; - }; - if (items.length > 1) { - // gallery (images and/or videos) -> carousel (slides), one slide per row - return `<div class="carousel slides">${items.map((it) => `<div><div>${render(it, true)}</div></div>`).join("")}</div>`; - } - return items.length ? render(items[0], false) : ""; -} - -// ---- walk top-level components in document order ---- - -const SELS = [".teaser", ".list", ".secondary-navigation", ".title", ".text", - ".accordion", ".tabs", ".carousel", ".multimedia"]; -const selJoin = SELS.join(","); - -function typeOf($el) { - const cls = ($el.attr("class") || "").split(/\s+/); - if (cls.includes("teaser")) { - if (cls.includes("teaser--expired")) return "skip"; - if (cls.some((c) => c === "teaser--hero")) return "hero"; - if (cls.some((c) => c === "teaser--full-width")) return "banner"; - if (cls.some((c) => c === "teaser--checkerboard")) return "columns"; - if (cls.some((c) => c === "teaser--tile")) return "tile"; - if (cls.some((c) => c === "teaser--banner")) return "banner-section"; - return "tile"; // bare/unknown teaser -> cards (horizontal), image left / text right - } - if (cls.includes("list")) { - if (cls.includes("list--content")) return "list-cards"; - if (cls.includes("list--links")) return "list-links"; // covers links & links-simple - if (cls.includes("list--detailed")) return "list-detailed"; - if (cls.includes("list--simple-product")) return "list-product"; - return "list?"; - } - if (cls.includes("secondary-navigation")) return "jump-nav"; - if (cls.includes("title")) return "title"; - if (cls.includes("text")) return "text"; - if (cls.includes("accordion")) return "accordion"; - if (cls.includes("tabs")) return "tabs"; - if (cls.includes("carousel")) return "carousel"; - if (cls.includes("multimedia")) return "media"; - return "unknown"; -} - -// Build the EDS document string from the currently-loaded `$`. -// The image src of an image-only multimedia component (null if it's a video). -function mediaImageSrc($mm) { - const hasVideo = $mm.find("[data-ytvideoid],[data-videoid]").length - || $mm.find("img").toArray().some((im) => /youtube|ytimg/.test($(im).attr("src") || "")); - if (hasVideo) return null; - return $mm.find("img").map((i, im) => $(im).attr("src")).get() - .find((x) => x && !/[?&](cc-s|fmt=)/.test(x) && !/youtube|ytimg/.test(x)) || null; -} - -// AEM grid width (1-12) of a component, or 12 if unsized. -function gridWidth($el) { - const $g = $el.is("[class*='aem-GridColumn--default--']") ? $el - : $el.closest("[class*='aem-GridColumn--default--']"); - const m = ($g.attr("class") || "").match(/aem-GridColumn--default--(\d+)/); - return m ? +m[1] : 12; -} - -// An image-multimedia paired with an adjacent sized text becomes a columns block: -// image side = DOM order (image-first -> left, text-first -> right) -// variant = equal widths -> regular columns; unequal -> columns (portrait) -function columnsPairBlock($img, $text, imgFirst, variant) { - const src = mediaImageSrc($img); - const alt = $img.find("img").first().attr("alt") || ""; - const imgCell = `<div>${pic(src, alt)}</div>`; - const textCell = `<div>${cleanRte($text.find(".cmp-text").first()) || cleanRte($text)}</div>`; - const cls = variant === "portrait" ? "columns portrait" : "columns"; - const row = imgFirst ? `${imgCell}${textCell}` : `${textCell}${imgCell}`; - return `<div class="${cls}"><div>${row}</div></div>`; -} - -// ---- Fragments ---------------------------------------------------------- -// Reusable blocks that repeat across many pages are extracted to a single -// fragment page and replaced inline with a plain link to it. Add rules here -// to identify more fragments/widgets across the site. -const listTargets = ($L) => $L.find("li.list__item") - .map((i, it) => $(it).find("#linkPagePath,#anchorValue").attr("value") - || $(it).find("a[href]").attr("href") || "").get().filter(Boolean); - -const FRAGMENT_HOST = "https://main--progressrail--aemsites.aem.page"; -const FRAGMENT_RULES = [ - { - // Fragment: generated, language-aware page; inline reference replaced by a link. - name: "leadership-execs", - path: (lang) => `${lang}/company/leadership/fragments/leadership`, - match(item) { - if (item.type !== "list-cards") return false; - const t = listTargets(item.$el); - return t.length >= 3 && t.every((x) => /\/Leadership\/[^/]+\.html$/i.test(x)); - }, - build(item) { return listCardsBlock(item.$el); }, - }, - { - // Widget: external page (not generated here); just link to its fixed URL. - name: "press-releases", - widget: "https://main--progressrail--aemsites.aem.page/widgets/press-releases/press-releases.html", - match(item) { - if (!/^list-/.test(item.type)) return false; - const t = listTargets(item.$el); - return t.length >= 3 && t.every((x) => /\/PressReleases\/[^/]+\.html$/i.test(x)); - }, - }, - { - // Widget: a "list children" component that renders populated facet filters is - // a product-listing page (PLP). Identified by rendered .filter__name options - // (the inert .list__filter-col wrapper is present on every list, so isn't the - // signal). Links to the PLP widget; no root param needed. - name: "plp", - widget: "https://main--progressrail--aemsites.aem.page/widgets/plp/plp.html", - match(item) { - if (!/^list/.test(item.type || "")) return false; - const $L = item.$el; - if (($L.find("#listFormValue").attr("value") || "").toLowerCase() !== "children") return false; - return $L.find(".filter__name").length > 0; - }, - }, - { - // Widget: any AEM "list children" component -> the card-list widget, with a - // stable, language-relative `root` query param: - // (no param) -> children of self - // ?root=parent -> children of the parent (siblings) - // ?root=/seg/... -> children of an explicit page (relative to lang root) - // ?root=unresolved -> couldn't infer from the crawl (revisit later) - name: "card-list", - match(item) { return cardListWidgetUrl(item) !== null; }, - widgetUrl(item) { return cardListWidgetUrl(item); }, - }, -]; -const CARD_LIST_WIDGET = "https://main--progressrail--aemsites.aem.page/widgets/card-list/card-list.html"; - -// For an AEM list with listFormValue=children, work out the widget URL + root -// param. Returns null if the list isn't a "children" list (leave it inline). -function cardListWidgetUrl(item) { - if (!/^list/.test(item.type || "")) return null; - const $L = item.$el; - if (($L.find("#listFormValue").attr("value") || "").toLowerCase() !== "children") return null; - const dir = (p) => p.replace(/\/[^/]*$/, ""); - const hrefs = $L.find("li.list__item a[href]").map((i, a) => $(a).attr("href") || "").get() - .filter((x) => /^\/(en|fr)\/[^"#?]+\.html?$/i.test(x)); - let root; // undefined => self (no param) - if (!hrefs.length) root = "unresolved"; - else { - const dirs = [...new Set(hrefs.map(dir))]; - const itemdir = dirs.length === 1 ? dirs[0] : null; // null => mixed/curated - if (!itemdir) root = "unresolved"; - else if (itemdir === currentPageAbs.replace(/\.html?$/i, "")) root = undefined; // self - else if (itemdir === dir(currentPageAbs)) root = "parent"; // siblings - else root = ("/" + kebabPath(itemdir.replace(/^\//, ""))).replace(/^\/(en|fr)/, ""); // explicit - } - const params = []; - if (root !== undefined) params.push(`root=${root}`); - // Title-only tiles (default is cards-with-text) when the list is the simple - // variant OR explicitly hides descriptions. - const cls = ($L.attr("class") || "").split(/\s+/); - const noDesc = cls.includes("list--simple") - || ($L.find("#showDescription").attr("value") || "").toLowerCase() === "false"; - if (noDesc) params.push("description=false"); - return params.length ? `${CARD_LIST_WIDGET}?${params.join("&")}` : CARD_LIST_WIDGET; -} -let currentLang = "en"; // set per file by transformFile() -let currentPageAbs = "/en/index.html"; // source path of the page being transformed -// Captured across the whole batch (module-level so runBatch can write the pages). -const fragments = {}; // outPath -> { content, url, name } -const fragmentUsage = {}; // url -> number of pages that referenced it - -function matchFragment(item) { - const rule = FRAGMENT_RULES.find((r) => r.match(item)); - if (!rule) return null; - if (rule.widgetUrl) { // widget with a dynamically-built URL - const url = rule.widgetUrl(item); - if (!url) return null; - fragmentUsage[url] = (fragmentUsage[url] || 0) + 1; - return `<p><a href="${url}">${url}</a></p>`; - } - if (rule.widget) { - fragmentUsage[rule.widget] = (fragmentUsage[rule.widget] || 0) + 1; - return `<p><a href="${rule.widget}">${rule.widget}</a></p>`; - } - const rel = rule.path(currentLang); - const url = `${FRAGMENT_HOST}/${rel}`; - const outPath = `${rel}.html`; - if (!fragments[outPath]) fragments[outPath] = { content: rule.build(item), url, name: rule.name }; - fragmentUsage[url] = (fragmentUsage[url] || 0) + 1; - return `<p><a href="${url}">${url}</a></p>`; -} - -// Pre-pass: normalize AEM media / JS embeds anywhere in the document so they -// don't leak raw widget markup when nested inside another block (tabs, accordion, -// columns ...), where the main walker never reaches them. -function cleanEmbeds() { - // Stray clientlib stylesheets AEM injects alongside media widgets. - $('link[rel="stylesheet"]').remove(); - // Cookie-consent placeholders are not real content. - $(".cookie-warning, .multimedia-cookie-warning").remove(); - // jsComponent: an <iframe> stuffed into a data-js attribute -> a plain link. - $(".jsComponent").each((_, el) => { - const $c = $(el); - const dj = $c.find("[data-js]").attr("data-js") || $c.attr("data-js") || ""; - const m = dj.match(/src\s*=\s*(?:"|"|"|')(.*?)(?:"|"|"|'|>)/i); - if (m && m[1].trim()) { - const url = m[1].replace(/&/g, "&").trim(); - $c.replaceWith(`<p><a href="${url}">${url}</a></p>`); - } else $c.remove(); - }); - // experienceFragment + AEM grid scaffolding nested in a block panel is pure - // layout -> unwrap to its inner content. (Scoped to accordion/tabs panels so - // the top-level grid-width detection used by the columns pre-pass is untouched.) - $(".experiencefragment").toArray().forEach((el) => { - const $xf = $(el); - if (!$xf.closest(".accordion, .cmp-accordion, .tabs, .cmp-tabs").length) return; - $xf.find(".aem-Grid, .aem-GridColumn, .xf-content-height, .container, .row, [class^='col-'], [class*=' col-']") - .toArray().forEach((w) => { const $w = $(w); $w.replaceWith($w.contents()); }); - $xf.replaceWith($xf.contents()); - }); - // Teasers nested inside another block (e.g. a tab/accordion panel) can't render - // as their own block in a cell -> flatten to heading + text + media + CTA. - // (Top-level teasers are handled by the main walker.) - $(".teaser").each((_, el) => { - const $t = $(el); - if (!$t.parents(selJoin).length) return; - if ($t.closest(".carousel").length) return; // carouselBlock renders its own teasers - const parts = []; - const h = $t.find(".teaserHeading").first().text().trim(); - if (h) parts.push(`<h3>${h}</h3>`); - teaserParas($t).forEach((p) => parts.push(`<p>${p}</p>`)); - const yt = $t.find("[data-ytvideoid]").attr("data-ytvideoid") - || $t.find("[data-videoid]").attr("data-videoid"); - if (yt) parts.push(`<p><a href="https://www.youtube.com/watch?v=${yt}">https://www.youtube.com/watch?v=${yt}</a></p>`); - else { const { src, alt } = firstImg($t); if (src) parts.push(pic(src, alt)); } - const cta = teaserCTA($t); - if (cta && cta.text) parts.push(`<p><a href="${cta.href}">${cta.text}</a></p>`); - $t.replaceWith(parts.join("")); - }); - // Multimedia nested inside another block -> its mediaContent rendering - // (video-only => YouTube link). Top-level multimedia is left for the walker. - $(".multimedia").each((_, el) => { - const $mm = $(el); - if (!$mm.parents(selJoin).length) return; - const repl = mediaContent($mm, true) || ""; // nested videos get a poster, like carousel slides - const $wrap = $mm.closest(".media-youtube"); - ($wrap.length ? $wrap : $mm).replaceWith(repl); - }); -} - -function transformDoc() { -cleanEmbeds(); - -// In-page anchors referenced by a jump-nav: each becomes a new section whose -// section-metadata carries an `id`, so the jump-nav links resolve in EDS. -const navIds = new Set(); -$(".secondary-navigation a[href^='#'], .jump-nav a[href^='#']").each((_, a) => { - const id = (($(a).attr("href") || "").slice(1)).trim().toLowerCase(); // ids are lowercased - if (id && id !== "top" && id !== "mainContent") navIds.add(id); -}); -const anchorOf = new Map(); // top-level component element -> id that starts its section -if (navIds.size) { - let pending = null; - $("*").each((_, el) => { - const $el = $(el); - const id = ($el.attr("id") || "").toLowerCase(); - if (id && navIds.has(id)) { pending = id; return; } // an anchor marker (id lowercased) - if (pending && $el.is(selJoin) && !$el.parents(selJoin).length) { - anchorOf.set(el, pending); pending = null; // the next top-level component - } - }); -} - -const raw = []; -$(selJoin).each((_, el) => { - const $el = $(el); - if ($el.parents(selJoin).length) return; // top-level only - raw.push({ $el, type: typeOf($el), anchorId: anchorOf.get(el) }); -}); - -// Pre-pass: a sized image-multimedia next to a sized text is a columns pair. -// (Full-width images stay as standalone default-content pictures.) -const comps = []; -const consumed = new Set(); -for (let i = 0; i < raw.length; i++) { - if (consumed.has(i)) continue; - const c = raw[i]; - if (c.type === "media" && mediaImageSrc(c.$el)) { - const wImg = gridWidth(c.$el); - if (wImg < 12) { - const next = raw[i + 1], prev = raw[i - 1]; - // image first -> image on the left - if (next && next.type === "text" && gridWidth(next.$el) < 12 && !consumed.has(i + 1)) { - const wT = gridWidth(next.$el); - comps.push({ type: "columns-pair", $img: c.$el, $text: next.$el, imgFirst: true, variant: wImg === wT ? "regular" : "portrait", anchorId: c.anchorId }); - consumed.add(i + 1); - continue; - } - // text first -> image on the right (replace the already-pushed text) - if (prev && prev.type === "text" && gridWidth(prev.$el) < 12 && comps.length && comps[comps.length - 1] === prev) { - const wT = gridWidth(prev.$el); - comps[comps.length - 1] = { type: "columns-pair", $img: c.$el, $text: prev.$el, imgFirst: false, variant: wImg === wT ? "regular" : "portrait", anchorId: prev.anchorId }; - continue; - } - } - } - comps.push(c); -} - -// Sections: each is { items: html[], style, id }; main renders one <div> per section. -const newSection = () => ({ items: [], style: null, id: null }); -const sections = [newSection()]; -const cur = () => sections[sections.length - 1]; -let heroSectioned = false; // section break after the first hero - -// Group consecutive same-type components (tiles -> cards, checkerboard -> columns). -let run = { type: null, items: [] }; -function flushRun() { - if (!run.items.length) return; - if (run.type === "tile") cur().items.push(cardsHorizontal(run.items)); - else if (run.type === "columns") cur().items.push(columnsBlock(run.items)); - run = { type: null, items: [] }; -} - -for (const item of comps) { - const { $el, type } = item; - // An in-page anchor referenced by the jump-nav starts a new section carrying its id. - if (item.anchorId) { - flushRun(); - if (cur().items.length || cur().id) sections.push(newSection()); - cur().id = item.anchorId; - } - if (type === "tile" || type === "columns") { - if (run.type && run.type !== type) flushRun(); - run.type = type; - run.items.push($el); - continue; - } - flushRun(); - const fragLink = matchFragment(item); - if (fragLink) { cur().items.push(fragLink); continue; } - switch (type) { - case "hero": - case "banner": // teaser--full-width also renders as a hero block - cur().items.push(type === "hero" ? heroBlock($el) : bannerBlock($el)); - if (!heroSectioned) { heroSectioned = true; sections.push(newSection()); } // break after first hero - break; - case "carousel": cur().items.push(carouselBlock($el)); break; - case "accordion": cur().items.push(accordionBlock($el)); break; - case "tabs": cur().items.push(tabsBlock($el)); break; - case "list-cards": cur().items.push(listCardsBlock($el)); break; - case "list-links": cur().items.push(listBlock($el, "links")); break; - case "list-detailed": cur().items.push(listBlock($el, "detailed")); break; - case "list-product": cur().items.push(listBlock($el, "product")); break; - case "jump-nav": // jump-nav always stands alone in its own section - if (cur().items.length || cur().id || cur().style) sections.push(newSection()); - cur().items.push(jumpNavBlock($el)); - sections.push(newSection()); - break; - case "title": { const c = defaultContent($el, "title"); if (c) cur().items.push(c); break; } - case "text": { const c = defaultContent($el, "text"); if (c) cur().items.push(c); break; } - case "media": { const c = mediaContent($el); if (c) cur().items.push(c); break; } - case "columns-pair": cur().items.push(columnsPairBlock(item.$img, item.$text, item.imgFirst, item.variant)); break; - case "banner-section": - // teaser--banner becomes its own dark section, then content resumes after. - { const s = newSection(); s.items.push(bannerSectionContent($el)); s.style = "dark"; sections.push(s); } - sections.push(newSection()); - break; - case "skip": break; - default: cur().items.push(`<!-- UNMAPPED component: ${type} -->`); - } -} -flushRun(); -cur().items.push(metadataBlock()); - -// section-metadata combining any Style + id this section carries. -function sectionMeta(s) { - const rows = []; - if (s.style) rows.push(`<div><div>Style</div><div>${s.style}</div></div>`); - if (s.id) rows.push(`<div><div>id</div><div>${s.id}</div></div>`); - return rows.length ? `<div class="section-metadata">${rows.join("")}</div>` : ""; -} -let mainInner = sections - .filter((s) => s.items.length || s.id || s.style) - .map((s) => `<div>${s.items.join("")}${sectionMeta(s)}</div>`) - .join(""); - -// --- General cleanup: strip empty rows from every block --- -// A row (direct child of a block) is "empty" when it has no visible text, no -// image with a real src, no <source srcset>, and no links. -function stripEmptyRows(html) { - const $$ = cheerio.load(html, { decodeEntities: false }); - const BLOCKS = ["hero", "banner", "cards", "columns", "carousel", "accordion", "tabs", "list", "jump-nav", "metadata", "section-metadata", "table"]; - - // Free-standing data tables -> a `table` block (row/cell divs). EDS treats a - // top-level table as a block definition, so a raw data table would be mis-parsed. - // Tables NESTED in another block are left as raw <table> (a block can't nest). - const BLOCK_SEL = "." + BLOCKS.join(",."); - $$("table").toArray().forEach((tbl) => { - const $t = $$(tbl); - if ($t.parents(BLOCK_SEL).length) return; // nested in a block -> leave untouched - const rows = $t.find("tr").toArray().map((tr) => { - const cells = $$(tr).children("td,th").toArray().map((td) => { - const $td = $$(td); - $td.find("b").toArray().forEach((b) => (b.tagName = "strong")); - $td.find("i").toArray().forEach((i) => (i.tagName = "em")); - $td.find("h1,h2,h3,h4,h5,h6").toArray().forEach((hd) => { const $h = $$(hd); $h.replaceWith($h.contents()); }); - $td.find("br").remove(); - const inner = ($td.html() || "").replace(/ | /g, " ").replace(/\s+/g, " ").trim(); - return `<div>${inner}</div>`; - }); - return `<div>${cells.join("")}</div>`; - }); - $t.replaceWith(`<div class="table">${rows.join("")}</div>`); - }); - const isEmpty = ($el) => { - if ($el.text().replace(/\s| /g, "").length) return false; - if ($el.find("img").toArray().some((i) => ($$(i).attr("src") || "").trim())) return false; - if ($el.find("source").toArray().some((s) => ($$(s).attr("srcset") || "").trim())) return false; - if ($el.find("a").toArray().some((a) => ($$(a).attr("href") || "").trim())) return false; - return true; - }; - $$("body > *, body").each(() => {}); // noop to ensure body context - BLOCKS.forEach((b) => { - $$("." + b).each((_, blk) => { - $$(blk).children("div").each((_, row) => { - if (isEmpty($$(row))) $$(row).remove(); - }); - }); - }); - // A block cell holding loose text / inline content must wrap it in a <p> - // (EDS/DA convention: cell text is never bare). Cells already carrying a - // block-level child (p, heading, list, picture, ...) are left untouched. - const BLOCK_CHILD = "p,h1,h2,h3,h4,h5,h6,ul,ol,table,picture,img,figure,blockquote,div"; - BLOCKS.forEach((b) => { - $$("." + b).each((_, blk) => { - $$(blk).children("div").each((_, row) => { - $$(row).children("div").each((_, cell) => { - const $c = $$(cell); - if (!$c.text().replace(/\s| /g, "").length && !$c.children("img,picture").length) return; - if ($c.children(BLOCK_CHILD).length) return; // already block-wrapped - $c.html(`<p>${$c.html()}</p>`); - }); - }); - }); - }); - // cheerio wraps fragments in <html><body>; return inner body. - // DA strips redundant <strong> inside headings — match that (unwrap them). - // unwrap <strong> inside headings (DA strips it) — snapshot to survive mutation - $$("h1,h2,h3,h4,h5,h6").find("strong").toArray().forEach((s) => { const $s = $$(s); $s.replaceWith($s.contents()); }); - // presentational <span class="..."> shouldn't pass through to EDS content — unwrap them - $$("span[class]").toArray().forEach((s) => { const $s = $$(s); $s.replaceWith($s.contents()); }); - $$("a[target]").removeAttr("target"); // drop target attributes from links - // Lowercase every in-page anchor href (#Foo -> #foo) to match lowercased section ids. - $$("a[href^='#']").each((_, a) => { - const $a = $$(a), h = $a.attr("href") || ""; - if (/[A-Z]/.test(h)) $a.attr("href", h.toLowerCase()); - }); - // Remove "back to top" links (EDS auto-blocks #top anchors). Leave the jump-nav. - $$("a").toArray().forEach((a) => { - const $a = $$(a); - if ($a.closest(".jump-nav").length) return; - const href = ($a.attr("href") || "").trim().toLowerCase(); - const txt = $a.text().replace(/\s+/g, " ").trim().toLowerCase(); - if (href !== "#top" && txt !== "back to top") return; - const $w = $a.closest("h1,h2,h3,h4,h5,h6,p"); - if ($w.length && $w.text().replace(/\s+/g, " ").trim().toLowerCase() === txt) $w.remove(); - else $a.remove(); - }); - return $$("body").html(); -} -mainInner = stripEmptyRows(mainInner); - -const doc = - "\n<body>\n <header></header>\n <main>" + - mainInner + - "</main>\n <footer></footer>\n</body>\n"; - - return daSerialize(doc); -} - -// Load one source file and return its transformed EDS document. -function transformFile(srcFile, lang) { - currentLang = lang || (/^\/?fr(\/|$|\.)/.test(srcFile.split(/[\\/]/).slice(-3).join("/")) ? "fr" : "en"); - const mm = srcFile.replace(/\\/g, "/").match(/\/((?:en|fr)\/.*\.html?)$/i); - currentPageAbs = mm ? "/" + mm[1] : `/${currentLang}/index.html`; - $ = cheerio.load(fs.readFileSync(srcFile, "utf8"), { decodeEntities: false }); - return transformDoc(); -} - -// --------------------------------------------------------------------------- -// Batch driver: transform a whole cleaned site into the content repo and -// record old -> new URL redirects. -// -// node transform-page.js <sourceRoot> <contentRoot> -// -// <sourceRoot> cleaned site root containing en.html, fr.html, en/, fr/ -// (e.g. .../downloads/www.progressrail.com/cleaned) -// <contentRoot> the EDS content repo to write into -// (e.g. .../content/aemsites/progressrail) -// -// Section homepages (<lang>.html) become <lang>/index.html. Individual press -// releases are skipped (they use a dedicated transformer) but their existing -// redirects are preserved. -// --------------------------------------------------------------------------- - -function walk(dir, acc = []) { - for (const e of fs.readdirSync(dir, { withFileTypes: true })) { - const f = path.join(dir, e.name); - if (e.isDirectory()) walk(f, acc); - else if (/\.html?$/i.test(e.name)) acc.push(f); - } - return acc; -} -const isPressRelease = (rel) => /\/Company\/News\/PressReleases\/[^/]+\.html$/i.test("/" + rel); - -function runBatch(sourceRoot, contentRoot) { - const SRC_ROOT = path.resolve(sourceRoot); - const REPO = path.resolve(contentRoot); - const redirects = []; - let ok = 0, skipPR = 0, skipProtected = 0, skipExcluded = 0, errs = []; - - // Authored EDS files (homepage, nav/footer fragments, search), the orphaned - // "right-rail" sidebar fragments, and stray test/scratch pages (test-page, - // test-product/*, nishanthi folder) — never real content. Excluded by default; - // --include-special overrides. - const EXCLUDE_RE = /^(?:en|fr)\/(?:index|nav|footer|search)\.html$|\/right-rail\.html$|(?:^|\/)test-(?:page|product)(?:\.html$|\/)/; - const includeSpecial = process.argv.includes("--include-special"); - - // Pages a human edited in DA (from reconcile.js) — never overwrite these. - let protectedSet = new Set(); - if (process.argv.includes("--skip-protected")) { - const pf = path.join(__dirname, "protected-files.json"); // written by reconcile.js, beside the scripts - if (fs.existsSync(pf)) protectedSet = new Set(JSON.parse(fs.readFileSync(pf, "utf8"))); - } - - const handle = (srcFile, isHome, lang) => { - const rel = path.relative(SRC_ROOT, srcFile); - if (!isHome && isPressRelease(rel)) { skipPR++; return; } - try { - const outRel = isHome ? path.join(lang, "index.html") : kebabPath(rel); - const outKey = outRel.replace(/\\/g, "/"); - if (!includeSpecial && EXCLUDE_RE.test(outKey)) { skipExcluded++; return; } // homepage/nav/footer - if (!isHome) { - redirects.push({ Source: "/" + rel, Destination: "/" + kebabPath(rel).replace(/\.html?$/i, "") }); - } - if (protectedSet.has(outRel)) { skipProtected++; return; } // keep the human edit - const doc = transformFile(srcFile, lang); - const out = path.join(REPO, outRel); - fs.mkdirSync(path.dirname(out), { recursive: true }); - fs.writeFileSync(out, doc); - ok++; - } catch (e) { errs.push(rel + " :: " + e.message); } - }; - - for (const lang of ["en", "fr"]) { - const home = path.join(SRC_ROOT, lang + ".html"); - if (fs.existsSync(home)) handle(home, true, lang); - const dir = path.join(SRC_ROOT, lang); - if (fs.existsSync(dir)) for (const f of walk(dir)) handle(f, false, lang); - } - - // Merge redirects with any existing entries (keep press-release/prior rows). - const redirectsPath = path.join(REPO, "redirects.json"); - let sheet = { ":colWidths": [274, 241], ":sheetname": "data", ":type": "sheet" }; - if (fs.existsSync(redirectsPath)) sheet = JSON.parse(fs.readFileSync(redirectsPath, "utf8")); - const existing = sheet.data || []; - const seen = new Set(redirects.map((r) => r.Source)); - // Drop superseded rows, and purge stale entries for now-excluded pages - // (right-rail fragments + test/scratch pages). - const PURGE_RE = /\/right-rail\.html$|(?:^|\/)test-(?:page|product)(?:\.html$|\/)/i; - const kept = existing.filter((r) => !seen.has(r.Source) && !PURGE_RE.test(r.Source)); - const merged = redirects.concat(kept).sort((a, b) => a.Source.localeCompare(b.Source)); - sheet.data = merged; sheet.total = merged.length; sheet.limit = merged.length; sheet.offset = 0; - fs.writeFileSync(redirectsPath, JSON.stringify(sheet)); - - // Write the fragment pages captured during transformation. - for (const [outPath, frag] of Object.entries(fragments)) { - const out = path.join(REPO, outPath); - fs.mkdirSync(path.dirname(out), { recursive: true }); - fs.writeFileSync(out, daSerialize(`\n<body>\n <header></header>\n <main><div>${frag.content}</div></main>\n <footer></footer>\n</body>\n`)); - console.log(`fragment ${frag.name}: ${outPath} (referenced on ${fragmentUsage[frag.url]} pages)`); - } - for (const rule of FRAGMENT_RULES) { - if (rule.widget && fragmentUsage[rule.widget]) { - console.log(`widget ${rule.name}: ${rule.widget} (referenced on ${fragmentUsage[rule.widget]} pages)`); - } - } - - console.log(`transformed: ${ok} | press-releases skipped: ${skipPR} | protected skipped: ${skipProtected} | special skipped: ${skipExcluded} | errors: ${errs.length}`); - console.log(`redirects: ${redirects.length} new + ${kept.length} kept = ${merged.length} total`); - errs.slice(0, 15).forEach((e) => console.log(" ERROR " + e)); -} - -if (require.main === module) { - const [sourceRoot, contentRoot] = process.argv.slice(2); - if (!sourceRoot || !contentRoot) { - console.error("usage: node transform-page.js <sourceRoot> <contentRoot>"); - process.exit(1); - } - runBatch(sourceRoot, contentRoot); -} - -module.exports = { transformFile, kebabPath, runBatch }; diff --git a/tools/import/transform-press-releases.js b/tools/import/transform-press-releases.js deleted file mode 100644 index c3b1eab..0000000 --- a/tools/import/transform-press-releases.js +++ /dev/null @@ -1,229 +0,0 @@ -#!/usr/bin/env node -/** - * transform-press-releases.js - * - * Transforms full Progress Rail press-release pages into the stripped AEM - * document-authoring format: - * - * <body> - * <header></header> - * <main><div><h1><strong>TITLE</strong></h1>...body paragraphs...</div></main> - * <footer></footer> - * </body> - * - * Content rules (derived from the hand-made example): - * - Title comes from <title> ("ProgressRail | Real Title") and is wrapped in - * <h1><strong>…</strong></h1>. - * - Body comes from the single <div class="cmp-text"> block. - * - <b> -> <strong> and <i> -> <em>. - * - Empty paragraphs (<p> </p>, <p></p>) are dropped. - * -   is normalised to a regular space. - * - * Filename: <year>-<slug>.html - * - year : 4-digit year from the dateline in the body. - * - slug : first few words of the title, kebab-cased, trailing stop-words - * trimmed, so names stay short. - * - * Usage: - * node transform-press-releases.js <srcDir> <outDir> [--map mapping.csv] - * - * Reads every *.html under srcDir, writes <year>-<slug>.html into outDir. - */ - -const fs = require("fs"); -const path = require("path"); -const cheerio = require("cheerio"); - -const SRC = path.resolve(process.argv[2]); -const OUT = path.resolve(process.argv[3]); -const mapIdx = process.argv.indexOf("--map"); -const MAP = mapIdx > -1 ? path.resolve(process.argv[mapIdx + 1]) : null; - -const STOPWORDS = new Set([ - "to", "and", "of", "the", "for", "a", "an", "with", "in", "on", "at", - "et", "de", "la", "le", "les", "des", "du", "pour", "un", "une", "au", -]); -const MAX_SLUG_WORDS = 6; // hard ceiling on words considered -const SLUG_CHAR_BUDGET = 34; // keep names short - -function slugify(title) { - const words = title - .toLowerCase() - .replace(/œ/g, "oe").replace(/æ/g, "ae") // expand ligatures to ASCII - .replace(/[‘’']/g, "") // drop apostrophes - .replace(/[^a-z0-9À-ſ]+/gi, " ") // keep letters (incl. accents) & digits - .trim() - .split(/\s+/) - .filter(Boolean); - - const picked = []; - let chars = 0; - for (const w of words) { - if (picked.length >= MAX_SLUG_WORDS) break; - if (picked.length >= 3 && chars + 1 + w.length > SLUG_CHAR_BUDGET) break; - picked.push(w); - chars += w.length + 1; - } - // Trim trailing stop-words so slugs don't end on "to"/"and"/etc. - while (picked.length > 3 && STOPWORDS.has(picked[picked.length - 1])) { - picked.pop(); - } - return picked - .join("-") - .normalize("NFD").replace(/[̀-ͯ]/g, ""); // strip accents for ascii filename -} - -function extractTitle($) { - let t = ($("title").first().text() || "").trim(); - const bar = t.indexOf("|"); - if (bar > -1) t = t.slice(bar + 1).trim(); // drop "ProgressRail | " prefix - return t; -} - -function transformBody($) { - const body = $("div.cmp-text").first(); - - // <b> -> <strong>, <i> -> <em> - body.find("b").each((_, el) => { el.tagName = "strong"; }); - body.find("i").each((_, el) => { el.tagName = "em"; }); - // DA strips redundant <strong> inside headings — match that. - body.find("h1,h2,h3,h4,h5,h6").find("strong").toArray().forEach((s) => { const $s = $(s); $s.replaceWith($s.contents()); }); - // presentational <span class="..."> shouldn't pass through to EDS content. - body.find("span[class]").toArray().forEach((s) => { const $s = $(s); $s.replaceWith($s.contents()); }); - body.find("a[target]").removeAttr("target"); // drop target attributes from links - - // Drop empty paragraphs. - body.find("p").each((_, el) => { - const $el = $(el); - const txt = $el.text().replace(/ /g, " ").trim(); - if (txt === "" && $el.find("img").length === 0) $el.remove(); - }); - - let html = body.html() || ""; - html = html.replace(/ | /g, " "); // normalise non-breaking spaces - html = html.replace(/>\s+</g, "><"); // drop whitespace-only gaps between tags - html = html.replace(/[ \t\r\n]+/g, " "); // collapse remaining whitespace - return html.trim(); -} - -function pic(src, alt = "") { - src = (src || "").replace(/^http:\/\//, "https://"); - return `<picture><source srcset="${src}">` - + `<source srcset="${src}" media="(min-width: 600px)">` - + `<img src="${src}" alt="${alt}" loading="lazy"></picture>`; -} - -// Trailing images/videos (multimedia components) appended after the body. -function extractMedia($) { - const out = []; - const seen = new Set(); - $(".multimedia").each((_, mm) => { - const $mm = $(mm); - const slides = $mm.find(".multimedia__slide"); - (slides.length ? slides.toArray() : [mm]).forEach((s) => { - const $s = $(s); - let ytid = $s.find("[data-ytvideoid]").attr("data-ytvideoid") - || $s.find("[data-videoid]").attr("data-videoid"); - if (!ytid) { - const thumb = $s.find("img").map((i, im) => $(im).attr("src")).get() - .find((x) => /(?:youtube\.com\/vi|ytimg\.com\/vi)\//.test(x || "")); - const m = thumb && thumb.match(/\/vi\/([^/]+)\//); - if (m) ytid = m[1]; - } - if (ytid) { - const url = `https://www.youtube.com/watch?v=${ytid}`; - if (!seen.has(url)) { seen.add(url); out.push(`<p><a href="${url}">${url}</a></p>`); } - return; - } - const src = $s.find("img").map((i, im) => $(im).attr("src")).get() - .find((x) => x && !/[?&](cc-s|fmt=)/.test(x) && !/youtube|ytimg/.test(x)); - if (src && !seen.has(src)) { - seen.add(src); - out.push(pic(src, $s.find("img").first().attr("alt") || "")); - const cap = $s.find("[class*=imedia__description]").first().text().replace(/\s+/g, " ").trim(); - if (cap) out.push(`<p>${cap}</p>`); - } - }); - }); - return out.join(""); -} - -// Publication date as YYYY-MM-DD from the pubDateOnly meta. -function extractPubDate($) { - const pub = $('meta[name="pubDateOnly"]').attr("content") || ""; - const m = pub.match(/(\d{4}-\d{2}-\d{2})/); - return m ? m[1] : null; -} - -function extractYear($) { - // Authoritative: the <meta name="pubDateOnly" content="YYYY-MM-DD..."> tag. - const pub = $('meta[name="pubDateOnly"]').attr("content"); - let m = pub && pub.match(/\b(19|20)\d{2}\b/); - if (m) return m[0]; - // Fallback: a year in the body dateline. - m = $("div.cmp-text").first().text().match(/\b(19|20)\d{2}\b/); - return m ? m[0] : null; -} - -function findHtml(dir, acc = []) { - for (const e of fs.readdirSync(dir, { withFileTypes: true })) { - const full = path.join(dir, e.name); - if (e.isDirectory()) findHtml(full, acc); - else if (e.isFile() && /\.html?$/i.test(e.name)) acc.push(full); - } - return acc; -} - -// Match DA's serialization: literal nbsp, & for ampersands in attributes. -const daSerialize = (html) => html.replace(/="[^"]*"/g, (m) => m.replace(/&/g, "&")); - -function build(title, bodyHtml, pubDate) { - const meta = pubDate - ? `<div class="metadata"><div><div><p>Publication Date</p></div><div><p>${pubDate}</p></div></div></div>` - : ""; - return daSerialize( - "\n<body>\n" + - " <header></header>\n" + - ` <main><div><h1>${title}</h1>${bodyHtml}${meta}</div></main>\n` + - " <footer></footer>\n" + - "</body>\n" - ); -} - -function main() { - fs.mkdirSync(OUT, { recursive: true }); - const files = findHtml(SRC).sort(); - const used = new Map(); - const rows = [["source", "new_filename", "year", "title"]]; - let noYear = 0; - - for (const file of files) { - const $ = cheerio.load(fs.readFileSync(file, "utf8"), { decodeEntities: false }); - const title = extractTitle($); - const year = extractYear($) || "0000"; - if (year === "0000") noYear++; - - let base = `${year}-${slugify(title)}`; - let name = `${base}.html`; - let n = 2; - while (used.has(name)) name = `${base}-${n++}.html`; // de-dupe collisions - used.set(name, true); - - const out = build(title, transformBody($) + extractMedia($), extractPubDate($)); - fs.writeFileSync(path.join(OUT, name), out); - rows.push([path.basename(file), name, year, title]); - } - - if (MAP) { - const csv = rows - .map((r) => r.map((c) => `"${String(c).replace(/"/g, '""')}"`).join(",")) - .join("\n"); - fs.writeFileSync(MAP, csv); - } - - console.log(`Transformed ${files.length} file(s) -> ${OUT}`); - if (noYear) console.log(` WARNING: ${noYear} file(s) had no detectable year (named 0000-…)`); - if (MAP) console.log(` Mapping written to ${MAP}`); -} - -main();