Rework

.gitattributes

@@ -1,2 +1,3 @@
 # Shell scripts copied into Linux containers must use LF only (CRLF breaks bash).
-cleandev/*.sh text eol=lf
+# (Scripts in the submodulizer/ submodule are governed by that repo's own .gitattributes.)
+submodulizer-local/*.sh text eol=lf

.github/workflows/submodulizer-tests.yml

@@ -0,0 +1,47 @@
+name: submodulizer tests
+
+on:
+  push:
+    paths:
+      - ".gitmodules"
+      - "submodulizer"
+      - "submodulizer-local/**"
+      - ".github/workflows/submodulizer-tests.yml"
+  pull_request:
+    paths:
+      - ".gitmodules"
+      - "submodulizer"
+      - "submodulizer-local/**"
+      - ".github/workflows/submodulizer-tests.yml"
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: recursive
+
+      - name: Install shellcheck
+        run: sudo apt-get update && sudo apt-get install -y shellcheck
+
+      # GitHub Actions runners ship without a git identity. The upstream
+      # submodulizer tests build temp repos and `git commit` into them, which
+      # fails with `fatal: empty ident name` unless we set one here.
+      - name: Configure git identity for upstream tests
+        run: |
+          git config --global user.email "ci@github.actions"
+          git config --global user.name "GitHub Actions (submodulizer-tests)"
+
+      # Upstream submodulizer tests (test-manifest-submod-redundant.sh,
+      # test-shellcheck.sh) expect manifest-submodulize-redundant.sh to live at
+      # the repo root next to submodulize.sh. In this project it ships as
+      # project-owned glue under submodulizer-local/. Symlink it into the
+      # submodule checkout for the duration of CI so the tests find it.
+      # CI-only: nothing here is committed and the submodule working tree is
+      # discarded with the runner.
+      - name: Expose manifest-submodulize-redundant.sh to upstream tests
+        run: ln -sf ../submodulizer-local/manifest-submodulize-redundant.sh submodulizer/manifest-submodulize-redundant.sh
+
+      - name: Run submodulizer test suite (upstream)
+        run: bash submodulizer/tests/run.sh

.gitignore

@@ -1,3 +1,8 @@
 confidential
-cleandev/github-runner.env
-cleandev/.github-token
+submodulizer-local/github-runner.env
+submodulizer-local/.github-token
+
+# Local sandbox repos used when iterating on submodulizer.sh. These are not
+# real submodules (no entries in .gitmodules, no remotes) -- keep them out of
+# git status / future stages.
+/demo/

.gitmodules

@@ -0,0 +1,4 @@
+[submodule "submodulizer"]
+	path = submodulizer
+	url = https://github.com/smatts3/submodulizer.git
+	branch = master

Dockerfile

@@ -3,11 +3,28 @@ FROM php:8.3-apache
 # Install basic tools
 RUN apt update && apt install git -y
 
-# Import dev source code
-RUN su -g www-data -c "git clone --branch develop --single-branch https://github.com/lsuonline/lsuce-moodle.git /var/www/html/"
+# Import dev source code. lsuonline/moodleus is private; the token comes in via
+# a BuildKit secret (id=github_token). docker-compose.yml maps it from the
+# GITHUB_TOKEN env var; build.sh passes --secret id=github_token,env=GITHUB_TOKEN.
+# Both entry points use submodulizer-local/lib-github-token.sh to populate
+# GITHUB_TOKEN. The token is consumed via `git -c url...insteadOf`, so neither
+# the token nor the username persist in image layers or /var/www/html/.git/config.
+# Falls back to an anonymous clone when no secret is provided (useful for testing
+# against any future public mirror; fails for the real private repo).
+RUN --mount=type=secret,id=github_token \
+    su -g www-data -c 'URL=https://github.com/lsuonline/moodleus.git; \
+        if [ -s /run/secrets/github_token ]; then \
+            TOKEN=$(cat /run/secrets/github_token); \
+            git -c "url.https://smatts3%40lsu.edu:${TOKEN}@github.com/.insteadOf=https://github.com/" \
+                clone --branch MOODLE_405_MAIN --single-branch "$URL" /var/www/html/; \
+        else \
+            git clone --branch MOODLE_405_MAIN --single-branch "$URL" /var/www/html/; \
+        fi'
 
-# Install dependencies
-RUN apt-get install -y --fix-missing \
+# Install dependencies. `apt-get update` must run in the same layer as install:
+# the base image ships a cached apt index that pins -security package versions
+# Debian removes over time (e.g. libpng1.6 .deb 404s on stale indexes).
+RUN apt-get update && apt-get install -y --fix-missing \
 		libpng-dev \
 		libonig-dev \
 		libjpeg-dev \

Dockerfile.txt

@@ -3,11 +3,23 @@ FROM php:8.3-apache
 # Install basic tools
 RUN apt update && apt install git -y
 
-# Import dev source code
-RUN su -g www-data -c "git clone --branch develop --single-branch https://github.com/lsuonline/lsuce-moodle.git /var/www/html/"
+# Import dev source code. See Dockerfile for full notes on auth: the token
+# comes in via a BuildKit secret (id=github_token) supplied by build.sh /
+# docker-compose.yml, consumed via `git -c url...insteadOf` so it never
+# persists in image layers or .git/config.
+RUN --mount=type=secret,id=github_token \
+    su -g www-data -c 'URL=https://github.com/lsuonline/moodleus.git; \
+        if [ -s /run/secrets/github_token ]; then \
+            TOKEN=$(cat /run/secrets/github_token); \
+            git -c "url.https://smatts3%40lsu.edu:${TOKEN}@github.com/.insteadOf=https://github.com/" \
+                clone --branch MOODLE_405_MAIN --single-branch "$URL" /var/www/html/; \
+        else \
+            git clone --branch MOODLE_405_MAIN --single-branch "$URL" /var/www/html/; \
+        fi'
 
-# Install dependencies
-RUN apt-get install -y \
+# Install dependencies. apt-get update must be in the same layer as install
+# (stale base-image apt index causes -security 404s on removed packages).
+RUN apt-get update && apt-get install -y \
         libpng-dev \
         libonig-dev \
         libjpeg-dev \

README.md

@@ -62,7 +62,7 @@ To convert vendored plugin directories in the container to Git submodules (see *
 ./new.sh NAME --submodulize
 ```
 
-GitHub auth for private `lsuonline/*` repos: `GITHUB_TOKEN` or `GH_TOKEN`, file `cleandev/.github-token` (gitignored), interactive prompt when run in a TTY, or `SUBMODULIZE_SSH=1` if SSH works inside the container.
+GitHub auth for private `lsuonline/*` repos: `GITHUB_TOKEN` or `GH_TOKEN`, file `submodulizer-local/.github-token` (gitignored), interactive prompt when run in a TTY, or `SUBMODULIZE_SSH=1` if SSH works inside the container.
 
 # Build
 
@@ -88,13 +88,23 @@ These are **branches on the Moodle repo** (`lsuce-moodle`), not branch names in
 - Porting changes **both ways** (submodule layout ↔ vendored layout).
 - **Redoing or replaying commits** with a **provenance note** (e.g. original commit SHA / branch)—so history stays traceable across layouts.
 
-**Current repo state:** This project ships **one-shot layout converters** (`submodulize.sh` / `unsubmodulize.sh`). It does **not** yet implement commit-by-commit replay, automatic provenance headers, or a `new.sh` mode that detects vendored vs submodulized state without manual choice.
+**Current repo state:** This project ships **layout converters** (`submodulize.sh` / `unsubmodulize.sh`). **Replay is the default:** they build `submodulized` / `unsubmodulized` branches with **one superproject commit per plugin-repo commit** (chronological ordering, carry-forward). **`--fork-point`** defaults to local **`master`** (else **`main`**) when omitted, if a default can be chosen safely; **`--source`** defaults for **unsub** to **`unsubmodulized`**, else **`master`**, else **`main`**, else **`submodulized`** (for **sub**: **`submodulized`**, **`master`**, **`main`**). Use **`--no-replay`** for one-shot conversion over the manifest only. A `new.sh` mode that detects vendored vs submodulized state without manual choice is still not implemented.
 
-**Branch policy, Docker image decision, and updating the manifest from the CSV:** [cleandev/TEAM-PROCESS.md](cleandev/TEAM-PROCESS.md).
+**Branch policy, Docker image decision, and updating the manifest from the CSV:** [submodulizer-local/TEAM-PROCESS.md](submodulizer-local/TEAM-PROCESS.md).
 
-# Submodule tooling (`cleandev/`)
+# Submodule tooling (`submodulizer/` + `submodulizer-local/`)
 
-## Manifest (`cleandev/plugin-submodules.manifest`)
+The conversion scripts (`submodulize.sh`, `unsubmodulize.sh`, their tests) live in **[smatts3/submodulizer](https://github.com/smatts3/submodulizer)**, vendored here as a Git submodule at `submodulizer/`. Project-owned glue (manifest, team process, runner config) lives next to it in `submodulizer-local/`.
+
+After cloning this repo:
+
+```bash
+git submodule update --init --recursive
+```
+
+## Manifest (`submodulizer-local/plugin-submodules.manifest`)
+
+This repo keeps a **canonical copy** under `submodulizer-local/` for linting and for `new.sh --submodulize` (copied into the Moodle tree at runtime). When you run `submodulize.sh` / `unsubmodulize.sh` against a checkout, the default manifest path is **`plugin-submodules.manifest` at the Moodle superproject root** (`--repo` / current directory), not next to the scripts; use `--manifest PATH` to override.
 
 - Format: `relative_path|clone_url|branch` (e.g. `mod/hvp|https://github.com/...|main`). Lines starting with `#` are ignored.
 - If the third field is empty, scripts default the branch to `main`; if `refs/heads/<branch>` is missing on the remote, they omit `-b` and use the remote’s default branch.
@@ -107,24 +117,30 @@ These are **branches on the Moodle repo** (`lsuce-moodle`), not branch names in
 
 | Script | Role |
 |--------|------|
-| `cleandev/submodulize.sh` | Vendored trees → submodules (sparse-checkout disabled first; skips paths already in `.gitmodules`; `GITHUB_TOKEN` passed via `-c url...insteadOf` for `ls-remote` / `submodule add`). |
-| `cleandev/unsubmodulize.sh` | Submodules → vendored trees (clone depth 1, drop nested `.git`, `git add`). Uses the same `GITHUB_TOKEN` `-c url...insteadOf` for `ls-remote` / `clone` as `submodulize.sh`, or `--ssh`. |
+| `submodulizer/submodulize.sh` | Default **replay**; **`--fork-point`** defaults to `master`/`main` when omitted (see script help). **`--no-replay`**: one-shot vendored → submodules (sparse-checkout disabled first; skips paths already in `.gitmodules`; `GITHUB_TOKEN` via `-c url...insteadOf` for `ls-remote` / `submodule add`). |
+| `submodulizer/unsubmodulize.sh` | Default **replay**; same **`--fork-point`** defaulting. **`--no-replay`**: one-shot submodules → vendored (clone depth 1, drop nested `.git`, `git add`). Same `GITHUB_TOKEN` / `--ssh` as `submodulize.sh`. |
 
 Run manually from a Moodle clone:
 
 ```bash
-./cleandev/submodulize.sh [--dry-run] [--no-commit] [--ssh] [--manifest PATH] [--repo ROOT]
-./cleandev/unsubmodulize.sh [--dry-run] [--no-commit] [--ssh] [--manifest PATH] [--repo ROOT]
+./submodulizer/submodulize.sh [--no-replay] [--dry-run] [--no-commit] [--ssh] [--manifest PATH] [--repo ROOT]
+./submodulizer/unsubmodulize.sh [--no-replay] [--dry-run] [--no-commit] [--ssh] [--manifest PATH] [--repo ROOT]
+```
+
+Automated tests (manifest lint, PAT wiring checks, `submodulize`/`unsubmodulize` round-trip in temp repos—no changes to your working tree):
+
+```bash
+bash submodulizer/tests/run.sh
 ```
 
 ## Container startup (`new.sh`)
 
 - Compose project name = first argument (containers `{NAME}-moodle`, etc.).
 - Web service builds from `.` per `docker-compose.yml`.
 - As `www-data`: `git fetch` / `git merge origin/develop`; removes `blocks/ues_people` and uses `skip-worktree` so it does not clash with `block_lsu_people` (see `config/moodle-pull` for the same idea on `git pull`).
-- With `--submodulize`: copies `submodulize.sh` + manifest into the container, resolves GitHub token, sets local `url...insteadOf` when using HTTPS token, runs `submodulize.sh --repo /var/www/html ... --no-commit` (optional SSH via `SUBMODULIZE_SSH=1`).
+- With `--submodulize`: copies scripts into the container, stages `submodulizer-local/plugin-submodules.manifest` into `/var/www/html/plugin-submodules.manifest` after the merge, resolves GitHub token, sets local `url...insteadOf` when using HTTPS token, runs `manifest-submodulize-redundant.sh` / `submodulize.sh --no-replay` with default manifest paths (`--repo /var/www/html`, optional SSH via `SUBMODULIZE_SSH=1`).
 
-Secrets: `cleandev/.github-token` is listed in `.gitignore`.
+Secrets: `submodulizer-local/.github-token` is listed in `.gitignore`.
 
 # Roadmap
 

ROADMAP.md

@@ -6,24 +6,25 @@ Progress toward **seamless `develop` ↔ `cleandev`** (on [lsuce-moodle](https:/
 
 - [x] Docker Compose + Traefik-oriented local stack via `new.sh`
 - [x] Web image built from repo `Dockerfile` (PHP 8.3) to avoid Moodle 4.5 + PHP 8.4 CLI `SerializableClosure` install failure (`docker-compose.yml` documents reverting to a pre-built image if desired)
-- [x] `cleandev/plugin-submodules.manifest` + `submodulize.sh` + `unsubmodulize.sh`
+- [x] `submodulizer-local/plugin-submodules.manifest` + `submodulizer/submodulize.sh` + `submodulizer/unsubmodulize.sh` (the scripts vendored as a Git submodule from [smatts3/submodulizer](https://github.com/smatts3/submodulizer))
 - [x] `new.sh --submodulize` with token file / env / prompt and optional SSH
 - [x] Sparse-checkout disabled in both scripts before mutating plugin paths
 - [x] `blocks/ues_people` handling in `new.sh` and `config/moodle-pull` (merge / pull alias)
-- [x] Git ignore for `cleandev/.github-token`
-- [x] **Team process, image strategy, manifest-from-CSV docs:** [cleandev/TEAM-PROCESS.md](cleandev/TEAM-PROCESS.md)
+- [x] Git ignore for `submodulizer-local/.github-token`
+- [x] **Team process, image strategy, manifest-from-CSV docs:** [submodulizer-local/TEAM-PROCESS.md](submodulizer-local/TEAM-PROCESS.md)
 
 ## TODO
 
 - [ ] **Commit portability:** Script or documented workflow to **replay or port commits** across layouts with a **header** noting the original commit (and branch/repo). *Not in repo today—only bulk layout conversion exists.*
-- [ ] **`new.sh` / tooling:** Detect or select **vendored vs submodulized** Moodle tree so one flow works without guessing flags.
+- [x] **`new.sh` / tooling:** With `--submodulize`, **detect** when every manifest path is already a submodule (cleandev-style) and **skip** `submodulize.sh` instead of a no-op pass. *Not done: auto-run conversion without passing `--submodulize`.*
 - [x] **`unsubmodulize.sh`:** Same `GITHUB_TOKEN` `-c url.insteadOf` pattern as `submodulize.sh` for private HTTPS (parity with submodule add).
-- [ ] **Monorepo manifest + scripts:** Support “clone once, map subpaths”, or publish **one-shot manual procedures** for commented monorepo lines.
+- [x] **Monorepo manifest + scripts:** **One-shot manual procedures** for commented monorepo lines: [submodulizer-local/TEAM-PROCESS.md](submodulizer-local/TEAM-PROCESS.md) (manifest regeneration → Monorepos). *Not done: automated “clone once, map subpaths” in `submodulize.sh`.*
 - [ ] **`local/ml`:** Point at the correct Git remote when known; uncomment or add manifest line.
+- [x] **CI (submodulizer):** GitHub Actions runs `submodulizer/tests/run.sh` (manifest lint, round-trip integration tests, shellcheck on Ubuntu).
 - [ ] **Optional CI:** Check that manifest paths still match **lsuce-moodle** `develop` plugin layout when plugins move.
 
 ---
 
-For branch policy, when to use `new.sh --submodulize`, image strategy, and refreshing `plugin-submodules.manifest` from the CSV, see [cleandev/TEAM-PROCESS.md](cleandev/TEAM-PROCESS.md).
+For branch policy, when to use `new.sh --submodulize`, image strategy, and refreshing `plugin-submodules.manifest` from the CSV, see [submodulizer-local/TEAM-PROCESS.md](submodulizer-local/TEAM-PROCESS.md).
 
-For a concise summary of flags and auth behavior, see comments at the top of `cleandev/submodulize.sh` and `cleandev/unsubmodulize.sh`.
+For a concise summary of flags and auth behavior, see comments at the top of `submodulizer/submodulize.sh` and `submodulizer/unsubmodulize.sh`.

build.sh

@@ -1,5 +1,16 @@
-#!/bin/sh
+#!/usr/bin/env bash
+set -euo pipefail
 
-SCRIPT_DIR="$(dirname "$(readlink -f "$0")")";
-cd $SCRIPT_DIR;
-docker build -t lsuonline/moodle-dev:latest "$@" .
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+cd "$SCRIPT_DIR"
+
+# lsuonline/moodleus is a private repo; the Dockerfile clones it via a BuildKit
+# secret. Resolve+export GITHUB_TOKEN, then pass --secret so docker build can
+# mount it. See submodulizer-local/lib-github-token.sh for sources/prompts.
+# shellcheck source=submodulizer-local/lib-github-token.sh
+. "${SCRIPT_DIR}/submodulizer-local/lib-github-token.sh"
+resolve_github_token || exit 1
+
+DOCKER_BUILDKIT=1 docker build \
+    --secret id=github_token,env=GITHUB_TOKEN \
+    -t lsuonline/moodle-dev:latest "$@" .