From 0749973044ed2b417c7bc1d1da6eb2e50a24b5a5 Mon Sep 17 00:00:00 2001 From: Steve Mattsen Date: Tue, 7 Apr 2026 10:27:54 -0400 Subject: [PATCH 01/18] Added automated tests --- .gitattributes | 1 + .github/workflows/cleandev-tests.yml | 23 +++++++ README.md | 6 ++ ROADMAP.md | 1 + cleandev/tests/lib.sh | 29 +++++++++ cleandev/tests/run.sh | 32 +++++++++ cleandev/tests/test-manifest-lint.sh | 86 +++++++++++++++++++++++++ cleandev/tests/test-pat-parity.sh | 22 +++++++ cleandev/tests/test-shellcheck.sh | 19 ++++++ cleandev/tests/test-sparse-submod.sh | 46 +++++++++++++ cleandev/tests/test-submod-roundtrip.sh | 61 ++++++++++++++++++ docker-compose.yml | 10 +-- 12 files changed, 331 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/cleandev-tests.yml create mode 100644 cleandev/tests/lib.sh create mode 100644 cleandev/tests/run.sh create mode 100644 cleandev/tests/test-manifest-lint.sh create mode 100644 cleandev/tests/test-pat-parity.sh create mode 100644 cleandev/tests/test-shellcheck.sh create mode 100644 cleandev/tests/test-sparse-submod.sh create mode 100644 cleandev/tests/test-submod-roundtrip.sh diff --git a/.gitattributes b/.gitattributes index 691422b..6a2d3c2 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,2 +1,3 @@ # Shell scripts copied into Linux containers must use LF only (CRLF breaks bash). cleandev/*.sh text eol=lf +cleandev/tests/*.sh text eol=lf diff --git a/.github/workflows/cleandev-tests.yml b/.github/workflows/cleandev-tests.yml new file mode 100644 index 0000000..7f9808c --- /dev/null +++ b/.github/workflows/cleandev-tests.yml @@ -0,0 +1,23 @@ +name: cleandev tests + +on: + push: + paths: + - "cleandev/**" + - ".github/workflows/cleandev-tests.yml" + pull_request: + paths: + - "cleandev/**" + - ".github/workflows/cleandev-tests.yml" + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Install shellcheck + run: sudo apt-get update && sudo apt-get install -y shellcheck + + - name: Run cleandev test suite + run: bash cleandev/tests/run.sh diff --git a/README.md b/README.md index f1c4b95..b729a36 100644 --- a/README.md +++ b/README.md @@ -117,6 +117,12 @@ Run manually from a Moodle clone: ./cleandev/unsubmodulize.sh [--dry-run] [--no-commit] [--ssh] [--manifest PATH] [--repo ROOT] ``` +Automated tests (manifest lint, PAT wiring checks, `submodulize`/`unsubmodulize` round-trip in temp repos—no changes to your working tree): + +```bash +bash cleandev/tests/run.sh +``` + ## Container startup (`new.sh`) - Compose project name = first argument (containers `{NAME}-moodle`, etc.). diff --git a/ROADMAP.md b/ROADMAP.md index fbba8e5..7cb0477 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -20,6 +20,7 @@ Progress toward **seamless `develop` ↔ `cleandev`** (on [lsuce-moodle](https:/ - [x] **`unsubmodulize.sh`:** Same `GITHUB_TOKEN` `-c url.insteadOf` pattern as `submodulize.sh` for private HTTPS (parity with submodule add). - [ ] **Monorepo manifest + scripts:** Support “clone once, map subpaths”, or publish **one-shot manual procedures** for commented monorepo lines. - [ ] **`local/ml`:** Point at the correct Git remote when known; uncomment or add manifest line. +- [x] **CI (cleandev):** GitHub Actions runs `cleandev/tests/run.sh` (manifest lint, round-trip integration tests, shellcheck on Ubuntu). - [ ] **Optional CI:** Check that manifest paths still match **lsuce-moodle** `develop` plugin layout when plugins move. --- diff --git a/cleandev/tests/lib.sh b/cleandev/tests/lib.sh new file mode 100644 index 0000000..b09f58c --- /dev/null +++ b/cleandev/tests/lib.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# Shared helpers for cleandev/tests. Sourced by test-*.sh; do not run directly. +# shellcheck shell=bash + +fail() { + echo "FAIL: $*" >&2 + exit 1 +} + +ok() { + echo "ok: $*" +} + +assert_file() { + [[ -f "$1" ]] || fail "expected file missing: $1" +} + +assert_dir() { + [[ -d "$1" ]] || fail "expected directory missing: $1" +} + +assert_no_file() { + [[ ! -e "$1" ]] || fail "expected missing path exists: $1" +} + +# CLEANDEV must be set by run.sh +require_cleandev() { + [[ -n "${CLEANDEV:-}" && -d "$CLEANDEV" ]] || fail "CLEANDEV not set or not a directory" +} diff --git a/cleandev/tests/run.sh b/cleandev/tests/run.sh new file mode 100644 index 0000000..cb7477e --- /dev/null +++ b/cleandev/tests/run.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# Run all cleandev automated tests. Uses only temp dirs under /tmp; does not modify the workspace. +# +# Usage: bash cleandev/tests/run.sh +# or: cd cleandev/tests && ./run.sh +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +CLEANDEV="$(cd "$SCRIPT_DIR/.." && pwd)" +export CLEANDEV + +echo "CLEANDEV=$CLEANDEV" +echo "" + +failed=0 +for t in "$SCRIPT_DIR"/test-*.sh; do + [[ -f "$t" ]] || continue + echo "---------- $(basename "$t") ----------" + if bash "$t"; then + echo "" + else + echo "FAILED: $(basename "$t")" >&2 + failed=1 + echo "" + fi +done + +if [[ "$failed" -ne 0 ]]; then + echo "Some tests failed." >&2 + exit 1 +fi +echo "All tests passed." diff --git a/cleandev/tests/test-manifest-lint.sh b/cleandev/tests/test-manifest-lint.sh new file mode 100644 index 0000000..5a94a78 --- /dev/null +++ b/cleandev/tests/test-manifest-lint.sh @@ -0,0 +1,86 @@ +#!/usr/bin/env bash +# Validate plugin-submodules.manifest: format, duplicate paths, monorepo-style duplicate URLs. +# Limitation: two active lines with the same URL but different branches still count as duplicate URL (intentional). +# Does not validate that URLs are reachable or that paths exist on disk. +# Portable: no associative arrays (works on macOS /bin/bash 3.2 if needed for local runs). +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=lib.sh +source "$SCRIPT_DIR/lib.sh" +require_cleandev + +MANIFEST="${1:-$CLEANDEV/plugin-submodules.manifest}" +[[ -f "$MANIFEST" ]] || fail "manifest not found: $MANIFEST" + +line_num=0 +active_count=0 +seen_paths="" + +url_exists() { + local u="$1" + local entry + while IFS= read -r entry; do + [[ "$entry" == "$u" ]] && return 0 + done <<< "${url_list:-}" + return 1 +} + +url_record() { + local u="$1" + url_list="${url_list:-}${url_list:+$'\n'}$u" +} + +path_exists_in_list() { + local p="$1" + local x + while IFS= read -r x; do + [[ "$x" == "$p" ]] && return 0 + done <<< "${seen_paths:-}" + return 1 +} + +url_list="" + +while IFS= read -r raw || [[ -n "$raw" ]]; do + line_num=$((line_num + 1)) + line="${raw//$'\r'/}" + trimmed="${line#"${line%%[![:space:]]*}"}" + trimmed="${trimmed%"${trimmed##*[![:space:]]}"}" + [[ -z "$trimmed" ]] && continue + [[ "$trimmed" =~ ^# ]] && continue + + if [[ "$trimmed" != *'|'* ]]; then + fail "line $line_num: expected path|url|branch (no pipe): $trimmed" + fi + + path="${trimmed%%|*}" + rest="${trimmed#*|}" + url="${rest%%|*}" + # Optional third field (branch) is ignored here; scripts default it. + + path="${path#"${path%%[![:space:]]*}"}" + path="${path%"${path##*[![:space:]]}"}" + url="${url#"${url%%[![:space:]]*}"}" + url="${url%"${url##*[![:space:]]}"}" + + [[ -n "$path" ]] || fail "line $line_num: empty path" + [[ -n "$url" ]] || fail "line $line_num: empty url for path $path" + + pipe_count=$(printf '%s' "$trimmed" | tr -cd '|' | wc -c | tr -d ' ') + [[ "$pipe_count" -le 2 ]] || fail "line $line_num: too many '|' fields (max 2): $trimmed" + + if path_exists_in_list "$path"; then + fail "duplicate manifest path '$path' (line $line_num)" + fi + seen_paths="${seen_paths:-}${seen_paths:+$'\n'}$path" + active_count=$((active_count + 1)) + + if url_exists "$url"; then + fail "monorepo-style duplicate clone URL for two active paths (submodulize cannot map 1:1): url='$url' (line $line_num)" + fi + url_record "$url" +done < "$MANIFEST" + +[[ "$active_count" -gt 0 ]] || fail "no active manifest lines (only comments/empty)" + +ok "manifest lint: $active_count active lines, $MANIFEST" diff --git a/cleandev/tests/test-pat-parity.sh b/cleandev/tests/test-pat-parity.sh new file mode 100644 index 0000000..5d4ca74 --- /dev/null +++ b/cleandev/tests/test-pat-parity.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +# Regression guard: source still contains the PAT git -c wiring (substring match). +# This does not prove HTTPS auth works at runtime; integration with GitHub needs manual/CI secrets tests. +# shellcheck disable=SC2016 # grep -F patterns intentionally contain literal ${git_github_pat_c[@]} +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=lib.sh +source "$SCRIPT_DIR/lib.sh" +require_cleandev + +SUB="$CLEANDEV/submodulize.sh" +UNS="$CLEANDEV/unsubmodulize.sh" + +grep -Fq 'git_github_pat_c=()' "$SUB" || fail "submodulize: missing git_github_pat_c init" +grep -Fq 'git "${git_github_pat_c[@]}" ls-remote' "$SUB" || fail "submodulize: ls-remote missing PAT -c array" +grep -Fq 'git "${git_github_pat_c[@]}" -c core.sparseCheckout=false' "$SUB" || fail "submodulize: submodule add missing PAT -c array" + +grep -Fq 'git_github_pat_c=()' "$UNS" || fail "unsubmodulize: missing git_github_pat_c init" +grep -Fq 'git "${git_github_pat_c[@]}" ls-remote' "$UNS" || fail "unsubmodulize: ls-remote missing PAT -c array" +grep -Fq 'git "${git_github_pat_c[@]}" clone' "$UNS" || fail "unsubmodulize: clone missing PAT -c array" + +ok "PAT -c parity present in submodulize.sh and unsubmodulize.sh" diff --git a/cleandev/tests/test-shellcheck.sh b/cleandev/tests/test-shellcheck.sh new file mode 100644 index 0000000..725db6a --- /dev/null +++ b/cleandev/tests/test-shellcheck.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# Optional: shellcheck cleandev scripts when shellcheck is installed. +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=lib.sh +source "$SCRIPT_DIR/lib.sh" +CLEANDEV="${CLEANDEV:-$(cd "$SCRIPT_DIR/.." && pwd)}" +export CLEANDEV +require_cleandev + +if ! command -v shellcheck >/dev/null 2>&1; then + echo "SKIP shellcheck (not installed)" + exit 0 +fi + +# Run from this directory so `source=lib.sh` directives resolve (shellcheck -x). +cd "$SCRIPT_DIR" +shellcheck -x lib.sh run.sh test-*.sh ../submodulize.sh ../unsubmodulize.sh +ok "shellcheck" diff --git a/cleandev/tests/test-sparse-submod.sh b/cleandev/tests/test-sparse-submod.sh new file mode 100644 index 0000000..cbcadd0 --- /dev/null +++ b/cleandev/tests/test-sparse-submod.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +# submodulize still works when the superproject had sparse-checkout enabled. +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=lib.sh +source "$SCRIPT_DIR/lib.sh" +require_cleandev + +TMP=$(mktemp -d) +export HOME="$TMP/_home" +mkdir -p "$HOME" +git config --global protocol.file.allow always +trap 'rm -rf "$TMP"' EXIT + +PLUGIN="$TMP/plugin-upstream" +MOODLE="$TMP/moodle" +mkdir -p "$PLUGIN" "$MOODLE" +git -C "$PLUGIN" init -b main +echo "p" > "$PLUGIN/a.txt" +git -C "$PLUGIN" add a.txt +git -C "$PLUGIN" commit -q -m "i" + +git -C "$MOODLE" init -b develop +echo "core" > "$MOODLE/README.txt" +mkdir -p "$MOODLE/mod/testplugin" +echo "v" > "$MOODLE/mod/testplugin/x.txt" +git -C "$MOODLE" add README.txt mod/testplugin +git -C "$MOODLE" commit -q -m "init" + +# Cone mode expects directory paths; include whole `mod/` so `mod/testplugin` is on disk. +git -C "$MOODLE" sparse-checkout init --cone +git -C "$MOODLE" sparse-checkout set mod + +MANIFEST="$TMP/manifest.txt" +printf '%s\n' 'mod/testplugin|../plugin-upstream|main' > "$MANIFEST" + +"$CLEANDEV/submodulize.sh" --repo "$MOODLE" --manifest "$MANIFEST" --no-commit + +assert_file "$MOODLE/mod/testplugin/a.txt" +# submodulize disables the same flags it clears in disable_sparse_checkout_if_needed() +[[ "$(git -C "$MOODLE" config --bool core.sparseCheckout 2>/dev/null || echo false)" != "true" ]] \ + || fail "expected core.sparseCheckout not true after submodulize" +[[ "$(git -C "$MOODLE" config --bool index.sparse 2>/dev/null || echo false)" != "true" ]] \ + || fail "expected index.sparse not true after submodulize" + +ok "submodulize with prior sparse-checkout" diff --git a/cleandev/tests/test-submod-roundtrip.sh b/cleandev/tests/test-submod-roundtrip.sh new file mode 100644 index 0000000..f2e5d03 --- /dev/null +++ b/cleandev/tests/test-submod-roundtrip.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +# Vendored plugin -> submodulize -> unsubmodulize round-trip in isolated temp repos (no network). +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=lib.sh +source "$SCRIPT_DIR/lib.sh" +require_cleandev + +TMP=$(mktemp -d) +export HOME="$TMP/_home" +mkdir -p "$HOME" +# Submodule clone resolves sibling paths via file://; allow only inside this test process. +git config --global protocol.file.allow always +trap 'rm -rf "$TMP"' EXIT + +PLUGIN="$TMP/plugin-upstream" +MOODLE="$TMP/moodle" +mkdir -p "$PLUGIN" +git -C "$PLUGIN" init -b main +echo "plug=1" > "$PLUGIN/version.txt" +git -C "$PLUGIN" add version.txt +git -C "$PLUGIN" commit -q -m "init plugin" + +mkdir -p "$MOODLE/mod/testplugin" +git -C "$MOODLE" init -b develop +echo "vendored-only" > "$MOODLE/mod/testplugin/local.txt" +git -C "$MOODLE" add mod/testplugin +git -C "$MOODLE" commit -q -m "vendored plugin" + +MANIFEST="$TMP/manifest.txt" +printf '%s\n' 'mod/testplugin|../plugin-upstream|main' > "$MANIFEST" + +# --- dry-run must not convert --- +"$CLEANDEV/submodulize.sh" --repo "$MOODLE" --manifest "$MANIFEST" --dry-run +assert_file "$MOODLE/mod/testplugin/local.txt" +assert_no_file "$MOODLE/.gitmodules" +assert_no_file "$MOODLE/.git/modules/mod/testplugin" + +# --- submodulize --- +"$CLEANDEV/submodulize.sh" --repo "$MOODLE" --manifest "$MANIFEST" --no-commit +assert_file "$MOODLE/.gitmodules" +assert_file "$MOODLE/mod/testplugin/version.txt" +assert_no_file "$MOODLE/mod/testplugin/local.txt" +[[ -f "$MOODLE/mod/testplugin/.git" || -d "$MOODLE/mod/testplugin/.git" ]] || fail "expected submodule gitlink" +git -C "$MOODLE" submodule status --recursive | grep -q 'mod/testplugin' || fail "expected mod/testplugin in submodule status" + +# --- unsubmodulize --- +"$CLEANDEV/unsubmodulize.sh" --repo "$MOODLE" --manifest "$MANIFEST" --no-commit +assert_file "$MOODLE/mod/testplugin/version.txt" +assert_no_file "$MOODLE/mod/testplugin/.git" +# Git may leave an empty .gitmodules; ensure no submodule entries remain. +if [[ -f "$MOODLE/.gitmodules" ]] && git -C "$MOODLE" config -f .gitmodules --get-regexp path 2>/dev/null | grep -q .; then + fail "expected no [submodule] entries in .gitmodules after unsubmodulize" +fi +# unsubmodulize stages vendored files; ensure superproject tracks them (not just loose files on disk). +git -C "$MOODLE" ls-files --error-unmatch mod/testplugin/version.txt >/dev/null 2>&1 \ + || fail "expected mod/testplugin/version.txt in git index after unsubmodulize" +! git -C "$MOODLE" submodule status 2>/dev/null | grep -q 'mod/testplugin' \ + || fail "expected mod/testplugin not to remain a submodule after unsubmodulize" + +ok "submodulize <-> unsubmodulize round-trip (local relative URL)" diff --git a/docker-compose.yml b/docker-compose.yml index b514d36..9484f5a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,11 +15,11 @@ services: retries: 10 web: - # Moodle 4.5.x is supported through PHP 8.3. PHP 8.4+ can break CLI install with: - # Exception - Cannot compile closures which import variables using the `use` keyword - # (Laravel SerializableClosure). Override if your registry publishes a PHP 8.3 image, e.g.: - # MOODLE_DEV_IMAGE=lsuonline/moodle-dev:php83 docker compose -p NAME up -d - image: ${MOODLE_DEV_IMAGE:-lsuonline/moodle-dev:latest} + # Build from ./Dockerfile (php:8.3-apache). lsuonline/moodle-dev:latest has shipped PHP 8.4, which + # breaks Moodle 4.5 CLI install: "Cannot compile closures which import variables using the use keyword". + # To use a pre-built image instead, remove `build:` below and set `image: your-registry/moodle-dev:tag`. + build: . + image: lsuce-moodle-web:local container_name: ${BRANCH_NAME}-moodle depends_on: db: From 97635aa27696cb2a82580890a48404cd2f9db835 Mon Sep 17 00:00:00 2001 From: Steve Mattsen Date: Wed, 8 Apr 2026 17:03:05 -0400 Subject: [PATCH 02/18] Enhance `new.sh` to skip `submodulize.sh` when all manifest paths are already submodules. Update checks for required files and improve error handling. Modify shellcheck tests to include the new script. --- ROADMAP.md | 2 +- cleandev/manifest-submodulize-redundant.sh | 68 ++++++++++++ .../tests/test-manifest-submod-redundant.sh | 101 ++++++++++++++++++ cleandev/tests/test-shellcheck.sh | 2 +- new.sh | 41 ++++--- 5 files changed, 195 insertions(+), 19 deletions(-) create mode 100644 cleandev/manifest-submodulize-redundant.sh create mode 100644 cleandev/tests/test-manifest-submod-redundant.sh diff --git a/ROADMAP.md b/ROADMAP.md index 7cb0477..8c767c9 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -16,7 +16,7 @@ Progress toward **seamless `develop` ↔ `cleandev`** (on [lsuce-moodle](https:/ ## TODO - [ ] **Commit portability:** Script or documented workflow to **replay or port commits** across layouts with a **header** noting the original commit (and branch/repo). *Not in repo today—only bulk layout conversion exists.* -- [ ] **`new.sh` / tooling:** Detect or select **vendored vs submodulized** Moodle tree so one flow works without guessing flags. +- [x] **`new.sh` / tooling:** With `--submodulize`, **detect** when every manifest path is already a submodule (cleandev-style) and **skip** `submodulize.sh` instead of a no-op pass. *Not done: auto-run conversion without passing `--submodulize`.* - [x] **`unsubmodulize.sh`:** Same `GITHUB_TOKEN` `-c url.insteadOf` pattern as `submodulize.sh` for private HTTPS (parity with submodule add). - [ ] **Monorepo manifest + scripts:** Support “clone once, map subpaths”, or publish **one-shot manual procedures** for commented monorepo lines. - [ ] **`local/ml`:** Point at the correct Git remote when known; uncomment or add manifest line. diff --git a/cleandev/manifest-submodulize-redundant.sh b/cleandev/manifest-submodulize-redundant.sh new file mode 100644 index 0000000..9ae802d --- /dev/null +++ b/cleandev/manifest-submodulize-redundant.sh @@ -0,0 +1,68 @@ +#!/usr/bin/env bash +# Exit 0 if running submodulize.sh would be a no-op: every active manifest path is already listed +# in .gitmodules and .gitmodules is non-empty. Exit 1 otherwise (submodulize should run). +# Used by new.sh --submodulize and cleandev/tests. +# +# Usage: ./cleandev/manifest-submodulize-redundant.sh --repo ROOT --manifest PATH +set -euo pipefail + +REPO_ROOT="" +MANIFEST="" + +while [[ $# -gt 0 ]]; do + case "$1" in + --repo) + REPO_ROOT="${2:?}" + shift 2 + ;; + --manifest) + MANIFEST="${2:?}" + shift 2 + ;; + -h|--help) + sed -n '1,12p' "$0" + exit 0 + ;; + *) + echo "Unknown option: $1" >&2 + exit 2 + ;; + esac +done + +[[ -n "$REPO_ROOT" && -n "$MANIFEST" ]] || { + echo "Usage: $0 --repo ROOT --manifest PATH" >&2 + exit 2 +} +[[ -f "$MANIFEST" ]] || { + echo "Manifest not found: $MANIFEST" >&2 + exit 2 +} + +cd "$REPO_ROOT" + +need_submod=0 +active=0 +while IFS="|" read -r p _rest || [[ -n "${p:-}" ]]; do + case "${p#[[:space:]]}" in + ""|\#*) continue ;; + esac + p="${p#"${p%%[![:space:]]*}"}" + p="${p%"${p##*[![:space:]]}"}" + case "$p" in + ""|\#*) continue ;; + esac + active=$((active + 1)) + if ! git config -f .gitmodules --get-regexp path 2>/dev/null | awk '{print $2}' | grep -Fxq "$p"; then + need_submod=1 + break + fi +done < "$MANIFEST" + +if [[ "$active" -eq 0 ]]; then + exit 1 +fi +if [[ "$need_submod" -eq 0 && -s .gitmodules ]]; then + exit 0 +fi +exit 1 diff --git a/cleandev/tests/test-manifest-submod-redundant.sh b/cleandev/tests/test-manifest-submod-redundant.sh new file mode 100644 index 0000000..ed71b34 --- /dev/null +++ b/cleandev/tests/test-manifest-submod-redundant.sh @@ -0,0 +1,101 @@ +#!/usr/bin/env bash +# Unit tests for cleandev/manifest-submodulize-redundant.sh (new.sh --submodulize skip logic). +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=lib.sh +source "$SCRIPT_DIR/lib.sh" +require_cleandev + +REDUNDANT_SH="$CLEANDEV/manifest-submodulize-redundant.sh" +assert_file "$REDUNDANT_SH" + +run_redundant() { + bash "$REDUNDANT_SH" --repo "$1" --manifest "$2" +} + +write_gitmodules() { + local repo="$1" + shift + local pair + : >"$repo/.gitmodules" + for pair in "$@"; do + local spath="${pair%%=*}" + local url="${pair#*=}" + { + printf '[submodule "%s"]\n' "$spath" + printf '\tpath = %s\n' "$spath" + printf '\turl = %s\n' "$url" + } >>"$repo/.gitmodules" + done +} + +expect_redundant() { + local repo="$1" man="$2" + if run_redundant "$repo" "$man"; then + return 0 + fi + fail "expected exit 0 (submodulize redundant), got 1 — repo=$repo manifest=$man" +} + +expect_not_redundant() { + local repo="$1" man="$2" + if ! run_redundant "$repo" "$man"; then + return 0 + fi + fail "expected exit 1 (submodulize needed), got 0 — repo=$repo manifest=$man" +} + +TMP_BASE="$(mktemp -d "${TMPDIR:-/tmp}/manifest-submod-redundant.XXXXXX")" +trap 'rm -rf "$TMP_BASE"' EXIT + +# --- all manifest paths in .gitmodules → redundant (exit 0) --- +R1="$TMP_BASE/r1" +mkdir -p "$R1" +git -C "$R1" init -q +write_gitmodules "$R1" \ + "mod/foo=https://example.com/foo.git" \ + "blocks/bar=https://example.com/bar.git" +M1="$TMP_BASE/m1.manifest" +{ + printf '# comment\n' + printf 'mod/foo|https://example.com/foo.git|main\n' + printf ' blocks/bar |https://example.com/bar.git|main\n' +} >"$M1" +expect_redundant "$R1" "$M1" + +# --- one path missing from .gitmodules → not redundant --- +R2="$TMP_BASE/r2" +mkdir -p "$R2" +git -C "$R2" init -q +write_gitmodules "$R2" "mod/foo=https://example.com/foo.git" +M2="$TMP_BASE/m2.manifest" +printf 'mod/foo|u|main\nmod/missing|u|main\n' >"$M2" +expect_not_redundant "$R2" "$M2" + +# --- no .gitmodules → not redundant --- +R3="$TMP_BASE/r3" +mkdir -p "$R3" +git -C "$R3" init -q +M3="$TMP_BASE/m3.manifest" +printf 'mod/foo|u|main\n' >"$M3" +expect_not_redundant "$R3" "$M3" + +# --- empty .gitmodules → not redundant ([ -s .gitmodules ] false) --- +R4="$TMP_BASE/r4" +mkdir -p "$R4" +git -C "$R4" init -q +: >"$R4/.gitmodules" +M4="$TMP_BASE/m4.manifest" +printf 'mod/foo|u|main\n' >"$M4" +expect_not_redundant "$R4" "$M4" + +# --- manifest only comments/blank → not redundant (no -s .gitmodules match with vacuous need_submod=0) --- +R5="$TMP_BASE/r5" +mkdir -p "$R5" +git -C "$R5" init -q +write_gitmodules "$R5" "mod/foo=https://example.com/foo.git" +M5="$TMP_BASE/m5.manifest" +printf '# only comments\n\n# foo\n' >"$M5" +expect_not_redundant "$R5" "$M5" + +ok "manifest-submodulize-redundant.sh (skip vs run submodulize)" diff --git a/cleandev/tests/test-shellcheck.sh b/cleandev/tests/test-shellcheck.sh index 725db6a..9154d55 100644 --- a/cleandev/tests/test-shellcheck.sh +++ b/cleandev/tests/test-shellcheck.sh @@ -15,5 +15,5 @@ fi # Run from this directory so `source=lib.sh` directives resolve (shellcheck -x). cd "$SCRIPT_DIR" -shellcheck -x lib.sh run.sh test-*.sh ../submodulize.sh ../unsubmodulize.sh +shellcheck -x lib.sh run.sh test-*.sh ../submodulize.sh ../unsubmodulize.sh ../manifest-submodulize-redundant.sh ok "shellcheck" diff --git a/new.sh b/new.sh index 2252ed1..01df0be 100755 --- a/new.sh +++ b/new.sh @@ -168,6 +168,7 @@ while [[ $# -gt 0 ]]; do -h --help Shows this text. -s --skip Skip automatic Moodle installation. --submodulize After merge, use cleandev/submodulize.sh for manifest plugins (submodules). + Safe on cleandev-style trees: paths already in .gitmodules are skipped (no bulk no-op run). GitHub auth (private lsuonline/*): GITHUB_TOKEN or GH_TOKEN, file cleandev/.github-token, or an interactive prompt (first run) saves the token there (gitignored). Alternatively SUBMODULIZE_SSH=1 with SSH usable inside the container."; @@ -208,15 +209,16 @@ BRANCH_NAME=$NAME docker compose -p "${NAME}" up -d # with block_lsu_people). skip-worktree on ues_people and config.php for a clean git status after startup. # enrol/workdaystudent and blocks/wdsprefs come from the lsuce-moodle tree (or --submodulize manifest). if [ "$SUBMODULIZE" = true ]; then - if [ ! -f "${PROJECT_ROOT}/cleandev/submodulize.sh" ] || [ ! -f "${PROJECT_ROOT}/cleandev/plugin-submodules.manifest" ]; then - echo "new.sh --submodulize requires cleandev/submodulize.sh and cleandev/plugin-submodules.manifest next to new.sh." >&2 + if [ ! -f "${PROJECT_ROOT}/cleandev/submodulize.sh" ] || [ ! -f "${PROJECT_ROOT}/cleandev/manifest-submodulize-redundant.sh" ] || [ ! -f "${PROJECT_ROOT}/cleandev/plugin-submodules.manifest" ]; then + echo "new.sh --submodulize requires cleandev/submodulize.sh, cleandev/manifest-submodulize-redundant.sh, and cleandev/plugin-submodules.manifest next to new.sh." >&2 exit 1 fi resolve_submod_github_token MSYS_NO_PATHCONV=1 docker cp "$(host_path_for_docker_cp "${PROJECT_ROOT}/cleandev/submodulize.sh")" "${NAME}-moodle:/tmp/submodulize.sh" + MSYS_NO_PATHCONV=1 docker cp "$(host_path_for_docker_cp "${PROJECT_ROOT}/cleandev/manifest-submodulize-redundant.sh")" "${NAME}-moodle:/tmp/manifest-submodulize-redundant.sh" MSYS_NO_PATHCONV=1 docker cp "$(host_path_for_docker_cp "${PROJECT_ROOT}/cleandev/plugin-submodules.manifest")" "${NAME}-moodle:/tmp/plugin-submodules.manifest" # docker cp leaves root-owned files; sed -i as www-data fails with "cannot rename: Operation not permitted" - MSYS_NO_PATHCONV=1 docker exec "${NAME}-moodle" sh -c 'sed -i '"'"'s/\r$//'"'"' /tmp/submodulize.sh /tmp/plugin-submodules.manifest && chmod +x /tmp/submodulize.sh && chown www-data:www-data /tmp/submodulize.sh /tmp/plugin-submodules.manifest' + MSYS_NO_PATHCONV=1 docker exec "${NAME}-moodle" sh -c 'sed -i '"'"'s/\r$//'"'"' /tmp/submodulize.sh /tmp/manifest-submodulize-redundant.sh /tmp/plugin-submodules.manifest && chmod +x /tmp/submodulize.sh /tmp/manifest-submodulize-redundant.sh && chown www-data:www-data /tmp/submodulize.sh /tmp/manifest-submodulize-redundant.sh /tmp/plugin-submodules.manifest' submod_docker_env=(-u www-data) if [ -n "$SUBMOD_GIT_TOKEN" ]; then submod_docker_env+=(-e "GITHUB_TOKEN=${SUBMOD_GIT_TOKEN}") @@ -224,20 +226,25 @@ if [ "$SUBMODULIZE" = true ]; then if [ "${SUBMODULIZE_SSH:-}" = "1" ]; then submod_docker_env+=(-e "SUBMODULIZE_USE_SSH=1") fi - MSYS_NO_PATHCONV=1 docker exec "${submod_docker_env[@]}" "${NAME}-moodle" sh -c ' - cd /var/www/html && \ - git config --add safe.directory /var/www/html && \ - if [ -n "${GITHUB_TOKEN:-}" ]; then \ - git config --local url."https://${GITHUB_TOKEN}@github.com/".insteadOf "https://github.com/"; \ - fi && \ - rm -rf blocks/ues_people && \ - git fetch origin develop && \ - git merge origin/develop && \ - git ls-files blocks/ues_people | xargs -r git update-index --skip-worktree && \ - rm -rf blocks/ues_people && \ - SMF="--no-commit" && \ - if [ "${SUBMODULIZE_USE_SSH:-}" = "1" ]; then SMF="$SMF --ssh"; fi && \ - bash /tmp/submodulize.sh --repo /var/www/html --manifest /tmp/plugin-submodules.manifest $SMF && \ + MSYS_NO_PATHCONV=1 docker exec "${submod_docker_env[@]}" "${NAME}-moodle" bash -c ' + set -euo pipefail + cd /var/www/html + git config --add safe.directory /var/www/html + if [ -n "${GITHUB_TOKEN:-}" ]; then + git config --local url."https://${GITHUB_TOKEN}@github.com/".insteadOf "https://github.com/" + fi + rm -rf blocks/ues_people + git fetch origin develop + git merge origin/develop + git ls-files blocks/ues_people | xargs -r git update-index --skip-worktree + rm -rf blocks/ues_people + SMF="--no-commit" + if [ "${SUBMODULIZE_USE_SSH:-}" = "1" ]; then SMF="$SMF --ssh"; fi + if bash /tmp/manifest-submodulize-redundant.sh --repo /var/www/html --manifest /tmp/plugin-submodules.manifest; then + echo "new.sh: All manifest plugin paths are already submodules; skipping submodulize.sh." + else + bash /tmp/submodulize.sh --repo /var/www/html --manifest /tmp/plugin-submodules.manifest $SMF + fi git update-index --skip-worktree config.php ' else From cb250eed4b24547171c133c7b06f97407aa1a721 Mon Sep 17 00:00:00 2001 From: Steve Mattsen Date: Tue, 14 Apr 2026 14:34:42 -0400 Subject: [PATCH 03/18] Improve `submodulize.sh` usage instructions and error messages. Allow passing the Moodle root as a bare argument and clarify command usage in the help output. --- cleandev/submodulize.sh | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/cleandev/submodulize.sh b/cleandev/submodulize.sh index ade80b6..9c6714d 100644 --- a/cleandev/submodulize.sh +++ b/cleandev/submodulize.sh @@ -20,6 +20,29 @@ NO_COMMIT=false USE_SSH=false REPO_ROOT="" +usage() { + cat <<'EOF' +Convert vendored plugin directories in the Moodle repo into git submodules. +Run from anywhere inside the Moodle clone (uses repo root unless --repo is set). + +Usage: + submodulize.sh [--dry-run] [--no-commit] [--ssh] [--manifest PATH] [--repo ROOT] + +Options: + --dry-run Print actions without changing the repo + --no-commit Stage submodule changes but do not commit + --ssh Use git@github.com URLs for github.com HTTPS entries + --manifest PATH Use a manifest file (default: cleandev/plugin-submodules.manifest next to this script) + --repo ROOT Moodle git root (default: git rev-parse --show-toplevel) + +Requires: git, and a clean enough working tree (commit or stash if plugin paths are dirty). + +Private GitHub over HTTPS: set GITHUB_TOKEN (PAT) so ls-remote / submodule add can authenticate, +or use --ssh. Without credentials in non-interactive environments you may see: + could not read Username for 'https://github.com' +EOF +} + while [[ $# -gt 0 ]]; do case "$1" in --dry-run) DRY_RUN=true; shift ;; @@ -34,7 +57,7 @@ while [[ $# -gt 0 ]]; do shift 2 ;; -h|--help) - sed -n '1,25p' "$0" + usage exit 0 ;; *) From 0fbd3dcd40d9b69e27042e37c11f34b01edc3216 Mon Sep 17 00:00:00 2001 From: Steve Mattsen Date: Tue, 14 Apr 2026 14:34:48 -0400 Subject: [PATCH 04/18] Refine `submodulize.sh` and `unsubmodulize.sh` usage instructions to clarify the passing of the Moodle root as a bare argument. Update error messages for better user guidance. Enhance `TEAM-PROCESS.md` with detailed one-shot procedures for managing vendored monorepo plugins. --- ROADMAP.md | 2 +- cleandev/TEAM-PROCESS.md | 28 +++++++++++++++++++- cleandev/submodulize.sh | 33 +++++++++++++++++++----- cleandev/unsubmodulize.sh | 54 +++++++++++++++++++++++++++++++++++---- 4 files changed, 103 insertions(+), 14 deletions(-) diff --git a/ROADMAP.md b/ROADMAP.md index 8c767c9..111a206 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -18,7 +18,7 @@ Progress toward **seamless `develop` ↔ `cleandev`** (on [lsuce-moodle](https:/ - [ ] **Commit portability:** Script or documented workflow to **replay or port commits** across layouts with a **header** noting the original commit (and branch/repo). *Not in repo today—only bulk layout conversion exists.* - [x] **`new.sh` / tooling:** With `--submodulize`, **detect** when every manifest path is already a submodule (cleandev-style) and **skip** `submodulize.sh` instead of a no-op pass. *Not done: auto-run conversion without passing `--submodulize`.* - [x] **`unsubmodulize.sh`:** Same `GITHUB_TOKEN` `-c url.insteadOf` pattern as `submodulize.sh` for private HTTPS (parity with submodule add). -- [ ] **Monorepo manifest + scripts:** Support “clone once, map subpaths”, or publish **one-shot manual procedures** for commented monorepo lines. +- [x] **Monorepo manifest + scripts:** **One-shot manual procedures** for commented monorepo lines: [cleandev/TEAM-PROCESS.md](cleandev/TEAM-PROCESS.md) (manifest regeneration → Monorepos). *Not done: automated “clone once, map subpaths” in `submodulize.sh`.* - [ ] **`local/ml`:** Point at the correct Git remote when known; uncomment or add manifest line. - [x] **CI (cleandev):** GitHub Actions runs `cleandev/tests/run.sh` (manifest lint, round-trip integration tests, shellcheck on Ubuntu). - [ ] **Optional CI:** Check that manifest paths still match **lsuce-moodle** `develop` plugin layout when plugins move. diff --git a/cleandev/TEAM-PROCESS.md b/cleandev/TEAM-PROCESS.md index 62279f1..2eb7601 100644 --- a/cleandev/TEAM-PROCESS.md +++ b/cleandev/TEAM-PROCESS.md @@ -90,7 +90,33 @@ Do **not** add separate submodule lines per element subdirectory. ### Monorepos -If **one Git repository** contains **multiple** top-level Moodle plugin paths (e.g. Kaltura, Microsoft o365-moodle, `lsu-enrol_ues`), **do not** add one manifest line per path with the same URL: `submodulize.sh` cannot “clone once, map subpaths.” Keep a **commented block** in `plugin-submodules.manifest` listing the URL and paths (see existing `# --- Monorepos ---` section), and leave those plugins **vendored** or handle them with a **manual** procedure until tooling supports it. +If **one Git repository** contains **multiple** top-level Moodle plugin paths (e.g. Kaltura, Microsoft o365-moodle, `lsu-enrol_ues`), **do not** add one manifest line per path with the same URL: `submodulize.sh` cannot “clone once, map subpaths.” Keep a **commented block** in `plugin-submodules.manifest` listing the URL and paths (see `# --- Monorepos ---` in the manifest). Treat those directories as **vendored** in the superproject unless you adopt an advanced layout (see below). + +#### One-shot: refresh vendored monorepo plugins (develop or cleandev) + +Use this when upstream shipped changes and you need the same Moodle paths updated without submodule tooling. + +1. **Note the source** from the manifest comment block (clone URL and the list of Moodle-relative paths, e.g. `mod/kalvidassign`). +2. **Clone upstream** somewhere outside the Moodle tree (temp is fine), on the branch or tag you intend to ship: + ```bash + git clone --depth 1 -b BRANCH_OR_TAG https://github.com/org/monorepo.git /tmp/monorepo-src + ``` + Use a full clone if you need history or a non-default branch tip. +3. **Map paths** — upstream layout varies by project. Under `/tmp/monorepo-src`, locate the directory that corresponds to each Moodle path (often the path matches the repo tree; if the project nests plugins under `moodle/` or similar, copy from there). +4. **Copy into lsuce-moodle** from the Moodle repo root, one path at a time (adjust source side to match step 3): + ```bash + rsync -a --delete /tmp/monorepo-src/mod/kalvidassign/ mod/kalvidassign/ + ``` + Prefer `--delete` only when you intend to mirror upstream exactly; otherwise omit it. On Windows without `rsync`, use a graphical diff tool or `cp -r` with care. +5. **Review** (`git status`, smoke test in Moodle), then **commit** in **lsuce-moodle** (single commit per upstream bump or per path—follow team convention). + +Repeat for each path listed under that URL in the manifest comment. Bundles such as **Kaltura** (`moodle_plugin`), **Kaltura gallery** (`moodle-local_kalturamediagallery`), **o365-moodle**, and **lsu-enrol_ues** are independent clones; refresh each comment group from its own remote. + +#### One-shot: cleandev when monorepo paths stay vendored + +On **cleandev**, submodule-backed plugins use `.gitmodules`; monorepo-backed paths stay **normal tracked directories** in the superproject (no submodule entry). Do not add duplicate manifest active lines for the same URL. After `git submodule update --init`, those paths behave like core tree: edit, commit, and push on **lsuce-moodle** unless your team splits them out later. + +*Automated “clone once, map subpaths” for the manifest is still not implemented; the steps above are the supported manual approach.* ### No usable HTTPS URL diff --git a/cleandev/submodulize.sh b/cleandev/submodulize.sh index 9c6714d..52dcf81 100644 --- a/cleandev/submodulize.sh +++ b/cleandev/submodulize.sh @@ -1,11 +1,12 @@ #!/usr/bin/env bash # Convert vendored plugin directories (plain files in the Moodle repo, e.g. lsuce-moodle develop) -# into git submodules. Run from anywhere inside the Moodle clone; uses repo root. +# into git submodules. Run from inside the clone, or pass the clone path as ROOT / --repo. # # Requires: git, a clean enough working tree (commit or stash first if paths are dirty). # # Usage: -# ./cleandev/submodulize.sh [--dry-run] [--no-commit] [--ssh] [--manifest PATH] [--repo ROOT] +# ./cleandev/submodulize.sh [ROOT] [--dry-run] [--no-commit] [--ssh] [--manifest PATH] [--repo ROOT] +# Bare ROOT is the same as --repo (optional; may appear before or after flags). # # Private GitHub repos over HTTPS need credentials. Set GITHUB_TOKEN (PAT) so HTTPS URLs are rewritten # for ls-remote / submodule add (parent repo url.insteadOf is not always applied to submodule clone). @@ -23,17 +24,27 @@ REPO_ROOT="" usage() { cat <<'EOF' Convert vendored plugin directories in the Moodle repo into git submodules. -Run from anywhere inside the Moodle clone (uses repo root unless --repo is set). +Moodle root defaults to the current directory’s git superproject (git rev-parse --show-toplevel), unless you set it explicitly. Usage: - submodulize.sh [--dry-run] [--no-commit] [--ssh] [--manifest PATH] [--repo ROOT] + submodulize.sh [ROOT] [OPTIONS...] + submodulize.sh [OPTIONS...] [ROOT] + + ROOT — optional path to the Moodle git checkout. Give it as a single bare argument (no leading -), + anywhere among the flags; same meaning as --repo. Only one repo path: do not pass a second + bare path, and do not put a bare path after --repo (that is rejected). + +Examples: + submodulize.sh ~/workspace/moodle + submodulize.sh --dry-run ~/workspace/moodle + submodulize.sh ~/workspace/moodle --no-commit --ssh Options: --dry-run Print actions without changing the repo --no-commit Stage submodule changes but do not commit --ssh Use git@github.com URLs for github.com HTTPS entries --manifest PATH Use a manifest file (default: cleandev/plugin-submodules.manifest next to this script) - --repo ROOT Moodle git root (default: git rev-parse --show-toplevel) + --repo ROOT Moodle git root (explicit form of a bare ROOT; overrides an earlier bare ROOT; a bare path after --repo is an error) Requires: git, and a clean enough working tree (commit or stash if plugin paths are dirty). @@ -60,10 +71,18 @@ while [[ $# -gt 0 ]]; do usage exit 0 ;; - *) + -*) echo "Unknown option: $1" >&2 exit 1 ;; + *) + if [[ -n "$REPO_ROOT" ]]; then + echo "Unexpected argument: $1 (repo already set via --repo or positional path)" >&2 + exit 1 + fi + REPO_ROOT="$1" + shift + ;; esac done @@ -74,7 +93,7 @@ fi if [[ -z "$REPO_ROOT" ]]; then REPO_ROOT="$(git rev-parse --show-toplevel 2>/dev/null)" || { - echo "Not inside a git repository. Use --repo /path/to/moodle" >&2 + echo "Not inside a git repository. Pass the Moodle root as a bare path or use --repo /path/to/moodle" >&2 exit 1 } fi diff --git a/cleandev/unsubmodulize.sh b/cleandev/unsubmodulize.sh index 2e00cff..9983216 100644 --- a/cleandev/unsubmodulize.sh +++ b/cleandev/unsubmodulize.sh @@ -1,10 +1,11 @@ #!/usr/bin/env bash # Replace git submodules with plain tracked directories (vendored plugins), matching lsuce-moodle # develop style. Clones each plugin repo at the given branch, drops nested .git, and adds files -# to the parent Moodle repository. +# to the parent Moodle repository. Run from inside the clone, or pass the clone path as ROOT / --repo. # # Usage: -# ./cleandev/unsubmodulize.sh [--dry-run] [--no-commit] [--ssh] [--manifest PATH] [--repo ROOT] +# ./cleandev/unsubmodulize.sh [ROOT] [--dry-run] [--no-commit] [--ssh] [--manifest PATH] [--repo ROOT] +# Bare ROOT is the same as --repo (optional; may appear before or after flags). # # Private GitHub repos over HTTPS: set GITHUB_TOKEN (PAT) so ls-remote / clone use # -c url.https://TOKEN@github.com/.insteadOf=https://github.com/ @@ -19,6 +20,41 @@ NO_COMMIT=false USE_SSH=false REPO_ROOT="" +usage() { + cat <<'EOF' +Replace git submodules listed in the manifest with plain tracked directories (vendored plugins). +For each entry: clones the plugin at the manifest branch, removes nested .git, deinits the submodule, +and stages the tree in the parent Moodle repo. + +Moodle root defaults to the current directory’s git superproject (git rev-parse --show-toplevel), unless you set it explicitly. + +Usage: + unsubmodulize.sh [ROOT] [OPTIONS...] + unsubmodulize.sh [OPTIONS...] [ROOT] + + ROOT — optional path to the Moodle git checkout. Give it as a single bare argument (no leading -), + anywhere among the flags; same meaning as --repo. Only one repo path: do not pass a second + bare path, and do not put a bare path after --repo (that is rejected). + +Examples: + unsubmodulize.sh ~/workspace/moodle + unsubmodulize.sh --dry-run ~/workspace/moodle + unsubmodulize.sh ~/workspace/moodle --no-commit --ssh + +Options: + --dry-run Print what would happen without changing the repo + --no-commit Stage vendored trees but do not commit + --ssh Use git@github.com URLs for github.com HTTPS entries + --manifest PATH Use a manifest file (default: cleandev/plugin-submodules.manifest next to this script) + --repo ROOT Moodle git root (explicit form of a bare ROOT; overrides an earlier bare ROOT; a bare path after --repo is an error) + +Requires: git, and clean submodule working trees for paths being converted (no uncommitted changes inside submodules). + +Private GitHub over HTTPS: set GITHUB_TOKEN (PAT) so ls-remote / clone can authenticate (same pattern as submodulize.sh), +or use --ssh. +EOF +} + while [[ $# -gt 0 ]]; do case "$1" in --dry-run) DRY_RUN=true; shift ;; @@ -33,13 +69,21 @@ while [[ $# -gt 0 ]]; do shift 2 ;; -h|--help) - sed -n '1,25p' "$0" + usage exit 0 ;; - *) + -*) echo "Unknown option: $1" >&2 exit 1 ;; + *) + if [[ -n "$REPO_ROOT" ]]; then + echo "Unexpected argument: $1 (repo already set via --repo or positional path)" >&2 + exit 1 + fi + REPO_ROOT="$1" + shift + ;; esac done @@ -50,7 +94,7 @@ fi if [[ -z "$REPO_ROOT" ]]; then REPO_ROOT="$(git rev-parse --show-toplevel 2>/dev/null)" || { - echo "Not inside a git repository. Use --repo /path/to/moodle" >&2 + echo "Not inside a git repository. Pass the Moodle root as a bare path or use --repo /path/to/moodle" >&2 exit 1 } fi From 524153521394cd96e76b4939350bc1f8cc4f0d1e Mon Sep 17 00:00:00 2001 From: Steve Mattsen Date: Tue, 14 Apr 2026 15:07:08 -0400 Subject: [PATCH 05/18] Refactor `new.sh`, `submodulize.sh`, and `unsubmodulize.sh` to streamline manifest handling. Update scripts to default to `ROOT/plugin-submodules.manifest` if no manifest is specified. Enhance README and TEAM-PROCESS.md with clarifications on manifest usage and script behavior. Improve test cases to reflect new default manifest path. --- README.md | 4 +- cleandev/TEAM-PROCESS.md | 2 + cleandev/manifest-submodulize-redundant.sh | 17 ++++++-- cleandev/submodulize.sh | 32 +++++++++------ cleandev/tests/test-manifest-default-repo.sh | 40 +++++++++++++++++++ .../tests/test-manifest-submod-redundant.sh | 28 ++++++++++--- cleandev/tests/test-sparse-submod.sh | 5 +-- cleandev/tests/test-submod-roundtrip.sh | 11 +++-- cleandev/unsubmodulize.sh | 30 +++++++++----- demo/plugin1 | 1 + demo/plugin2 | 1 + demo/plugin3 | 1 + demo/raw | 1 + demo/sub | 1 + new.sh | 5 ++- 15 files changed, 135 insertions(+), 44 deletions(-) create mode 100644 cleandev/tests/test-manifest-default-repo.sh create mode 160000 demo/plugin1 create mode 160000 demo/plugin2 create mode 160000 demo/plugin3 create mode 160000 demo/raw create mode 160000 demo/sub diff --git a/README.md b/README.md index b729a36..484b817 100644 --- a/README.md +++ b/README.md @@ -96,6 +96,8 @@ These are **branches on the Moodle repo** (`lsuce-moodle`), not branch names in ## Manifest (`cleandev/plugin-submodules.manifest`) +This repo keeps a **canonical copy** under `cleandev/` for linting and for `new.sh --submodulize` (copied into the Moodle tree at runtime). When you run `submodulize.sh` / `unsubmodulize.sh` against a checkout, the default manifest path is **`plugin-submodules.manifest` at the Moodle superproject root** (`--repo` / current directory), not next to the scripts; use `--manifest PATH` to override. + - Format: `relative_path|clone_url|branch` (e.g. `mod/hvp|https://github.com/...|main`). Lines starting with `#` are ignored. - If the third field is empty, scripts default the branch to `main`; if `refs/heads/` is missing on the remote, they omit `-b` and use the remote’s default branch. - **Active lines:** one Git repo root per Moodle path (works with `git submodule add` and shallow clone + copy). @@ -128,7 +130,7 @@ bash cleandev/tests/run.sh - Compose project name = first argument (containers `{NAME}-moodle`, etc.). - Web service builds from `.` per `docker-compose.yml`. - As `www-data`: `git fetch` / `git merge origin/develop`; removes `blocks/ues_people` and uses `skip-worktree` so it does not clash with `block_lsu_people` (see `config/moodle-pull` for the same idea on `git pull`). -- With `--submodulize`: copies `submodulize.sh` + manifest into the container, resolves GitHub token, sets local `url...insteadOf` when using HTTPS token, runs `submodulize.sh --repo /var/www/html ... --no-commit` (optional SSH via `SUBMODULIZE_SSH=1`). +- With `--submodulize`: copies scripts into the container, stages `cleandev/plugin-submodules.manifest` into `/var/www/html/plugin-submodules.manifest` after the merge, resolves GitHub token, sets local `url...insteadOf` when using HTTPS token, runs `manifest-submodulize-redundant.sh` / `submodulize.sh` with default manifest paths (`--repo /var/www/html`, optional SSH via `SUBMODULIZE_SSH=1`). Secrets: `cleandev/.github-token` is listed in `.gitignore`. diff --git a/cleandev/TEAM-PROCESS.md b/cleandev/TEAM-PROCESS.md index 2eb7601..61dbc0e 100644 --- a/cleandev/TEAM-PROCESS.md +++ b/cleandev/TEAM-PROCESS.md @@ -2,6 +2,8 @@ This document covers **branch policy** for [lsuce-moodle](https://github.com/lsuonline/lsuce-moodle), the **Docker image decision** for this repo, and **how to refresh** `plugin-submodules.manifest` from the plugin inventory CSV. +**Where the manifest lives:** On **cleandev**, treat `plugin-submodules.manifest` as part of the Moodle superproject (repo root), alongside `.gitmodules`. The copy in **this** repo under [`cleandev/plugin-submodules.manifest`](plugin-submodules.manifest) is the maintained source for CSV refresh and CI lint; `submodulize.sh` / `unsubmodulize.sh` default to `ROOT/plugin-submodules.manifest` unless you pass `--manifest`. + --- ## 1. Team process (develop ↔ cleandev) diff --git a/cleandev/manifest-submodulize-redundant.sh b/cleandev/manifest-submodulize-redundant.sh index 9ae802d..2c3f33a 100644 --- a/cleandev/manifest-submodulize-redundant.sh +++ b/cleandev/manifest-submodulize-redundant.sh @@ -2,12 +2,14 @@ # Exit 0 if running submodulize.sh would be a no-op: every active manifest path is already listed # in .gitmodules and .gitmodules is non-empty. Exit 1 otherwise (submodulize should run). # Used by new.sh --submodulize and cleandev/tests. +# Default manifest: ROOT/plugin-submodules.manifest (same as submodulize.sh). # -# Usage: ./cleandev/manifest-submodulize-redundant.sh --repo ROOT --manifest PATH +# Usage: ./cleandev/manifest-submodulize-redundant.sh --repo ROOT [--manifest PATH] set -euo pipefail REPO_ROOT="" MANIFEST="" +MANIFEST_EXPLICIT=false while [[ $# -gt 0 ]]; do case "$1" in @@ -17,10 +19,11 @@ while [[ $# -gt 0 ]]; do ;; --manifest) MANIFEST="${2:?}" + MANIFEST_EXPLICIT=true shift 2 ;; -h|--help) - sed -n '1,12p' "$0" + sed -n '1,13p' "$0" exit 0 ;; *) @@ -30,10 +33,16 @@ while [[ $# -gt 0 ]]; do esac done -[[ -n "$REPO_ROOT" && -n "$MANIFEST" ]] || { - echo "Usage: $0 --repo ROOT --manifest PATH" >&2 +[[ -n "$REPO_ROOT" ]] || { + echo "Usage: $0 --repo ROOT [--manifest PATH]" >&2 + echo " Default manifest: ROOT/plugin-submodules.manifest" >&2 exit 2 } + +if ! $MANIFEST_EXPLICIT; then + MANIFEST="${REPO_ROOT%/}/plugin-submodules.manifest" +fi + [[ -f "$MANIFEST" ]] || { echo "Manifest not found: $MANIFEST" >&2 exit 2 diff --git a/cleandev/submodulize.sh b/cleandev/submodulize.sh index 52dcf81..a2ee722 100644 --- a/cleandev/submodulize.sh +++ b/cleandev/submodulize.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash -# Convert vendored plugin directories (plain files in the Moodle repo, e.g. lsuce-moodle develop) -# into git submodules. Run from inside the clone, or pass the clone path as ROOT / --repo. +# Convert vendored plugin directories (plain files in the Moodle superproject) into git submodules. +# Intended for the submodule-layout Moodle checkout: keep plugin-submodules.manifest at the superproject +# root (same repo as .gitmodules will live in). Run from inside that clone, or pass ROOT / --repo. # # Requires: git, a clean enough working tree (commit or stash first if paths are dirty). # @@ -14,8 +15,8 @@ set -euo pipefail -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -MANIFEST="${SCRIPT_DIR}/plugin-submodules.manifest" +MANIFEST="" +MANIFEST_EXPLICIT=false DRY_RUN=false NO_COMMIT=false USE_SSH=false @@ -23,8 +24,10 @@ REPO_ROOT="" usage() { cat <<'EOF' -Convert vendored plugin directories in the Moodle repo into git submodules. -Moodle root defaults to the current directory’s git superproject (git rev-parse --show-toplevel), unless you set it explicitly. +Convert vendored plugin directories in the Moodle superproject into git submodules (cleandev-style). +The manifest lists paths and clone URLs; it normally lives in the superproject root as +plugin-submodules.manifest (not under cleandev/). Moodle root defaults to the current directory’s +git superproject (git rev-parse --show-toplevel), unless you set it explicitly. Usage: submodulize.sh [ROOT] [OPTIONS...] @@ -43,7 +46,7 @@ Options: --dry-run Print actions without changing the repo --no-commit Stage submodule changes but do not commit --ssh Use git@github.com URLs for github.com HTTPS entries - --manifest PATH Use a manifest file (default: cleandev/plugin-submodules.manifest next to this script) + --manifest PATH Plugin manifest (default: ROOT/plugin-submodules.manifest) --repo ROOT Moodle git root (explicit form of a bare ROOT; overrides an earlier bare ROOT; a bare path after --repo is an error) Requires: git, and a clean enough working tree (commit or stash if plugin paths are dirty). @@ -61,6 +64,7 @@ while [[ $# -gt 0 ]]; do --ssh) USE_SSH=true; shift ;; --manifest) MANIFEST="${2:?}" + MANIFEST_EXPLICIT=true shift 2 ;; --repo) @@ -86,11 +90,6 @@ while [[ $# -gt 0 ]]; do esac done -if [[ ! -f "$MANIFEST" ]]; then - echo "Manifest not found: $MANIFEST" >&2 - exit 1 -fi - if [[ -z "$REPO_ROOT" ]]; then REPO_ROOT="$(git rev-parse --show-toplevel 2>/dev/null)" || { echo "Not inside a git repository. Pass the Moodle root as a bare path or use --repo /path/to/moodle" >&2 @@ -98,6 +97,15 @@ if [[ -z "$REPO_ROOT" ]]; then } fi +if ! $MANIFEST_EXPLICIT; then + MANIFEST="${REPO_ROOT%/}/plugin-submodules.manifest" +fi + +if [[ ! -f "$MANIFEST" ]]; then + echo "Manifest not found: $MANIFEST" >&2 + exit 1 +fi + cd "$REPO_ROOT" # Moodle dev images often use sparse-checkout (cone, index.sparse, or only .git/info/sparse-checkout). diff --git a/cleandev/tests/test-manifest-default-repo.sh b/cleandev/tests/test-manifest-default-repo.sh new file mode 100644 index 0000000..ec930ee --- /dev/null +++ b/cleandev/tests/test-manifest-default-repo.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +# Default manifest path: ROOT/plugin-submodules.manifest when --manifest is omitted. +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=lib.sh +source "$SCRIPT_DIR/lib.sh" +require_cleandev + +TMP=$(mktemp -d) +export HOME="$TMP/_home" +mkdir -p "$HOME" +trap 'rm -rf "$TMP"' EXIT + +MOODLE="$TMP/moodle" +mkdir -p "$MOODLE" +git -C "$MOODLE" init -b develop -q +echo "core" >"$MOODLE/README.txt" +git -C "$MOODLE" add README.txt +git -C "$MOODLE" commit -q -m "init" + +# No plugin-submodules.manifest at repo root → scripts fail +if "$CLEANDEV/submodulize.sh" --repo "$MOODLE" --dry-run 2>/dev/null; then + fail "expected submodulize to fail when default manifest is missing" +fi +if "$CLEANDEV/unsubmodulize.sh" --repo "$MOODLE" --dry-run 2>/dev/null; then + fail "expected unsubmodulize to fail when default manifest is missing" +fi + +set +e +sub_out=$("$CLEANDEV/submodulize.sh" --repo "$MOODLE" --dry-run 2>&1) +sub_ec=$? +red_out=$(bash "$CLEANDEV/manifest-submodulize-redundant.sh" --repo "$MOODLE" 2>&1) +red_ec=$? +set -e +[[ "$sub_ec" -ne 0 ]] || fail "expected submodulize nonzero exit" +[[ "$red_ec" -ne 0 ]] || fail "expected manifest-submodulize-redundant nonzero exit" +echo "$sub_out" | grep -Fq "Manifest not found" || fail "expected submodulize stderr to mention Manifest not found" +echo "$red_out" | grep -Fq "Manifest not found" || fail "expected redundant stderr to mention Manifest not found" + +ok "default manifest path requires ROOT/plugin-submodules.manifest when --manifest omitted" diff --git a/cleandev/tests/test-manifest-submod-redundant.sh b/cleandev/tests/test-manifest-submod-redundant.sh index ed71b34..aca33ba 100644 --- a/cleandev/tests/test-manifest-submod-redundant.sh +++ b/cleandev/tests/test-manifest-submod-redundant.sh @@ -10,7 +10,13 @@ REDUNDANT_SH="$CLEANDEV/manifest-submodulize-redundant.sh" assert_file "$REDUNDANT_SH" run_redundant() { - bash "$REDUNDANT_SH" --repo "$1" --manifest "$2" + local repo="$1" + local man="${2:-}" + if [[ -n "$man" ]]; then + bash "$REDUNDANT_SH" --repo "$repo" --manifest "$man" + else + bash "$REDUNDANT_SH" --repo "$repo" + fi } write_gitmodules() { @@ -30,19 +36,21 @@ write_gitmodules() { } expect_redundant() { - local repo="$1" man="$2" + local repo="$1" + local man="${2:-}" if run_redundant "$repo" "$man"; then return 0 fi - fail "expected exit 0 (submodulize redundant), got 1 — repo=$repo manifest=$man" + fail "expected exit 0 (submodulize redundant), got 1 — repo=$repo manifest=${man:-}" } expect_not_redundant() { - local repo="$1" man="$2" + local repo="$1" + local man="${2:-}" if ! run_redundant "$repo" "$man"; then return 0 fi - fail "expected exit 1 (submodulize needed), got 0 — repo=$repo manifest=$man" + fail "expected exit 1 (submodulize needed), got 0 — repo=$repo manifest=${man:-}" } TMP_BASE="$(mktemp -d "${TMPDIR:-/tmp}/manifest-submod-redundant.XXXXXX")" @@ -98,4 +106,14 @@ M5="$TMP_BASE/m5.manifest" printf '# only comments\n\n# foo\n' >"$M5" expect_not_redundant "$R5" "$M5" +# --- default manifest at ROOT/plugin-submodules.manifest (no --manifest flag) --- +R6="$TMP_BASE/r6" +mkdir -p "$R6" +git -C "$R6" init -q +write_gitmodules "$R6" \ + "mod/foo=https://example.com/foo.git" \ + "blocks/bar=https://example.com/bar.git" +printf 'mod/foo|https://example.com/foo.git|main\nblocks/bar|https://example.com/bar.git|main\n' >"$R6/plugin-submodules.manifest" +expect_redundant "$R6" + ok "manifest-submodulize-redundant.sh (skip vs run submodulize)" diff --git a/cleandev/tests/test-sparse-submod.sh b/cleandev/tests/test-sparse-submod.sh index cbcadd0..3fa1908 100644 --- a/cleandev/tests/test-sparse-submod.sh +++ b/cleandev/tests/test-sparse-submod.sh @@ -31,10 +31,9 @@ git -C "$MOODLE" commit -q -m "init" git -C "$MOODLE" sparse-checkout init --cone git -C "$MOODLE" sparse-checkout set mod -MANIFEST="$TMP/manifest.txt" -printf '%s\n' 'mod/testplugin|../plugin-upstream|main' > "$MANIFEST" +printf '%s\n' 'mod/testplugin|../plugin-upstream|main' > "$MOODLE/plugin-submodules.manifest" -"$CLEANDEV/submodulize.sh" --repo "$MOODLE" --manifest "$MANIFEST" --no-commit +"$CLEANDEV/submodulize.sh" --repo "$MOODLE" --no-commit assert_file "$MOODLE/mod/testplugin/a.txt" # submodulize disables the same flags it clears in disable_sparse_checkout_if_needed() diff --git a/cleandev/tests/test-submod-roundtrip.sh b/cleandev/tests/test-submod-roundtrip.sh index f2e5d03..6546ea1 100644 --- a/cleandev/tests/test-submod-roundtrip.sh +++ b/cleandev/tests/test-submod-roundtrip.sh @@ -27,17 +27,16 @@ echo "vendored-only" > "$MOODLE/mod/testplugin/local.txt" git -C "$MOODLE" add mod/testplugin git -C "$MOODLE" commit -q -m "vendored plugin" -MANIFEST="$TMP/manifest.txt" -printf '%s\n' 'mod/testplugin|../plugin-upstream|main' > "$MANIFEST" +printf '%s\n' 'mod/testplugin|../plugin-upstream|main' > "$MOODLE/plugin-submodules.manifest" # --- dry-run must not convert --- -"$CLEANDEV/submodulize.sh" --repo "$MOODLE" --manifest "$MANIFEST" --dry-run +"$CLEANDEV/submodulize.sh" --repo "$MOODLE" --dry-run assert_file "$MOODLE/mod/testplugin/local.txt" assert_no_file "$MOODLE/.gitmodules" assert_no_file "$MOODLE/.git/modules/mod/testplugin" # --- submodulize --- -"$CLEANDEV/submodulize.sh" --repo "$MOODLE" --manifest "$MANIFEST" --no-commit +"$CLEANDEV/submodulize.sh" --repo "$MOODLE" --no-commit assert_file "$MOODLE/.gitmodules" assert_file "$MOODLE/mod/testplugin/version.txt" assert_no_file "$MOODLE/mod/testplugin/local.txt" @@ -45,7 +44,7 @@ assert_no_file "$MOODLE/mod/testplugin/local.txt" git -C "$MOODLE" submodule status --recursive | grep -q 'mod/testplugin' || fail "expected mod/testplugin in submodule status" # --- unsubmodulize --- -"$CLEANDEV/unsubmodulize.sh" --repo "$MOODLE" --manifest "$MANIFEST" --no-commit +"$CLEANDEV/unsubmodulize.sh" --repo "$MOODLE" --no-commit assert_file "$MOODLE/mod/testplugin/version.txt" assert_no_file "$MOODLE/mod/testplugin/.git" # Git may leave an empty .gitmodules; ensure no submodule entries remain. @@ -58,4 +57,4 @@ git -C "$MOODLE" ls-files --error-unmatch mod/testplugin/version.txt >/dev/null ! git -C "$MOODLE" submodule status 2>/dev/null | grep -q 'mod/testplugin' \ || fail "expected mod/testplugin not to remain a submodule after unsubmodulize" -ok "submodulize <-> unsubmodulize round-trip (local relative URL)" +ok "submodulize <-> unsubmodulize round-trip (default ROOT/plugin-submodules.manifest)" diff --git a/cleandev/unsubmodulize.sh b/cleandev/unsubmodulize.sh index 9983216..a61945e 100644 --- a/cleandev/unsubmodulize.sh +++ b/cleandev/unsubmodulize.sh @@ -1,7 +1,9 @@ #!/usr/bin/env bash # Replace git submodules with plain tracked directories (vendored plugins), matching lsuce-moodle -# develop style. Clones each plugin repo at the given branch, drops nested .git, and adds files -# to the parent Moodle repository. Run from inside the clone, or pass the clone path as ROOT / --repo. +# develop style. For the submodule-layout Moodle superproject: manifest and .gitmodules live in that +# repo; this script reads plugin-submodules.manifest at the superproject root unless --manifest is set. +# Clones each plugin repo at the given branch, drops nested .git, and adds files to the parent. +# Run from inside the clone, or pass the clone path as ROOT / --repo. # # Usage: # ./cleandev/unsubmodulize.sh [ROOT] [--dry-run] [--no-commit] [--ssh] [--manifest PATH] [--repo ROOT] @@ -13,8 +15,8 @@ set -euo pipefail -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -MANIFEST="${SCRIPT_DIR}/plugin-submodules.manifest" +MANIFEST="" +MANIFEST_EXPLICIT=false DRY_RUN=false NO_COMMIT=false USE_SSH=false @@ -24,7 +26,8 @@ usage() { cat <<'EOF' Replace git submodules listed in the manifest with plain tracked directories (vendored plugins). For each entry: clones the plugin at the manifest branch, removes nested .git, deinits the submodule, -and stages the tree in the parent Moodle repo. +and stages the tree in the parent Moodle superproject (cleandev-style). The manifest normally lives +at the superproject root as plugin-submodules.manifest. Moodle root defaults to the current directory’s git superproject (git rev-parse --show-toplevel), unless you set it explicitly. @@ -45,7 +48,7 @@ Options: --dry-run Print what would happen without changing the repo --no-commit Stage vendored trees but do not commit --ssh Use git@github.com URLs for github.com HTTPS entries - --manifest PATH Use a manifest file (default: cleandev/plugin-submodules.manifest next to this script) + --manifest PATH Plugin manifest (default: ROOT/plugin-submodules.manifest) --repo ROOT Moodle git root (explicit form of a bare ROOT; overrides an earlier bare ROOT; a bare path after --repo is an error) Requires: git, and clean submodule working trees for paths being converted (no uncommitted changes inside submodules). @@ -62,6 +65,7 @@ while [[ $# -gt 0 ]]; do --ssh) USE_SSH=true; shift ;; --manifest) MANIFEST="${2:?}" + MANIFEST_EXPLICIT=true shift 2 ;; --repo) @@ -87,11 +91,6 @@ while [[ $# -gt 0 ]]; do esac done -if [[ ! -f "$MANIFEST" ]]; then - echo "Manifest not found: $MANIFEST" >&2 - exit 1 -fi - if [[ -z "$REPO_ROOT" ]]; then REPO_ROOT="$(git rev-parse --show-toplevel 2>/dev/null)" || { echo "Not inside a git repository. Pass the Moodle root as a bare path or use --repo /path/to/moodle" >&2 @@ -99,6 +98,15 @@ if [[ -z "$REPO_ROOT" ]]; then } fi +if ! $MANIFEST_EXPLICIT; then + MANIFEST="${REPO_ROOT%/}/plugin-submodules.manifest" +fi + +if [[ ! -f "$MANIFEST" ]]; then + echo "Manifest not found: $MANIFEST" >&2 + exit 1 +fi + cd "$REPO_ROOT" disable_sparse_checkout_if_needed() { diff --git a/demo/plugin1 b/demo/plugin1 new file mode 160000 index 0000000..6e9895b --- /dev/null +++ b/demo/plugin1 @@ -0,0 +1 @@ +Subproject commit 6e9895b10c577bd0277bb41be86c2a1a5759c071 diff --git a/demo/plugin2 b/demo/plugin2 new file mode 160000 index 0000000..fab9bed --- /dev/null +++ b/demo/plugin2 @@ -0,0 +1 @@ +Subproject commit fab9bede9d1d76afb4032e7788280f2f500b80de diff --git a/demo/plugin3 b/demo/plugin3 new file mode 160000 index 0000000..c23687a --- /dev/null +++ b/demo/plugin3 @@ -0,0 +1 @@ +Subproject commit c23687ab630cde28a770a4bd1165b7d81d6603a4 diff --git a/demo/raw b/demo/raw new file mode 160000 index 0000000..2103bac --- /dev/null +++ b/demo/raw @@ -0,0 +1 @@ +Subproject commit 2103bac06d1d55ddfa5ae3cd4509d56f30c81d8f diff --git a/demo/sub b/demo/sub new file mode 160000 index 0000000..a3d253c --- /dev/null +++ b/demo/sub @@ -0,0 +1 @@ +Subproject commit a3d253c7431987438bc2d413a2c6c31f6236f752 diff --git a/new.sh b/new.sh index 01df0be..9ca20e1 100755 --- a/new.sh +++ b/new.sh @@ -238,12 +238,13 @@ if [ "$SUBMODULIZE" = true ]; then git merge origin/develop git ls-files blocks/ues_people | xargs -r git update-index --skip-worktree rm -rf blocks/ues_people + cp /tmp/plugin-submodules.manifest /var/www/html/plugin-submodules.manifest SMF="--no-commit" if [ "${SUBMODULIZE_USE_SSH:-}" = "1" ]; then SMF="$SMF --ssh"; fi - if bash /tmp/manifest-submodulize-redundant.sh --repo /var/www/html --manifest /tmp/plugin-submodules.manifest; then + if bash /tmp/manifest-submodulize-redundant.sh --repo /var/www/html; then echo "new.sh: All manifest plugin paths are already submodules; skipping submodulize.sh." else - bash /tmp/submodulize.sh --repo /var/www/html --manifest /tmp/plugin-submodules.manifest $SMF + bash /tmp/submodulize.sh --repo /var/www/html $SMF fi git update-index --skip-worktree config.php ' From ed382ecb63b804c16e033422f2b2a1600fbc067f Mon Sep 17 00:00:00 2001 From: Steve Mattsen Date: Tue, 14 Apr 2026 15:49:02 -0400 Subject: [PATCH 06/18] Update `new.sh`, `submodulize.sh`, and `unsubmodulize.sh` to implement `--no-replay` option for one-shot conversions. Revise README and TEAM-PROCESS.md to clarify default replay behavior and usage of the new flag. Adjust test cases to ensure compatibility with the updated script functionality. --- README.md | 12 +- cleandev/TEAM-PROCESS.md | 6 +- cleandev/submodulize.sh | 293 ++++++++++++++++++++- cleandev/tests/test-replay-both.sh | 70 +++++ cleandev/tests/test-sparse-submod.sh | 2 +- cleandev/tests/test-submod-roundtrip.sh | 6 +- cleandev/unsubmodulize.sh | 323 +++++++++++++++++++++++- demo/sub | 2 +- new.sh | 4 +- 9 files changed, 697 insertions(+), 21 deletions(-) create mode 100644 cleandev/tests/test-replay-both.sh diff --git a/README.md b/README.md index 484b817..2d116d7 100644 --- a/README.md +++ b/README.md @@ -88,7 +88,7 @@ These are **branches on the Moodle repo** (`lsuce-moodle`), not branch names in - Porting changes **both ways** (submodule layout ↔ vendored layout). - **Redoing or replaying commits** with a **provenance note** (e.g. original commit SHA / branch)—so history stays traceable across layouts. -**Current repo state:** This project ships **one-shot layout converters** (`submodulize.sh` / `unsubmodulize.sh`). It does **not** yet implement commit-by-commit replay, automatic provenance headers, or a `new.sh` mode that detects vendored vs submodulized state without manual choice. +**Current repo state:** This project ships **layout converters** (`submodulize.sh` / `unsubmodulize.sh`). **Replay is the default:** they build `submodulized` / `unsubmodulized` branches with **one superproject commit per plugin-repo commit** (chronological ordering, carry-forward; **`--fork-point`** required). Use **`--no-replay`** for one-shot conversion over the manifest only. A `new.sh` mode that detects vendored vs submodulized state without manual choice is still not implemented. **Branch policy, Docker image decision, and updating the manifest from the CSV:** [cleandev/TEAM-PROCESS.md](cleandev/TEAM-PROCESS.md). @@ -109,14 +109,14 @@ This repo keeps a **canonical copy** under `cleandev/` for linting and for `new. | Script | Role | |--------|------| -| `cleandev/submodulize.sh` | Vendored trees → submodules (sparse-checkout disabled first; skips paths already in `.gitmodules`; `GITHUB_TOKEN` passed via `-c url...insteadOf` for `ls-remote` / `submodule add`). | -| `cleandev/unsubmodulize.sh` | Submodules → vendored trees (clone depth 1, drop nested `.git`, `git add`). Uses the same `GITHUB_TOKEN` `-c url...insteadOf` for `ls-remote` / `clone` as `submodulize.sh`, or `--ssh`. | +| `cleandev/submodulize.sh` | Default **replay** (`--fork-point`): one superproject commit per plugin commit (gitlinks + `.gitmodules`). **`--no-replay`**: one-shot vendored → submodules (sparse-checkout disabled first; skips paths already in `.gitmodules`; `GITHUB_TOKEN` via `-c url...insteadOf` for `ls-remote` / `submodule add`). | +| `cleandev/unsubmodulize.sh` | Default **replay** (`--fork-point`): one superproject commit per plugin commit (vendored trees). **`--no-replay`**: one-shot submodules → vendored (clone depth 1, drop nested `.git`, `git add`). Same `GITHUB_TOKEN` / `--ssh` as `submodulize.sh`. | Run manually from a Moodle clone: ```bash -./cleandev/submodulize.sh [--dry-run] [--no-commit] [--ssh] [--manifest PATH] [--repo ROOT] -./cleandev/unsubmodulize.sh [--dry-run] [--no-commit] [--ssh] [--manifest PATH] [--repo ROOT] +./cleandev/submodulize.sh [--no-replay] [--dry-run] [--no-commit] [--ssh] [--manifest PATH] [--repo ROOT] +./cleandev/unsubmodulize.sh [--no-replay] [--dry-run] [--no-commit] [--ssh] [--manifest PATH] [--repo ROOT] ``` Automated tests (manifest lint, PAT wiring checks, `submodulize`/`unsubmodulize` round-trip in temp repos—no changes to your working tree): @@ -130,7 +130,7 @@ bash cleandev/tests/run.sh - Compose project name = first argument (containers `{NAME}-moodle`, etc.). - Web service builds from `.` per `docker-compose.yml`. - As `www-data`: `git fetch` / `git merge origin/develop`; removes `blocks/ues_people` and uses `skip-worktree` so it does not clash with `block_lsu_people` (see `config/moodle-pull` for the same idea on `git pull`). -- With `--submodulize`: copies scripts into the container, stages `cleandev/plugin-submodules.manifest` into `/var/www/html/plugin-submodules.manifest` after the merge, resolves GitHub token, sets local `url...insteadOf` when using HTTPS token, runs `manifest-submodulize-redundant.sh` / `submodulize.sh` with default manifest paths (`--repo /var/www/html`, optional SSH via `SUBMODULIZE_SSH=1`). +- With `--submodulize`: copies scripts into the container, stages `cleandev/plugin-submodules.manifest` into `/var/www/html/plugin-submodules.manifest` after the merge, resolves GitHub token, sets local `url...insteadOf` when using HTTPS token, runs `manifest-submodulize-redundant.sh` / `submodulize.sh --no-replay` with default manifest paths (`--repo /var/www/html`, optional SSH via `SUBMODULIZE_SSH=1`). Secrets: `cleandev/.github-token` is listed in `.gitignore`. diff --git a/cleandev/TEAM-PROCESS.md b/cleandev/TEAM-PROCESS.md index 61dbc0e..4edb3a7 100644 --- a/cleandev/TEAM-PROCESS.md +++ b/cleandev/TEAM-PROCESS.md @@ -4,6 +4,8 @@ This document covers **branch policy** for [lsuce-moodle](https://github.com/lsu **Where the manifest lives:** On **cleandev**, treat `plugin-submodules.manifest` as part of the Moodle superproject (repo root), alongside `.gitmodules`. The copy in **this** repo under [`cleandev/plugin-submodules.manifest`](plugin-submodules.manifest) is the maintained source for CSV refresh and CI lint; `submodulize.sh` / `unsubmodulize.sh` default to `ROOT/plugin-submodules.manifest` unless you pass `--manifest`. +**Branch replay (default):** `unsubmodulize.sh` and `submodulize.sh` run replay unless you pass **`--no-replay`**. Replay builds histories with **one Moodle superproject commit per plugin-repo commit** (chronological order, carry-forward). It requires `--fork-point` and defaults `--source` / `--target` branch names (`submodulized`, `unsubmodulized`). If the fork-point checkout is **vendored** at a manifest path, the script matches the embedded tree to a plugin commit (or use `--plugin-base path=SHA`). `new.sh --submodulize` runs **`submodulize.sh --no-replay`** (one-shot only). + --- ## 1. Team process (develop ↔ cleandev) @@ -30,7 +32,7 @@ This document covers **branch policy** for [lsuce-moodle](https://github.com/lsu ### Local Docker stack (`new.sh`) - **Without `--submodulize`:** Container tracks **vendored** layout after merge from `origin/develop` (matches **develop**). -- **With `--submodulize`:** After that merge, runs `submodulize.sh` so manifest-listed paths become **submodules** (closer to **cleandev**). Requires `GITHUB_TOKEN` / `cleandev/.github-token` or `SUBMODULIZE_SSH=1` for private GitHub repos. +- **With `--submodulize`:** After that merge, runs `submodulize.sh --no-replay` so manifest-listed paths become **submodules** (closer to **cleandev**). Requires `GITHUB_TOKEN` / `cleandev/.github-token` or `SUBMODULIZE_SSH=1` for private GitHub repos. - Never commit **`cleandev/.github-token`** (gitignored). ### Submodule hygiene @@ -126,7 +128,7 @@ If the CSV has no working public URL (internal, defunct, empty), add or keep a * ### After editing the manifest -1. Run **`submodulize.sh --dry-run`** from a **test clone** of lsuce-moodle to sanity-check paths and remotes. +1. Run **`submodulize.sh --no-replay --dry-run`** from a **test clone** of lsuce-moodle to sanity-check paths and remotes. 2. Commit manifest changes in **this** repo (`lsuce_moodle_project`) when the tooling repo owns the file; if the manifest is versioned only in lsuce-moodle, commit there per your layout. For script flags and `GITHUB_TOKEN` behavior, see the headers in `submodulize.sh` and `unsubmodulize.sh`. diff --git a/cleandev/submodulize.sh b/cleandev/submodulize.sh index a2ee722..bb3f92a 100644 --- a/cleandev/submodulize.sh +++ b/cleandev/submodulize.sh @@ -2,11 +2,15 @@ # Convert vendored plugin directories (plain files in the Moodle superproject) into git submodules. # Intended for the submodule-layout Moodle checkout: keep plugin-submodules.manifest at the superproject # root (same repo as .gitmodules will live in). Run from inside that clone, or pass ROOT / --repo. +# Default mode is replay: builds branch submodulized with one superproject commit per plugin-repo commit. +# Use --no-replay for one-shot conversion (manifest loop only). # # Requires: git, a clean enough working tree (commit or stash first if paths are dirty). # # Usage: # ./cleandev/submodulize.sh [ROOT] [--dry-run] [--no-commit] [--ssh] [--manifest PATH] [--repo ROOT] +# ./cleandev/submodulize.sh --fork-point BASE ... # replay (default) +# ./cleandev/submodulize.sh --no-replay ... # one-shot # Bare ROOT is the same as --repo (optional; may appear before or after flags). # # Private GitHub repos over HTTPS need credentials. Set GITHUB_TOKEN (PAT) so HTTPS URLs are rewritten @@ -21,6 +25,16 @@ DRY_RUN=false NO_COMMIT=false USE_SSH=false REPO_ROOT="" +REPLAY=true +FORK_POINT="" +SOURCE_BRANCH="submodulized" +TARGET_BRANCH="submodulized" +REPLAY_ORDER="chronological" +FORCE_REPLAY=false +REPLAY_SOURCE_EXPLICIT=false +REPLAY_TARGET_EXPLICIT=false +REPLAY_ORDER_EXPLICIT=false +declare -a PLUGIN_BASE_OVERRIDES=() usage() { cat <<'EOF' @@ -38,9 +52,9 @@ Usage: bare path, and do not put a bare path after --repo (that is rejected). Examples: - submodulize.sh ~/workspace/moodle - submodulize.sh --dry-run ~/workspace/moodle - submodulize.sh ~/workspace/moodle --no-commit --ssh + submodulize.sh --no-replay ~/workspace/moodle + submodulize.sh --dry-run --no-replay ~/workspace/moodle + submodulize.sh --fork-point abc123 --source submodulized Options: --dry-run Print actions without changing the repo @@ -49,6 +63,16 @@ Options: --manifest PATH Plugin manifest (default: ROOT/plugin-submodules.manifest) --repo ROOT Moodle git root (explicit form of a bare ROOT; overrides an earlier bare ROOT; a bare path after --repo is an error) +Replay (default — one superproject commit per plugin-repo commit on --target; --fork-point required): + --no-replay One-shot mode: add submodules per manifest (no branch replay) + --replay Replay mode (default; explicit if you toggled --no-replay earlier on the command line) + --fork-point BASE Start of replay (vendored plugin dirs and/or existing gitlinks) + --source BR Branch whose tip supplies final submodule SHAs per path (default: submodulized) + --target BR Branch to create/update (default: submodulized) + --order NAME Only chronological + --force Overwrite --target branch if it exists (replay only) + --plugin-base P=S Optional start SHA for manifest path P + Requires: git, and a clean enough working tree (commit or stash if plugin paths are dirty). Private GitHub over HTTPS: set GITHUB_TOKEN (PAT) so ls-remote / submodule add can authenticate, @@ -62,6 +86,35 @@ while [[ $# -gt 0 ]]; do --dry-run) DRY_RUN=true; shift ;; --no-commit) NO_COMMIT=true; shift ;; --ssh) USE_SSH=true; shift ;; + --no-replay) REPLAY=false; shift ;; + --replay) REPLAY=true; shift ;; + --fork-point) + FORK_POINT="${2:?}" + shift 2 + ;; + --source) + SOURCE_BRANCH="${2:?}" + REPLAY_SOURCE_EXPLICIT=true + shift 2 + ;; + --target) + TARGET_BRANCH="${2:?}" + REPLAY_TARGET_EXPLICIT=true + shift 2 + ;; + --order) + REPLAY_ORDER="${2:?}" + REPLAY_ORDER_EXPLICIT=true + shift 2 + ;; + --force) + FORCE_REPLAY=true + shift + ;; + --plugin-base) + PLUGIN_BASE_OVERRIDES+=("${2:?}") + shift 2 + ;; --manifest) MANIFEST="${2:?}" MANIFEST_EXPLICIT=true @@ -106,6 +159,42 @@ if [[ ! -f "$MANIFEST" ]]; then exit 1 fi +if $REPLAY; then + [[ -n "$FORK_POINT" ]] || { + echo "Required: --fork-point BASE (replay is the default; use --no-replay for one-shot conversion without replay)" >&2 + exit 1 + } + [[ "$REPLAY_ORDER" == "chronological" ]] || { + echo "Only --order chronological is supported: $REPLAY_ORDER" >&2 + exit 1 + } +else + [[ -z "$FORK_POINT" ]] || { + echo "--fork-point is only used in replay mode (omit --no-replay)" >&2 + exit 1 + } + ((${#PLUGIN_BASE_OVERRIDES[@]} == 0)) || { + echo "--plugin-base is only valid in replay mode (omit --no-replay)" >&2 + exit 1 + } + $FORCE_REPLAY && { + echo "--force is only valid in replay mode (omit --no-replay)" >&2 + exit 1 + } + $REPLAY_SOURCE_EXPLICIT && { + echo "--source is only valid in replay mode (omit --no-replay)" >&2 + exit 1 + } + $REPLAY_TARGET_EXPLICIT && { + echo "--target is only valid in replay mode (omit --no-replay)" >&2 + exit 1 + } + $REPLAY_ORDER_EXPLICIT && { + echo "--order is only valid in replay mode (omit --no-replay)" >&2 + exit 1 + } +fi + cd "$REPO_ROOT" # Moodle dev images often use sparse-checkout (cone, index.sparse, or only .git/info/sparse-checkout). @@ -156,6 +245,204 @@ run() { fi } +submodulize_replay_mode() { + git rev-parse --verify "$FORK_POINT^{commit}" >/dev/null + git rev-parse --verify "$SOURCE_BRANCH^{commit}" >/dev/null + + if git show-ref --verify --quiet "refs/heads/$TARGET_BRANCH" && ! $FORCE_REPLAY; then + echo "Branch $TARGET_BRANCH exists; pass --force" >&2 + exit 1 + fi + + ls_path_mode_sha() { + git ls-tree "$1" -- "$2" 2>/dev/null | awk '{print $1 "\t" $3}' | head -n1 + } + + plugin_base_override_for() { + local want="$1" entry + for entry in "${PLUGIN_BASE_OVERRIDES[@]:-}"; do + [[ "${entry%%=*}" == "$want" ]] && { echo "${entry#*=}"; return 0; } + done + return 1 + } + + find_plugin_commit_for_tree() { + local pdir="$1" want_tree="$2" c t + while IFS= read -r c; do + t="$(git -C "$pdir" rev-parse "$c^{tree}" 2>/dev/null || true)" + [[ "$t" == "$want_tree" ]] && { echo "$c"; return 0; } + done < <(git -C "$pdir" rev-list --first-parent --reverse --all) + return 1 + } + + declare -a M_PATHS=() M_URLS=() M_BRANCHES=() + + while IFS='|' read -r raw_path raw_url raw_branch; do + path="${raw_path#"${raw_path%%[![:space:]]*}"}" + path="${path%"${path##*[![:space:]]}"}" + url="${raw_url#"${raw_url%%[![:space:]]*}"}" + url="${url%"${url##*[![:space:]]}"}" + branch="${raw_branch#"${raw_branch%%[![:space:]]*}"}" + branch="${branch%"${branch##*[![:space:]]}"}" + [[ -z "$path" || "$path" =~ ^# ]] && continue + [[ -z "$url" ]] && { echo "Manifest: missing URL for $path" >&2; exit 1; } + [[ -z "$branch" ]] && branch="main" + M_PATHS+=("$path") + M_URLS+=("$url") + M_BRANCHES+=("$branch") + done < "$MANIFEST" + + [[ ${#M_PATHS[@]} -gt 0 ]] || { echo "No manifest entries." >&2; exit 1; } + + SOURCE_TIP="$(git rev-parse "$SOURCE_BRANCH^{commit}")" + TMPD="$(mktemp -d "${TMPDIR:-/tmp}/sub-replay.XXXXXX")" + trap 'rm -rf "$TMPD"' EXIT + + declare -a EVENT_LINES=() + + for i in "${!M_PATHS[@]}"; do + P="${M_PATHS[$i]}" + URL="$(rewrite_github_url_to_ssh "${M_URLS[$i]}")" + PDIR="$TMPD/plugin_${i}_$(echo "$P" | tr '/' '_')" + + ms="$(ls_path_mode_sha "$FORK_POINT" "$P")" + mode="${ms%%$'\t'*}" + from_sha="${ms#*$'\t'}" + + ms2="$(ls_path_mode_sha "$SOURCE_TIP" "$P")" + to_sha="${ms2#*$'\t'}" + mode2="${ms2%%$'\t'*}" + + if [[ "$mode2" != "160000" ]] || [[ -z "$to_sha" ]]; then + echo "Path $P: source tip must be gitlink for end SHA" >&2 + exit 1 + fi + + if ! plugin_base_override_for "$P" >/dev/null; then + if [[ "$mode" == "160000" && -n "$from_sha" ]]; then + : + elif [[ "$mode" == "040000" ]]; then + tr_sha="$(git rev-parse "$FORK_POINT:$P^{tree}" 2>/dev/null || true)" + if [[ -z "$tr_sha" ]]; then + echo "No tree at $FORK_POINT:$P" >&2 + exit 1 + fi + if [[ ! -d "$PDIR/.git" ]]; then + GIT_TERMINAL_PROMPT=0 git "${git_github_pat_c[@]}" clone --bare "$URL" "$PDIR" + fi + from_sha="$(find_plugin_commit_for_tree "$PDIR" "$tr_sha")" || { + echo "Could not find plugin commit matching tree at $FORK_POINT:$P" >&2 + exit 1 + } + else + echo "Path $P at fork-point must be gitlink or directory; use --plugin-base ${P}=SHA" >&2 + exit 1 + fi + else + from_sha="$(plugin_base_override_for "$P")" + fi + + if [[ ! -d "$PDIR/.git" ]]; then + GIT_TERMINAL_PROMPT=0 git "${git_github_pat_c[@]}" clone --bare "$URL" "$PDIR" + fi + GIT_TERMINAL_PROMPT=0 git -C "$PDIR" fetch -q origin 2>/dev/null || true + GIT_TERMINAL_PROMPT=0 git -C "$PDIR" fetch -q "$URL" "+refs/*:refs/remotes/import/*" 2>/dev/null || true + + GIT_TERMINAL_PROMPT=0 git -C "$PDIR" cat-file -e "${from_sha}^{commit}" 2>/dev/null || { + echo "Missing plugin object $from_sha for $P" >&2 + exit 1 + } + GIT_TERMINAL_PROMPT=0 git -C "$PDIR" cat-file -e "${to_sha}^{commit}" 2>/dev/null || { + echo "Missing plugin object $to_sha for $P" >&2 + exit 1 + } + + mapfile -t PCOMMITS < <(GIT_TERMINAL_PROMPT=0 git -C "$PDIR" rev-list --first-parent --reverse "${from_sha}..${to_sha}") + [[ ${#PCOMMITS[@]} -eq 0 ]] && continue + + for c in "${PCOMMITS[@]}"; do + ct="$(git -C "$PDIR" show -s --format=%ct "$c")" + EVENT_LINES+=("${ct}"$'\t'"${P}"$'\t'"${c}"$'\t'"${PDIR}"$'\t'"${M_URLS[$i]}") + done + done + + [[ ${#EVENT_LINES[@]} -gt 0 ]] || { echo "No plugin commits to replay." >&2; exit 0; } + + IFS=$'\n' + sorted="$(printf '%s\n' "${EVENT_LINES[@]}" | LC_ALL=C sort -t $'\t' -k1,1n -k2,2 -k3,3)" + unset IFS + + if $DRY_RUN; then + echo "Planned ${#EVENT_LINES[@]} commits on $TARGET_BRANCH (from $FORK_POINT)" + while IFS=$'\t' read -r ct p sha pdir url; do + [[ -z "${ct:-}" ]] && continue + echo " $ct $p $sha ($(git -C "$pdir" show -s --format=%s "$sha"))" + done <<< "$sorted" + exit 0 + fi + + $FORCE_REPLAY && git show-ref --verify --quiet "refs/heads/$TARGET_BRANCH" && git branch -D "$TARGET_BRANCH" 2>/dev/null || true + git worktree prune 2>/dev/null || true + + WT="$TMPD/wt" + rm -rf "$WT" + + manifest_url_for_path() { + local want="$1" _i + for _i in "${!M_PATHS[@]}"; do + if [[ "${M_PATHS[$_i]}" == "$want" ]]; then + echo "${M_URLS[$_i]}" + return 0 + fi + done + return 1 + } + + rebuild_gitmodules() { + rm -f "$WT/.gitmodules" + git -C "$WT" ls-files -s | while read -r mode sha _st path; do + [[ "$mode" == "160000" ]] || continue + u="$(manifest_url_for_path "$path")" || continue + printf '[submodule "%s"]\n\tpath = %s\n\turl = %s\n' "$path" "$path" "$u" >>"$WT/.gitmodules" + done + } + + GIT_TERMINAL_PROMPT=0 git "${git_github_pat_c[@]}" worktree add -B "$TARGET_BRANCH" "$WT" "$FORK_POINT" --force + + while IFS=$'\t' read -r ct P csha pdir url; do + [[ -z "${ct:-}" ]] && continue + git -C "$WT" reset --hard -q HEAD + git -C "$WT" rm -rf --cached --ignore-unmatch "$P" 2>/dev/null || true + rm -rf "${WT:?}/${P}" + git -C "$WT" update-index --add --cacheinfo "160000,$csha,$P" + rebuild_gitmodules + git -C "$WT" add -f .gitmodules 2>/dev/null || true + + an="$(git -C "$pdir" show -s --format=%an "$csha")" + ae="$(git -C "$pdir" show -s --format=%ae "$csha")" + adate="$(git -C "$pdir" show -s --format=%ai "$csha")" + body="$(git -C "$pdir" show -s --format=%B "$csha")" + { + printf '%s\n\n' "$body" + printf 'Replayed-from: %s\n' "$csha" + printf 'Plugin-path: %s\n' "$P" + printf 'gitlink: submodulize.sh (replay)\n' + } >"$TMPD/commitmsg.txt" + + GIT_AUTHOR_NAME="$an" GIT_AUTHOR_EMAIL="$ae" GIT_AUTHOR_DATE="$adate" \ + GIT_COMMITTER_NAME="$an" GIT_COMMITTER_EMAIL="$ae" GIT_COMMITTER_DATE="$adate" \ + git -C "$WT" commit -F "$TMPD/commitmsg.txt" + done <<< "$sorted" + + git -C "$REPO_ROOT" worktree remove -f "$WT" 2>/dev/null || true + echo "Done. Branch $TARGET_BRANCH -> $(git rev-parse "$TARGET_BRANCH")" +} + +if $REPLAY; then + submodulize_replay_mode + exit 0 +fi + manifest_entries=0 while IFS='|' read -r raw_path raw_url raw_branch; do path="${raw_path#"${raw_path%%[![:space:]]*}"}" diff --git a/cleandev/tests/test-replay-both.sh b/cleandev/tests/test-replay-both.sh new file mode 100644 index 0000000..de177de --- /dev/null +++ b/cleandev/tests/test-replay-both.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash +# Replay mode (default): unsubmodulize + submodulize on temp repos (file:// plugin, no network). +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=lib.sh +source "$SCRIPT_DIR/lib.sh" +require_cleandev + +TMP=$(mktemp -d) +export HOME="$TMP/_home" +mkdir -p "$HOME" +git config --global protocol.file.allow always +trap 'rm -rf "$TMP"' EXIT + +PLUGIN="$TMP/plugin" +MOODLE="$TMP/moodle" +mkdir -p "$PLUGIN" "$MOODLE" + +git -C "$PLUGIN" init -b main +echo "v1" > "$PLUGIN/f.txt" +git -C "$PLUGIN" add f.txt +git -C "$PLUGIN" commit -q -m "p1" +echo "v2" >> "$PLUGIN/f.txt" +git -C "$PLUGIN" commit -q -am "p2" + +C0="$(git -C "$PLUGIN" rev-parse HEAD~1)" +C1="$(git -C "$PLUGIN" rev-parse HEAD)" + +git -C "$MOODLE" init -b main +printf 'core\n' > "$MOODLE/README.md" +mkdir -p "$MOODLE/mod/foo" +git -C "$PLUGIN" archive "$C0" | tar -x -C "$MOODLE/mod/foo" +git -C "$MOODLE" add README.md mod/foo +git -C "$MOODLE" commit -q -m "vendored base" + +BASE="$(git -C "$MOODLE" rev-parse HEAD)" + +git -C "$MOODLE" config -f .gitmodules submodule.mod/foo.path mod/foo +git -C "$MOODLE" config -f .gitmodules submodule.mod/foo.url "$PLUGIN" +git -C "$MOODLE" add .gitmodules +git -C "$MOODLE" rm -rf --cached mod/foo +git -C "$MOODLE" update-index --add --cacheinfo "160000,$C0,mod/foo" +git -C "$MOODLE" commit -q -m "sub at c0" + +git -C "$MOODLE" checkout -q -b submodulized +git -C "$MOODLE" update-index --add --cacheinfo "160000,$C1,mod/foo" +git -C "$MOODLE" commit -q -m "sub at c1" + +MANIFEST="$MOODLE/plugin-submodules.manifest" +printf 'mod/foo|%s|main\n' "$PLUGIN" > "$MANIFEST" + +# --- unsub replay: one commit (c1) in plugin range c0..c1 (explicit plugin base avoids CRLF tree mismatch) --- +"$CLEANDEV/unsubmodulize.sh" --repo "$MOODLE" --fork-point "$BASE" --source submodulized \ + --target unsubmodulized --force --plugin-base "mod/foo=$C0" + +N="$(git -C "$MOODLE" rev-list --count "${BASE}..unsubmodulized" 2>/dev/null || echo 0)" +[[ "$N" -eq 1 ]] || fail "expected 1 commit on unsubmodulized past BASE, got $N" +git -C "$MOODLE" ls-tree -r unsubmodulized --name-only | grep -q '^mod/foo/f.txt$' || fail "expected vendored file path" +git -C "$MOODLE" show unsubmodulized:mod/foo/f.txt | grep -q v2 || fail "expected v2 in tree" + +# --- sub replay from vendored BASE: match tree at BASE to plugin c0 --- +git -C "$MOODLE" checkout -q main +"$CLEANDEV/submodulize.sh" --repo "$MOODLE" --fork-point "$BASE" --source submodulized \ + --target sub2 --force --plugin-base "mod/foo=$C0" + +N2="$(git -C "$MOODLE" rev-list --count "${BASE}..sub2" 2>/dev/null || echo 0)" +[[ "$N2" -eq 1 ]] || fail "expected 1 commit on sub2 past BASE, got $N2" +git -C "$MOODLE" ls-tree sub2 mod/foo | grep -q 160000 || fail "expected gitlink at mod/foo" + +ok "unsubmodulize + submodulize replay (fixture)" diff --git a/cleandev/tests/test-sparse-submod.sh b/cleandev/tests/test-sparse-submod.sh index 3fa1908..3463948 100644 --- a/cleandev/tests/test-sparse-submod.sh +++ b/cleandev/tests/test-sparse-submod.sh @@ -33,7 +33,7 @@ git -C "$MOODLE" sparse-checkout set mod printf '%s\n' 'mod/testplugin|../plugin-upstream|main' > "$MOODLE/plugin-submodules.manifest" -"$CLEANDEV/submodulize.sh" --repo "$MOODLE" --no-commit +"$CLEANDEV/submodulize.sh" --no-replay --repo "$MOODLE" --no-commit assert_file "$MOODLE/mod/testplugin/a.txt" # submodulize disables the same flags it clears in disable_sparse_checkout_if_needed() diff --git a/cleandev/tests/test-submod-roundtrip.sh b/cleandev/tests/test-submod-roundtrip.sh index 6546ea1..ffdee20 100644 --- a/cleandev/tests/test-submod-roundtrip.sh +++ b/cleandev/tests/test-submod-roundtrip.sh @@ -30,13 +30,13 @@ git -C "$MOODLE" commit -q -m "vendored plugin" printf '%s\n' 'mod/testplugin|../plugin-upstream|main' > "$MOODLE/plugin-submodules.manifest" # --- dry-run must not convert --- -"$CLEANDEV/submodulize.sh" --repo "$MOODLE" --dry-run +"$CLEANDEV/submodulize.sh" --no-replay --repo "$MOODLE" --dry-run assert_file "$MOODLE/mod/testplugin/local.txt" assert_no_file "$MOODLE/.gitmodules" assert_no_file "$MOODLE/.git/modules/mod/testplugin" # --- submodulize --- -"$CLEANDEV/submodulize.sh" --repo "$MOODLE" --no-commit +"$CLEANDEV/submodulize.sh" --no-replay --repo "$MOODLE" --no-commit assert_file "$MOODLE/.gitmodules" assert_file "$MOODLE/mod/testplugin/version.txt" assert_no_file "$MOODLE/mod/testplugin/local.txt" @@ -44,7 +44,7 @@ assert_no_file "$MOODLE/mod/testplugin/local.txt" git -C "$MOODLE" submodule status --recursive | grep -q 'mod/testplugin' || fail "expected mod/testplugin in submodule status" # --- unsubmodulize --- -"$CLEANDEV/unsubmodulize.sh" --repo "$MOODLE" --no-commit +"$CLEANDEV/unsubmodulize.sh" --no-replay --repo "$MOODLE" --no-commit assert_file "$MOODLE/mod/testplugin/version.txt" assert_no_file "$MOODLE/mod/testplugin/.git" # Git may leave an empty .gitmodules; ensure no submodule entries remain. diff --git a/cleandev/unsubmodulize.sh b/cleandev/unsubmodulize.sh index a61945e..e4e9e5b 100644 --- a/cleandev/unsubmodulize.sh +++ b/cleandev/unsubmodulize.sh @@ -4,9 +4,13 @@ # repo; this script reads plugin-submodules.manifest at the superproject root unless --manifest is set. # Clones each plugin repo at the given branch, drops nested .git, and adds files to the parent. # Run from inside the clone, or pass the clone path as ROOT / --repo. +# Default mode is replay: builds branch unsubmodulized with one superproject commit per plugin-repo commit. +# Use --no-replay for one-shot conversion (manifest loop only). # # Usage: # ./cleandev/unsubmodulize.sh [ROOT] [--dry-run] [--no-commit] [--ssh] [--manifest PATH] [--repo ROOT] +# ./cleandev/unsubmodulize.sh --fork-point BASE ... # replay (default) +# ./cleandev/unsubmodulize.sh --no-replay ... # one-shot # Bare ROOT is the same as --repo (optional; may appear before or after flags). # # Private GitHub repos over HTTPS: set GITHUB_TOKEN (PAT) so ls-remote / clone use @@ -21,6 +25,16 @@ DRY_RUN=false NO_COMMIT=false USE_SSH=false REPO_ROOT="" +REPLAY=true +FORK_POINT="" +SOURCE_BRANCH="submodulized" +TARGET_BRANCH="unsubmodulized" +REPLAY_ORDER="chronological" +FORCE_REPLAY=false +REPLAY_SOURCE_EXPLICIT=false +REPLAY_TARGET_EXPLICIT=false +REPLAY_ORDER_EXPLICIT=false +declare -a PLUGIN_BASE_OVERRIDES=() usage() { cat <<'EOF' @@ -40,9 +54,9 @@ Usage: bare path, and do not put a bare path after --repo (that is rejected). Examples: - unsubmodulize.sh ~/workspace/moodle - unsubmodulize.sh --dry-run ~/workspace/moodle - unsubmodulize.sh ~/workspace/moodle --no-commit --ssh + unsubmodulize.sh --no-replay ~/workspace/moodle + unsubmodulize.sh --dry-run --no-replay ~/workspace/moodle + unsubmodulize.sh --fork-point abc123 --source submodulized --target unsubmodulized Options: --dry-run Print what would happen without changing the repo @@ -51,6 +65,16 @@ Options: --manifest PATH Plugin manifest (default: ROOT/plugin-submodules.manifest) --repo ROOT Moodle git root (explicit form of a bare ROOT; overrides an earlier bare ROOT; a bare path after --repo is an error) +Replay (default — one superproject commit per plugin-repo commit on --target; --fork-point required): + --no-replay One-shot mode: convert submodules per manifest (no branch replay) + --replay Replay mode (default; explicit if you toggled --no-replay earlier on the command line) + --fork-point BASE Superproject commit where replay starts (parent of first synthetic commit) + --source BR Branch with submodule pointers for end SHAs (default: submodulized) + --target BR Branch to create/update (default: unsubmodulized) + --order NAME Only chronological (committer date, then path, then SHA) + --force Overwrite --target branch if it exists (replay only) + --plugin-base P=S Optional start SHA for manifest path P (e.g. when BASE has no gitlink) + Requires: git, and clean submodule working trees for paths being converted (no uncommitted changes inside submodules). Private GitHub over HTTPS: set GITHUB_TOKEN (PAT) so ls-remote / clone can authenticate (same pattern as submodulize.sh), @@ -63,6 +87,35 @@ while [[ $# -gt 0 ]]; do --dry-run) DRY_RUN=true; shift ;; --no-commit) NO_COMMIT=true; shift ;; --ssh) USE_SSH=true; shift ;; + --no-replay) REPLAY=false; shift ;; + --replay) REPLAY=true; shift ;; + --fork-point) + FORK_POINT="${2:?}" + shift 2 + ;; + --source) + SOURCE_BRANCH="${2:?}" + REPLAY_SOURCE_EXPLICIT=true + shift 2 + ;; + --target) + TARGET_BRANCH="${2:?}" + REPLAY_TARGET_EXPLICIT=true + shift 2 + ;; + --order) + REPLAY_ORDER="${2:?}" + REPLAY_ORDER_EXPLICIT=true + shift 2 + ;; + --force) + FORCE_REPLAY=true + shift + ;; + --plugin-base) + PLUGIN_BASE_OVERRIDES+=("${2:?}") + shift 2 + ;; --manifest) MANIFEST="${2:?}" MANIFEST_EXPLICIT=true @@ -107,6 +160,42 @@ if [[ ! -f "$MANIFEST" ]]; then exit 1 fi +if $REPLAY; then + [[ -n "$FORK_POINT" ]] || { + echo "Required: --fork-point BASE (replay is the default; use --no-replay for one-shot conversion without replay)" >&2 + exit 1 + } + [[ "$REPLAY_ORDER" == "chronological" ]] || { + echo "Unsupported --order (only chronological): $REPLAY_ORDER" >&2 + exit 1 + } +else + [[ -z "$FORK_POINT" ]] || { + echo "--fork-point is only used in replay mode (omit --no-replay)" >&2 + exit 1 + } + ((${#PLUGIN_BASE_OVERRIDES[@]} == 0)) || { + echo "--plugin-base is only valid in replay mode (omit --no-replay)" >&2 + exit 1 + } + $FORCE_REPLAY && { + echo "--force is only valid in replay mode (omit --no-replay)" >&2 + exit 1 + } + $REPLAY_SOURCE_EXPLICIT && { + echo "--source is only valid in replay mode (omit --no-replay)" >&2 + exit 1 + } + $REPLAY_TARGET_EXPLICIT && { + echo "--target is only valid in replay mode (omit --no-replay)" >&2 + exit 1 + } + $REPLAY_ORDER_EXPLICIT && { + echo "--order is only valid in replay mode (omit --no-replay)" >&2 + exit 1 + } +fi + cd "$REPO_ROOT" disable_sparse_checkout_if_needed() { @@ -146,6 +235,234 @@ rewrite_github_url_to_ssh() { fi } +unsubmodulize_replay_mode() { + git rev-parse --verify "$FORK_POINT^{commit}" >/dev/null + git rev-parse --verify "$SOURCE_BRANCH^{commit}" >/dev/null + + if git show-ref --verify --quiet "refs/heads/$TARGET_BRANCH" && ! $FORCE_REPLAY; then + echo "Branch $TARGET_BRANCH exists; pass --force to overwrite" >&2 + exit 1 + fi + + ls_path_mode_sha() { + local commit="$1" path="$2" + git ls-tree "$commit" -- "$path" 2>/dev/null | awk '{print $1 "\t" $3}' | head -n1 + } + + plugin_base_override_for() { + local want="$1" + local entry + for entry in "${PLUGIN_BASE_OVERRIDES[@]:-}"; do + if [[ "${entry%%=*}" == "$want" ]]; then + printf '%s\n' "${entry#*=}" + return 0 + fi + done + return 1 + } + + find_plugin_commit_for_tree() { + local pdir="$1" want_tree="$2" c t + while IFS= read -r c; do + t="$(git -C "$pdir" rev-parse "$c^{tree}" 2>/dev/null || true)" + [[ "$t" == "$want_tree" ]] && { echo "$c"; return 0; } + done < <(git -C "$pdir" rev-list --first-parent --reverse --all) + return 1 + } + + declare -a M_PATHS=() + declare -a M_URLS=() + declare -a M_BRANCHES=() + + while IFS='|' read -r raw_path raw_url raw_branch; do + path="${raw_path#"${raw_path%%[![:space:]]*}"}" + path="${path%"${path##*[![:space:]]}"}" + url="${raw_url#"${raw_url%%[![:space:]]*}"}" + url="${url%"${url##*[![:space:]]}"}" + branch="${raw_branch#"${raw_branch%%[![:space:]]*}"}" + branch="${branch%"${branch##*[![:space:]]}"}" + [[ -z "$path" || "$path" =~ ^# ]] && continue + [[ -z "$url" ]] && { echo "Manifest: missing URL for path $path" >&2; exit 1; } + [[ -z "$branch" ]] && branch="main" + M_PATHS+=("$path") + M_URLS+=("$url") + M_BRANCHES+=("$branch") + done < "$MANIFEST" + + if [[ ${#M_PATHS[@]} -eq 0 ]]; then + echo "No manifest entries." >&2 + exit 1 + fi + + SOURCE_TIP="$(git rev-parse "$SOURCE_BRANCH^{commit}")" + + TMPD="$(mktemp -d "${TMPDIR:-/tmp}/unsub-replay.XXXXXX")" + trap 'rm -rf "$TMPD"' EXIT + + declare -a EVENT_LINES=() + + for i in "${!M_PATHS[@]}"; do + P="${M_PATHS[$i]}" + URL="${M_URLS[$i]}" + URL="$(rewrite_github_url_to_ssh "$URL")" + + ms="$(ls_path_mode_sha "$FORK_POINT" "$P")" + mode="${ms%%$'\t'*}" + from_sha="${ms#*$'\t'}" + + ms2="$(ls_path_mode_sha "$SOURCE_TIP" "$P")" + mode2="${ms2%%$'\t'*}" + to_sha="${ms2#*$'\t'}" + + if [[ "$mode2" != "160000" ]] || [[ -z "$to_sha" ]]; then + echo "Path $P is not a gitlink on $SOURCE_BRANCH tip — cannot determine end SHA" >&2 + exit 1 + fi + + PDIR="$TMPD/plugin_${i}_$(echo "$P" | tr '/' '_')" + if [[ ! -d "$PDIR/.git" ]]; then + GIT_TERMINAL_PROMPT=0 git "${git_github_pat_c[@]}" clone --bare "$URL" "$PDIR" + fi + + GIT_TERMINAL_PROMPT=0 git -C "$PDIR" fetch -q origin 2>/dev/null || true + GIT_TERMINAL_PROMPT=0 git -C "$PDIR" fetch -q "$URL" "+refs/*:refs/remotes/import/*" 2>/dev/null || true + + if plugin_base_override_for "$P" >/dev/null; then + from_sha="$(plugin_base_override_for "$P")" + elif [[ "$mode" == "160000" && -n "$from_sha" ]]; then + : + elif [[ "$mode" == "040000" ]]; then + tr_sha="$(git rev-parse "$FORK_POINT:$P^{tree}" 2>/dev/null || true)" + if [[ -z "$tr_sha" ]]; then + echo "No tree at $FORK_POINT:$P" >&2 + exit 1 + fi + from_sha="$(find_plugin_commit_for_tree "$PDIR" "$tr_sha")" || { + echo "Could not find plugin commit matching tree at $FORK_POINT:$P" >&2 + exit 1 + } + else + echo "Path $P at fork-point must be gitlink or directory; use --plugin-base ${P}=SHA" >&2 + exit 1 + fi + + if ! GIT_TERMINAL_PROMPT=0 git -C "$PDIR" cat-file -e "${from_sha}^{commit}" 2>/dev/null; then + echo "Plugin object $from_sha not found for path $P (fetch the remote?)" >&2 + exit 1 + fi + if ! GIT_TERMINAL_PROMPT=0 git -C "$PDIR" cat-file -e "${to_sha}^{commit}" 2>/dev/null; then + echo "Plugin object $to_sha not found for path $P" >&2 + exit 1 + fi + + mapfile -t PCOMMITS < <(GIT_TERMINAL_PROMPT=0 git -C "$PDIR" rev-list --first-parent --reverse "${from_sha}..${to_sha}") + + if [[ ${#PCOMMITS[@]} -eq 0 ]]; then + echo "No plugin commits in range ${from_sha}..${to_sha} for $P (already at tip?)" >&2 + continue + fi + + for c in "${PCOMMITS[@]}"; do + ct="$(git -C "$PDIR" show -s --format=%ct "$c")" + EVENT_LINES+=("${ct}"$'\t'"${P}"$'\t'"${c}"$'\t'"${PDIR}") + done + done + + if [[ ${#EVENT_LINES[@]} -eq 0 ]]; then + echo "No plugin commits to replay." >&2 + exit 0 + fi + + IFS=$'\n' + sorted="$(printf '%s\n' "${EVENT_LINES[@]}" | LC_ALL=C sort -t $'\t' -k1,1n -k2,2 -k3,3)" + unset IFS + + if $DRY_RUN; then + echo "Planned ${#EVENT_LINES[@]} commits on $TARGET_BRANCH (from $FORK_POINT)" + while IFS=$'\t' read -r ct p sha pdir; do + [[ -z "${ct:-}" ]] && continue + echo " $ct $p $sha ($(git -C "$pdir" show -s --format=%s "$sha"))" + done <<< "$sorted" + exit 0 + fi + + if $FORCE_REPLAY && git show-ref --verify --quiet "refs/heads/$TARGET_BRANCH"; then + git branch -D "$TARGET_BRANCH" 2>/dev/null || true + fi + git worktree prune 2>/dev/null || true + + WT="$TMPD/wt" + rm -rf "$WT" + FORK_R="$(git rev-parse "$FORK_POINT^{commit}")" + + materialize_commit_at() { + local commit="$1" dest="$2" + rm -rf "${dest:?}/"* + mkdir -p "$dest" + git archive --format=tar "$commit" | tar -x -C "$dest" + local _i _P _ms _mode _sha _PDIR + for _i in "${!M_PATHS[@]}"; do + _P="${M_PATHS[$_i]}" + _ms="$(ls_path_mode_sha "$commit" "$_P")" + _mode="${_ms%%$'\t'*}" + _sha="${_ms#*$'\t'}" + if [[ "$_mode" == "160000" && -n "$_sha" ]]; then + _PDIR="$TMPD/plugin_${_i}_$(echo "$_P" | tr '/' '_')" + rm -rf "$dest/$_P" + mkdir -p "$dest/$_P" + git -C "$_PDIR" archive --format=tar "$_sha" | tar -x -C "$dest/$_P" + fi + done + rm -f "$dest/.gitmodules" + } + + fill_worktree_from_parent() { + local parent="$1" dest="$2" + if [[ "$(git rev-parse "$parent^{commit}")" == "$FORK_R" ]]; then + materialize_commit_at "$FORK_POINT" "$dest" + else + rm -rf "${dest:?}/"* + mkdir -p "$dest" + git archive --format=tar "$parent" | tar -x -C "$dest" + fi + } + + GIT_TERMINAL_PROMPT=0 git "${git_github_pat_c[@]}" worktree add -B "$TARGET_BRANCH" "$WT" "$FORK_POINT" --force + + while IFS=$'\t' read -r ct P csha pdir; do + [[ -z "${ct:-}" ]] && continue + cur="$(git -C "$WT" rev-parse HEAD)" + fill_worktree_from_parent "$cur" "$WT" + rm -rf "${WT:?}/${P}" + mkdir -p "${WT}/${P}" + git -C "$pdir" archive --format=tar "$csha" | tar -x -C "${WT}/${P}" + rm -f "$WT/.gitmodules" + + an="$(git -C "$pdir" show -s --format=%an "$csha")" + ae="$(git -C "$pdir" show -s --format=%ae "$csha")" + adate="$(git -C "$pdir" show -s --format=%ai "$csha")" + body="$(git -C "$pdir" show -s --format=%B "$csha")" + { + printf '%s\n\n' "$body" + printf 'Replayed-from: %s\n' "$csha" + printf 'Plugin-path: %s\n' "$P" + } >"$TMPD/commitmsg.txt" + + git -C "$WT" add -A + GIT_AUTHOR_NAME="$an" GIT_AUTHOR_EMAIL="$ae" GIT_AUTHOR_DATE="$adate" \ + GIT_COMMITTER_NAME="$an" GIT_COMMITTER_EMAIL="$ae" GIT_COMMITTER_DATE="$adate" \ + git -C "$WT" commit -F "$TMPD/commitmsg.txt" + done <<< "$sorted" + + git -C "$REPO_ROOT" worktree remove -f "$WT" 2>/dev/null || true + echo "Done. Branch $TARGET_BRANCH -> $(git rev-parse "$TARGET_BRANCH")" +} + +if $REPLAY; then + unsubmodulize_replay_mode + exit 0 +fi + manifest_entries=0 while IFS='|' read -r raw_path raw_url raw_branch; do path="${raw_path#"${raw_path%%[![:space:]]*}"}" diff --git a/demo/sub b/demo/sub index a3d253c..2103bac 160000 --- a/demo/sub +++ b/demo/sub @@ -1 +1 @@ -Subproject commit a3d253c7431987438bc2d413a2c6c31f6236f752 +Subproject commit 2103bac06d1d55ddfa5ae3cd4509d56f30c81d8f diff --git a/new.sh b/new.sh index 9ca20e1..b430821 100755 --- a/new.sh +++ b/new.sh @@ -167,7 +167,7 @@ while [[ $# -gt 0 ]]; do Options: -h --help Shows this text. -s --skip Skip automatic Moodle installation. - --submodulize After merge, use cleandev/submodulize.sh for manifest plugins (submodules). + --submodulize After merge, run cleandev/submodulize.sh --no-replay for manifest plugins (submodules). Safe on cleandev-style trees: paths already in .gitmodules are skipped (no bulk no-op run). GitHub auth (private lsuonline/*): GITHUB_TOKEN or GH_TOKEN, file cleandev/.github-token, or an interactive prompt (first run) saves the token there (gitignored). Alternatively @@ -244,7 +244,7 @@ if [ "$SUBMODULIZE" = true ]; then if bash /tmp/manifest-submodulize-redundant.sh --repo /var/www/html; then echo "new.sh: All manifest plugin paths are already submodules; skipping submodulize.sh." else - bash /tmp/submodulize.sh --repo /var/www/html $SMF + bash /tmp/submodulize.sh --no-replay --repo /var/www/html $SMF fi git update-index --skip-worktree config.php ' From 7e860aaa7a38d2d81f758c39a92b70a0543b5446 Mon Sep 17 00:00:00 2001 From: Steve Mattsen Date: Tue, 14 Apr 2026 15:52:22 -0400 Subject: [PATCH 07/18] Ensure `submodulize.sh` and `unsubmodulize.sh` require bash for execution. Added checks to prevent running with sh, improving script reliability and user guidance. --- cleandev/submodulize.sh | 7 +++++++ cleandev/unsubmodulize.sh | 7 +++++++ 2 files changed, 14 insertions(+) diff --git a/cleandev/submodulize.sh b/cleandev/submodulize.sh index bb3f92a..82d1100 100644 --- a/cleandev/submodulize.sh +++ b/cleandev/submodulize.sh @@ -16,6 +16,13 @@ # Private GitHub repos over HTTPS need credentials. Set GITHUB_TOKEN (PAT) so HTTPS URLs are rewritten # for ls-remote / submodule add (parent repo url.insteadOf is not always applied to submodule clone). # Or use --ssh. In Docker with no TTY you see: "could not read Username for 'https://github.com'". +# +# Requires bash (arrays, pipefail). Do not run as `sh this-script.sh`; use `bash` or execute directly. + +if [ -z "${BASH_VERSION:-}" ]; then + printf '%s: requires bash, not sh. Example: bash %q "$@"\n' "${0##*/}" "$0" >&2 + exit 1 +fi set -euo pipefail diff --git a/cleandev/unsubmodulize.sh b/cleandev/unsubmodulize.sh index e4e9e5b..bdec50a 100644 --- a/cleandev/unsubmodulize.sh +++ b/cleandev/unsubmodulize.sh @@ -16,6 +16,13 @@ # Private GitHub repos over HTTPS: set GITHUB_TOKEN (PAT) so ls-remote / clone use # -c url.https://TOKEN@github.com/.insteadOf=https://github.com/ # (same as submodulize.sh). Or use --ssh. +# +# Requires bash (arrays, pipefail). Do not run as `sh this-script.sh`; use `bash` or execute directly. + +if [ -z "${BASH_VERSION:-}" ]; then + printf '%s: requires bash, not sh. Example: bash %q "$@"\n' "${0##*/}" "$0" >&2 + exit 1 +fi set -euo pipefail From 38b8c7d30e906e86fb9a9175e22e5c2f78f7dcce Mon Sep 17 00:00:00 2001 From: Steve Mattsen Date: Tue, 14 Apr 2026 15:53:42 -0400 Subject: [PATCH 08/18] Update error messages in `submodulize.sh` and `unsubmodulize.sh` to provide clearer usage instructions for executing the scripts with bash. This enhances user guidance and maintains consistency across scripts. --- cleandev/submodulize.sh | 2 +- cleandev/unsubmodulize.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cleandev/submodulize.sh b/cleandev/submodulize.sh index 82d1100..1bdc0c8 100644 --- a/cleandev/submodulize.sh +++ b/cleandev/submodulize.sh @@ -20,7 +20,7 @@ # Requires bash (arrays, pipefail). Do not run as `sh this-script.sh`; use `bash` or execute directly. if [ -z "${BASH_VERSION:-}" ]; then - printf '%s: requires bash, not sh. Example: bash %q "$@"\n' "${0##*/}" "$0" >&2 + printf '%s: requires bash, not sh. Example: bash "%s" ./ ...your args...\n' "${0##*/}" "$0" >&2 exit 1 fi diff --git a/cleandev/unsubmodulize.sh b/cleandev/unsubmodulize.sh index bdec50a..dfb0d3e 100644 --- a/cleandev/unsubmodulize.sh +++ b/cleandev/unsubmodulize.sh @@ -20,7 +20,7 @@ # Requires bash (arrays, pipefail). Do not run as `sh this-script.sh`; use `bash` or execute directly. if [ -z "${BASH_VERSION:-}" ]; then - printf '%s: requires bash, not sh. Example: bash %q "$@"\n' "${0##*/}" "$0" >&2 + printf '%s: requires bash, not sh. Example: bash "%s" ./ ...your args...\n' "${0##*/}" "$0" >&2 exit 1 fi From 242caeea134b9424f542ccaecbc501bc97028251 Mon Sep 17 00:00:00 2001 From: Steve Mattsen Date: Tue, 14 Apr 2026 16:06:18 -0400 Subject: [PATCH 09/18] Enhance `submodulize.sh` and `unsubmodulize.sh` to default `--fork-point` to local `master` or `main` when omitted, improving usability. Update `README.md` and `TEAM-PROCESS.md` to reflect changes in default behavior for `--source` and `--fork-point`, clarifying the replay process and branch handling. Adjust error messages for better user guidance. --- README.md | 6 +-- cleandev/TEAM-PROCESS.md | 2 +- cleandev/submodulize.sh | 88 ++++++++++++++++++++++++++++++--------- cleandev/unsubmodulize.sh | 83 +++++++++++++++++++++++++++++------- 4 files changed, 142 insertions(+), 37 deletions(-) diff --git a/README.md b/README.md index 2d116d7..ab8e23d 100644 --- a/README.md +++ b/README.md @@ -88,7 +88,7 @@ These are **branches on the Moodle repo** (`lsuce-moodle`), not branch names in - Porting changes **both ways** (submodule layout ↔ vendored layout). - **Redoing or replaying commits** with a **provenance note** (e.g. original commit SHA / branch)—so history stays traceable across layouts. -**Current repo state:** This project ships **layout converters** (`submodulize.sh` / `unsubmodulize.sh`). **Replay is the default:** they build `submodulized` / `unsubmodulized` branches with **one superproject commit per plugin-repo commit** (chronological ordering, carry-forward; **`--fork-point`** required). Use **`--no-replay`** for one-shot conversion over the manifest only. A `new.sh` mode that detects vendored vs submodulized state without manual choice is still not implemented. +**Current repo state:** This project ships **layout converters** (`submodulize.sh` / `unsubmodulize.sh`). **Replay is the default:** they build `submodulized` / `unsubmodulized` branches with **one superproject commit per plugin-repo commit** (chronological ordering, carry-forward). **`--fork-point`** defaults to local **`master`** (else **`main`**) when omitted, if a default can be chosen safely; **`--source`** defaults for **unsub** to **`unsubmodulized`**, else **`master`**, else **`main`**, else **`submodulized`** (for **sub**: **`submodulized`**, **`master`**, **`main`**). Use **`--no-replay`** for one-shot conversion over the manifest only. A `new.sh` mode that detects vendored vs submodulized state without manual choice is still not implemented. **Branch policy, Docker image decision, and updating the manifest from the CSV:** [cleandev/TEAM-PROCESS.md](cleandev/TEAM-PROCESS.md). @@ -109,8 +109,8 @@ This repo keeps a **canonical copy** under `cleandev/` for linting and for `new. | Script | Role | |--------|------| -| `cleandev/submodulize.sh` | Default **replay** (`--fork-point`): one superproject commit per plugin commit (gitlinks + `.gitmodules`). **`--no-replay`**: one-shot vendored → submodules (sparse-checkout disabled first; skips paths already in `.gitmodules`; `GITHUB_TOKEN` via `-c url...insteadOf` for `ls-remote` / `submodule add`). | -| `cleandev/unsubmodulize.sh` | Default **replay** (`--fork-point`): one superproject commit per plugin commit (vendored trees). **`--no-replay`**: one-shot submodules → vendored (clone depth 1, drop nested `.git`, `git add`). Same `GITHUB_TOKEN` / `--ssh` as `submodulize.sh`. | +| `cleandev/submodulize.sh` | Default **replay**; **`--fork-point`** defaults to `master`/`main` when omitted (see script help). **`--no-replay`**: one-shot vendored → submodules (sparse-checkout disabled first; skips paths already in `.gitmodules`; `GITHUB_TOKEN` via `-c url...insteadOf` for `ls-remote` / `submodule add`). | +| `cleandev/unsubmodulize.sh` | Default **replay**; same **`--fork-point`** defaulting. **`--no-replay`**: one-shot submodules → vendored (clone depth 1, drop nested `.git`, `git add`). Same `GITHUB_TOKEN` / `--ssh` as `submodulize.sh`. | Run manually from a Moodle clone: diff --git a/cleandev/TEAM-PROCESS.md b/cleandev/TEAM-PROCESS.md index 4edb3a7..b3f79b2 100644 --- a/cleandev/TEAM-PROCESS.md +++ b/cleandev/TEAM-PROCESS.md @@ -4,7 +4,7 @@ This document covers **branch policy** for [lsuce-moodle](https://github.com/lsu **Where the manifest lives:** On **cleandev**, treat `plugin-submodules.manifest` as part of the Moodle superproject (repo root), alongside `.gitmodules`. The copy in **this** repo under [`cleandev/plugin-submodules.manifest`](plugin-submodules.manifest) is the maintained source for CSV refresh and CI lint; `submodulize.sh` / `unsubmodulize.sh` default to `ROOT/plugin-submodules.manifest` unless you pass `--manifest`. -**Branch replay (default):** `unsubmodulize.sh` and `submodulize.sh` run replay unless you pass **`--no-replay`**. Replay builds histories with **one Moodle superproject commit per plugin-repo commit** (chronological order, carry-forward). It requires `--fork-point` and defaults `--source` / `--target` branch names (`submodulized`, `unsubmodulized`). If the fork-point checkout is **vendored** at a manifest path, the script matches the embedded tree to a plugin commit (or use `--plugin-base path=SHA`). `new.sh --submodulize` runs **`submodulize.sh --no-replay`** (one-shot only). +**Branch replay (default):** `unsubmodulize.sh` and `submodulize.sh` run replay unless you pass **`--no-replay`**. Replay builds histories with **one Moodle superproject commit per plugin-repo commit** (chronological order, carry-forward). **`--fork-point`** defaults to local **`master`** (else **`main`**) when omitted, when the script can infer it; otherwise pass it explicitly. **`--source`** defaults when omitted: **unsub** prefers **`unsubmodulized`**, then **`master`**, **`main`**, **`submodulized`**; **sub** prefers **`submodulized`**, **`master`**, **`main`**. Source tips may be **gitlinks or vendored trees** (tree matched to plugin commits). Defaults **`--target`** branch names (`submodulized`, `unsubmodulized`). If the fork-point checkout is **vendored** at a manifest path, the script matches the embedded tree to a plugin commit (or use `--plugin-base path=SHA`). `new.sh --submodulize` runs **`submodulize.sh --no-replay`** (one-shot only). --- diff --git a/cleandev/submodulize.sh b/cleandev/submodulize.sh index 1bdc0c8..e6e01c5 100644 --- a/cleandev/submodulize.sh +++ b/cleandev/submodulize.sh @@ -34,7 +34,7 @@ USE_SSH=false REPO_ROOT="" REPLAY=true FORK_POINT="" -SOURCE_BRANCH="submodulized" +SOURCE_BRANCH="" TARGET_BRANCH="submodulized" REPLAY_ORDER="chronological" FORCE_REPLAY=false @@ -70,11 +70,11 @@ Options: --manifest PATH Plugin manifest (default: ROOT/plugin-submodules.manifest) --repo ROOT Moodle git root (explicit form of a bare ROOT; overrides an earlier bare ROOT; a bare path after --repo is an error) -Replay (default — one superproject commit per plugin-repo commit on --target; --fork-point required): +Replay (default — one superproject commit per plugin-repo commit on --target): --no-replay One-shot mode: add submodules per manifest (no branch replay) --replay Replay mode (default; explicit if you toggled --no-replay earlier on the command line) - --fork-point BASE Start of replay (vendored plugin dirs and/or existing gitlinks) - --source BR Branch whose tip supplies final submodule SHAs per path (default: submodulized) + --fork-point BASE Start of replay (default: local master, else main, when omitted — see docs) + --source BR End state per path (gitlinks and/or vendored trees; default: submodulized, else master, else main) --target BR Branch to create/update (default: submodulized) --order NAME Only chronological --force Overwrite --target branch if it exists (replay only) @@ -167,10 +167,6 @@ if [[ ! -f "$MANIFEST" ]]; then fi if $REPLAY; then - [[ -n "$FORK_POINT" ]] || { - echo "Required: --fork-point BASE (replay is the default; use --no-replay for one-shot conversion without replay)" >&2 - exit 1 - } [[ "$REPLAY_ORDER" == "chronological" ]] || { echo "Only --order chronological is supported: $REPLAY_ORDER" >&2 exit 1 @@ -204,6 +200,51 @@ fi cd "$REPO_ROOT" +# When --fork-point is omitted in replay mode, default to local master (else main) if safe. +if $REPLAY && [[ -z "$FORK_POINT" ]]; then + base_ref="" + if git show-ref --verify --quiet refs/heads/master; then + base_ref=master + elif git show-ref --verify --quiet refs/heads/main; then + base_ref=main + else + echo "Replay needs --fork-point (no local master or main branch to use as default)." >&2 + exit 1 + fi + head_sha="$(git rev-parse HEAD)" + master_sha="$(git rev-parse "${base_ref}^{commit}")" + target_exists=false + git show-ref --verify --quiet "refs/heads/$TARGET_BRANCH" && target_exists=true + if ! $target_exists; then + FORK_POINT="$base_ref" + echo "Default --fork-point $FORK_POINT ($TARGET_BRANCH does not exist yet)." >&2 + else + target_sha="$(git rev-parse "$TARGET_BRANCH^{commit}")" + if [[ "$head_sha" == "$master_sha" ]] || [[ "$head_sha" == "$target_sha" ]]; then + FORK_POINT="$base_ref" + echo "Default --fork-point $FORK_POINT (HEAD is ${base_ref} or $TARGET_BRANCH)." >&2 + else + FORK_POINT="$base_ref" + echo "Default --fork-point $FORK_POINT (recreating $TARGET_BRANCH from ${base_ref}; use --force if the branch already exists)." >&2 + fi + fi +fi + +# When --source is omitted in replay mode: prefer submodulized, else master, else main. +if $REPLAY && ! $REPLAY_SOURCE_EXPLICIT; then + if git show-ref --verify --quiet refs/heads/submodulized; then + SOURCE_BRANCH=submodulized + elif git show-ref --verify --quiet refs/heads/master; then + SOURCE_BRANCH=master + elif git show-ref --verify --quiet refs/heads/main; then + SOURCE_BRANCH=main + else + echo "Replay: pass --source BR (no local submodulized, master, or main branch found)." >&2 + exit 1 + fi + echo "Default --source $SOURCE_BRANCH" >&2 +fi + # Moodle dev images often use sparse-checkout (cone, index.sparse, or only .git/info/sparse-checkout). # If any of that is active, git rm / submodule add can refuse paths "outside" the cone. disable_sparse_checkout_if_needed() { @@ -320,8 +361,26 @@ submodulize_replay_mode() { to_sha="${ms2#*$'\t'}" mode2="${ms2%%$'\t'*}" - if [[ "$mode2" != "160000" ]] || [[ -z "$to_sha" ]]; then - echo "Path $P: source tip must be gitlink for end SHA" >&2 + if [[ ! -d "$PDIR/.git" ]]; then + GIT_TERMINAL_PROMPT=0 git "${git_github_pat_c[@]}" clone --bare "$URL" "$PDIR" + fi + GIT_TERMINAL_PROMPT=0 git -C "$PDIR" fetch -q origin 2>/dev/null || true + GIT_TERMINAL_PROMPT=0 git -C "$PDIR" fetch -q "$URL" "+refs/*:refs/remotes/import/*" 2>/dev/null || true + + if [[ "$mode2" == "160000" && -n "$to_sha" ]]; then + : + elif [[ "$mode2" == "040000" ]]; then + tr_end="$(git rev-parse "$SOURCE_TIP:$P^{tree}" 2>/dev/null || true)" + if [[ -z "$tr_end" ]]; then + echo "No tree at $SOURCE_BRANCH:$P (source tip)" >&2 + exit 1 + fi + to_sha="$(find_plugin_commit_for_tree "$PDIR" "$tr_end")" || { + echo "Could not match plugin commit to vendored tree at $SOURCE_BRANCH:$P" >&2 + exit 1 + } + else + echo "Path $P: source tip must be gitlink or vendored directory for end SHA" >&2 exit 1 fi @@ -334,9 +393,6 @@ submodulize_replay_mode() { echo "No tree at $FORK_POINT:$P" >&2 exit 1 fi - if [[ ! -d "$PDIR/.git" ]]; then - GIT_TERMINAL_PROMPT=0 git "${git_github_pat_c[@]}" clone --bare "$URL" "$PDIR" - fi from_sha="$(find_plugin_commit_for_tree "$PDIR" "$tr_sha")" || { echo "Could not find plugin commit matching tree at $FORK_POINT:$P" >&2 exit 1 @@ -349,12 +405,6 @@ submodulize_replay_mode() { from_sha="$(plugin_base_override_for "$P")" fi - if [[ ! -d "$PDIR/.git" ]]; then - GIT_TERMINAL_PROMPT=0 git "${git_github_pat_c[@]}" clone --bare "$URL" "$PDIR" - fi - GIT_TERMINAL_PROMPT=0 git -C "$PDIR" fetch -q origin 2>/dev/null || true - GIT_TERMINAL_PROMPT=0 git -C "$PDIR" fetch -q "$URL" "+refs/*:refs/remotes/import/*" 2>/dev/null || true - GIT_TERMINAL_PROMPT=0 git -C "$PDIR" cat-file -e "${from_sha}^{commit}" 2>/dev/null || { echo "Missing plugin object $from_sha for $P" >&2 exit 1 diff --git a/cleandev/unsubmodulize.sh b/cleandev/unsubmodulize.sh index dfb0d3e..5a895d3 100644 --- a/cleandev/unsubmodulize.sh +++ b/cleandev/unsubmodulize.sh @@ -34,7 +34,7 @@ USE_SSH=false REPO_ROOT="" REPLAY=true FORK_POINT="" -SOURCE_BRANCH="submodulized" +SOURCE_BRANCH="" TARGET_BRANCH="unsubmodulized" REPLAY_ORDER="chronological" FORCE_REPLAY=false @@ -63,7 +63,7 @@ Usage: Examples: unsubmodulize.sh --no-replay ~/workspace/moodle unsubmodulize.sh --dry-run --no-replay ~/workspace/moodle - unsubmodulize.sh --fork-point abc123 --source submodulized --target unsubmodulized + unsubmodulize.sh --fork-point abc123 --target unsubmodulized Options: --dry-run Print what would happen without changing the repo @@ -72,11 +72,11 @@ Options: --manifest PATH Plugin manifest (default: ROOT/plugin-submodules.manifest) --repo ROOT Moodle git root (explicit form of a bare ROOT; overrides an earlier bare ROOT; a bare path after --repo is an error) -Replay (default — one superproject commit per plugin-repo commit on --target; --fork-point required): +Replay (default — one superproject commit per plugin-repo commit on --target): --no-replay One-shot mode: convert submodules per manifest (no branch replay) --replay Replay mode (default; explicit if you toggled --no-replay earlier on the command line) - --fork-point BASE Superproject commit where replay starts (parent of first synthetic commit) - --source BR Branch with submodule pointers for end SHAs (default: submodulized) + --fork-point BASE Superproject where replay starts (default: local master, else main, when omitted — see docs) + --source BR End state per path: gitlinks and/or vendored trees (default: unsubmodulized, else master, else main, else submodulized) --target BR Branch to create/update (default: unsubmodulized) --order NAME Only chronological (committer date, then path, then SHA) --force Overwrite --target branch if it exists (replay only) @@ -168,10 +168,6 @@ if [[ ! -f "$MANIFEST" ]]; then fi if $REPLAY; then - [[ -n "$FORK_POINT" ]] || { - echo "Required: --fork-point BASE (replay is the default; use --no-replay for one-shot conversion without replay)" >&2 - exit 1 - } [[ "$REPLAY_ORDER" == "chronological" ]] || { echo "Unsupported --order (only chronological): $REPLAY_ORDER" >&2 exit 1 @@ -205,6 +201,53 @@ fi cd "$REPO_ROOT" +# When --fork-point is omitted in replay mode, default to local master (else main) if safe; see resolve below. +if $REPLAY && [[ -z "$FORK_POINT" ]]; then + base_ref="" + if git show-ref --verify --quiet refs/heads/master; then + base_ref=master + elif git show-ref --verify --quiet refs/heads/main; then + base_ref=main + else + echo "Replay needs --fork-point (no local master or main branch to use as default)." >&2 + exit 1 + fi + head_sha="$(git rev-parse HEAD)" + master_sha="$(git rev-parse "${base_ref}^{commit}")" + target_exists=false + git show-ref --verify --quiet "refs/heads/$TARGET_BRANCH" && target_exists=true + if ! $target_exists; then + FORK_POINT="$base_ref" + echo "Default --fork-point $FORK_POINT ($TARGET_BRANCH does not exist yet)." >&2 + else + target_sha="$(git rev-parse "$TARGET_BRANCH^{commit}")" + if [[ "$head_sha" == "$master_sha" ]] || [[ "$head_sha" == "$target_sha" ]]; then + FORK_POINT="$base_ref" + echo "Default --fork-point $FORK_POINT (HEAD is ${base_ref} or $TARGET_BRANCH)." >&2 + else + FORK_POINT="$base_ref" + echo "Default --fork-point $FORK_POINT (recreating $TARGET_BRANCH from ${base_ref}; use --force if the branch already exists)." >&2 + fi + fi +fi + +# When --source is omitted in replay mode: prefer unsubmodulized (vendored end state), else master/main/submodulized. +if $REPLAY && ! $REPLAY_SOURCE_EXPLICIT; then + if git show-ref --verify --quiet "refs/heads/unsubmodulized"; then + SOURCE_BRANCH=unsubmodulized + elif git show-ref --verify --quiet refs/heads/master; then + SOURCE_BRANCH=master + elif git show-ref --verify --quiet refs/heads/main; then + SOURCE_BRANCH=main + elif git show-ref --verify --quiet refs/heads/submodulized; then + SOURCE_BRANCH=submodulized + else + echo "Replay: pass --source BR (no local unsubmodulized, master, main, or submodulized branch found)." >&2 + exit 1 + fi + echo "Default --source $SOURCE_BRANCH" >&2 +fi + disable_sparse_checkout_if_needed() { $DRY_RUN && return 0 local active= @@ -321,11 +364,6 @@ unsubmodulize_replay_mode() { mode2="${ms2%%$'\t'*}" to_sha="${ms2#*$'\t'}" - if [[ "$mode2" != "160000" ]] || [[ -z "$to_sha" ]]; then - echo "Path $P is not a gitlink on $SOURCE_BRANCH tip — cannot determine end SHA" >&2 - exit 1 - fi - PDIR="$TMPD/plugin_${i}_$(echo "$P" | tr '/' '_')" if [[ ! -d "$PDIR/.git" ]]; then GIT_TERMINAL_PROMPT=0 git "${git_github_pat_c[@]}" clone --bare "$URL" "$PDIR" @@ -334,6 +372,23 @@ unsubmodulize_replay_mode() { GIT_TERMINAL_PROMPT=0 git -C "$PDIR" fetch -q origin 2>/dev/null || true GIT_TERMINAL_PROMPT=0 git -C "$PDIR" fetch -q "$URL" "+refs/*:refs/remotes/import/*" 2>/dev/null || true + if [[ "$mode2" == "160000" && -n "$to_sha" ]]; then + : + elif [[ "$mode2" == "040000" ]]; then + tr_end="$(git rev-parse "$SOURCE_TIP:$P^{tree}" 2>/dev/null || true)" + if [[ -z "$tr_end" ]]; then + echo "No tree at $SOURCE_BRANCH:$P (source tip)" >&2 + exit 1 + fi + to_sha="$(find_plugin_commit_for_tree "$PDIR" "$tr_end")" || { + echo "Could not match plugin commit to vendored tree at $SOURCE_BRANCH:$P" >&2 + exit 1 + } + else + echo "Path $P at $SOURCE_BRANCH tip must be gitlink or vendored directory — cannot determine end SHA" >&2 + exit 1 + fi + if plugin_base_override_for "$P" >/dev/null; then from_sha="$(plugin_base_override_for "$P")" elif [[ "$mode" == "160000" && -n "$from_sha" ]]; then From 87ed036f15ddb92a00e502afdac3738c412c5e4c Mon Sep 17 00:00:00 2001 From: Steve Mattsen Date: Tue, 14 Apr 2026 16:19:36 -0400 Subject: [PATCH 10/18] Enhance `submodulize.sh` by introducing a `--bootstrap` option for streamlined submodule management. This option allows for automatic creation of submodules from a vendored tree and handles cases where the submodulized branch does not exist. Update usage instructions and error handling to improve user experience and prevent conflicting options. Additionally, implement a new function for applying the manifest in one-shot mode, ensuring better management of plugin submodules. --- cleandev/submodulize.sh | 273 ++++++++++++++++++++----------- cleandev/tests/test-bootstrap.sh | 39 +++++ 2 files changed, 216 insertions(+), 96 deletions(-) create mode 100644 cleandev/tests/test-bootstrap.sh diff --git a/cleandev/submodulize.sh b/cleandev/submodulize.sh index e6e01c5..b94c042 100644 --- a/cleandev/submodulize.sh +++ b/cleandev/submodulize.sh @@ -26,6 +26,8 @@ fi set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + MANIFEST="" MANIFEST_EXPLICIT=false DRY_RUN=false @@ -41,6 +43,9 @@ FORCE_REPLAY=false REPLAY_SOURCE_EXPLICIT=false REPLAY_TARGET_EXPLICIT=false REPLAY_ORDER_EXPLICIT=false +FORK_POINT_EXPLICIT=false +BOOTSTRAP=false +BOOTSTRAP_EXPLICIT=false declare -a PLUGIN_BASE_OVERRIDES=() usage() { @@ -60,6 +65,8 @@ Usage: Examples: submodulize.sh --no-replay ~/workspace/moodle + submodulize.sh ./ --bootstrap + submodulize.sh ./ # same as --bootstrap when submodulized branch does not exist yet submodulize.sh --dry-run --no-replay ~/workspace/moodle submodulize.sh --fork-point abc123 --source submodulized @@ -70,6 +77,10 @@ Options: --manifest PATH Plugin manifest (default: ROOT/plugin-submodules.manifest) --repo ROOT Moodle git root (explicit form of a bare ROOT; overrides an earlier bare ROOT; a bare path after --repo is an error) +Bootstrap (from vendored tree + manifest → submodulized + unsubmodulized): + --bootstrap Add submodules on branch submodulized, then run unsubmodulize replay to create unsubmodulized. + If local branch submodulized is missing, this runs automatically (same as passing --bootstrap). + Replay (default — one superproject commit per plugin-repo commit on --target): --no-replay One-shot mode: add submodules per manifest (no branch replay) --replay Replay mode (default; explicit if you toggled --no-replay earlier on the command line) @@ -95,8 +106,15 @@ while [[ $# -gt 0 ]]; do --ssh) USE_SSH=true; shift ;; --no-replay) REPLAY=false; shift ;; --replay) REPLAY=true; shift ;; + --bootstrap) + BOOTSTRAP=true + BOOTSTRAP_EXPLICIT=true + REPLAY=false + shift + ;; --fork-point) FORK_POINT="${2:?}" + FORK_POINT_EXPLICIT=true shift 2 ;; --source) @@ -166,11 +184,45 @@ if [[ ! -f "$MANIFEST" ]]; then exit 1 fi +if $BOOTSTRAP_EXPLICIT; then + $FORK_POINT_EXPLICIT && { + echo "Do not combine --bootstrap with --fork-point (bootstrap records the current commit as the fork)." >&2 + exit 1 + } + $REPLAY_SOURCE_EXPLICIT && { + echo "Do not combine --bootstrap with --source (bootstrap uses submodulized → unsubmodulized)." >&2 + exit 1 + } + $REPLAY_TARGET_EXPLICIT && { + echo "Do not combine --bootstrap with --target." >&2 + exit 1 + } + $REPLAY_ORDER_EXPLICIT && { + echo "Do not combine --bootstrap with --order." >&2 + exit 1 + } + ((${#PLUGIN_BASE_OVERRIDES[@]} == 0)) || { + echo "--plugin-base is not used with --bootstrap." >&2 + exit 1 + } +fi + +cd "$REPO_ROOT" + +# Greenfield: no submodulized branch yet — run bootstrap (one-shot + unsub replay) instead of sub replay. +if $REPLAY && ! $FORK_POINT_EXPLICIT && ! $BOOTSTRAP_EXPLICIT && ! git show-ref --verify --quiet refs/heads/submodulized; then + BOOTSTRAP=true + REPLAY=false + echo "submodulize: no local branch submodulized — bootstrap (add submodules, then build unsubmodulized)." >&2 +fi + if $REPLAY; then [[ "$REPLAY_ORDER" == "chronological" ]] || { echo "Only --order chronological is supported: $REPLAY_ORDER" >&2 exit 1 } +elif $BOOTSTRAP; then + : else [[ -z "$FORK_POINT" ]] || { echo "--fork-point is only used in replay mode (omit --no-replay)" >&2 @@ -181,7 +233,7 @@ else exit 1 } $FORCE_REPLAY && { - echo "--force is only valid in replay mode (omit --no-replay)" >&2 + echo "--force is only valid in replay mode or bootstrap (omit --no-replay)" >&2 exit 1 } $REPLAY_SOURCE_EXPLICIT && { @@ -198,8 +250,6 @@ else } fi -cd "$REPO_ROOT" - # When --fork-point is omitted in replay mode, default to local master (else main) if safe. if $REPLAY && [[ -z "$FORK_POINT" ]]; then base_ref="" @@ -293,6 +343,124 @@ run() { fi } +submodulize_one_shot_apply_manifest() { + manifest_entries=0 + while IFS='|' read -r raw_path raw_url raw_branch; do + path="${raw_path#"${raw_path%%[![:space:]]*}"}" + path="${path%"${path##*[![:space:]]}"}" + url="${raw_url#"${raw_url%%[![:space:]]*}"}" + url="${url%"${url##*[![:space:]]}"}" + branch="${raw_branch#"${raw_branch%%[![:space:]]*}"}" + branch="${branch%"${branch##*[![:space:]]}"}" + + [[ -z "$path" || "$path" =~ ^# ]] && continue + [[ -z "$url" ]] && { echo "Manifest: missing URL for path $path" >&2; exit 1; } + [[ -z "$branch" ]] && branch="main" + url="$(rewrite_github_url_to_ssh "$url")" + + ((++manifest_entries)) || true + + if [[ -f .gitmodules ]] && git config -f .gitmodules --get-regexp path 2>/dev/null | awk '{print $2}' | grep -Fxq "$path"; then + echo "Already a submodule (per .gitmodules): $path — skipping" + continue + fi + + if [[ -d "$path/.git" ]] || [[ -f "$path/.git" ]]; then + echo "Path already looks like a nested git repo: $path" >&2 + echo " Remove or convert it manually, or run unsubmodulize first." >&2 + exit 1 + fi + + if [[ -e "$path" ]] && ! $DRY_RUN; then + if ! git diff --quiet -- "$path" 2>/dev/null || ! git diff --cached --quiet -- "$path" 2>/dev/null; then + echo "Uncommitted changes under $path — commit or stash first." >&2 + exit 1 + fi + fi + + echo "Submodulizing: $path ← $url (branch $branch)" + + parent="$(dirname "$path")" + if [[ "$parent" != "." ]]; then + run mkdir -p "$parent" + fi + + if [[ -n "$(git ls-files -- "$path" 2>/dev/null)" ]]; then + if $DRY_RUN; then + printf '[dry-run] git rm -rf [--sparse] -- %q\n' "$path" + else + git rm -rf --sparse -- "$path" 2>/dev/null || git rm -rf -- "$path" + fi + elif [[ -e "$path" ]]; then + run rm -rf -- "$path" + fi + + if $DRY_RUN; then + printf '[dry-run] git submodule add -f (-b %q if exists on remote, else default branch) -- %q %q\n' "$branch" "$url" "$path" + else + submod_args=(-f) + if [[ -n "$branch" ]] && GIT_TERMINAL_PROMPT=0 git "${git_github_pat_c[@]}" ls-remote --heads "$url" "refs/heads/$branch" 2>/dev/null | grep -q .; then + submod_args+=(-b "$branch") + else + [[ -n "$branch" ]] && echo "Remote has no branch '$branch' for $path; using repository default branch." >&2 + fi + if ! GIT_TERMINAL_PROMPT=0 git "${git_github_pat_c[@]}" -c core.sparseCheckout=false -c index.sparse=false submodule add "${submod_args[@]}" -- "$url" "$path"; then + echo "submodulize: failed to add submodule $path ← $url" >&2 + echo " If the repo is private: configure HTTPS credentials, or re-run with --ssh (needs GitHub SSH access)." >&2 + echo " After a failed add you may need: git submodule deinit -f -- $path 2>/dev/null; rm -rf .git/modules/$path $path" >&2 + exit 1 + fi + fi + done < "$MANIFEST" + + if [[ "$manifest_entries" -eq 0 ]]; then + echo "No entries in manifest." >&2 + exit 1 + fi + + if $DRY_RUN; then + echo "Dry run complete." + $BOOTSTRAP && echo "Bootstrap: would commit on submodulized, then run unsubmodulize replay (omit --dry-run)." >&2 + exit 0 + fi + + if ! $NO_COMMIT; then + if git diff --cached --quiet 2>/dev/null; then + echo "Nothing staged; skipping commit." + else + git commit -m "chore: add plugin submodules per plugin-submodules.manifest" + fi + fi + + $BOOTSTRAP || echo "Done. Submodule layout is ready (clean repo)." +} + +submodulize_bootstrap_pipeline() { + local vend_tip unsub_args + vend_tip="$(git rev-parse HEAD)" + echo "Bootstrap: vendored fork-point for unsub replay: $vend_tip" >&2 + + if git show-ref --verify --quiet refs/heads/submodulized && ! $FORCE_REPLAY; then + echo "Branch submodulized already exists. Use --force to replace it, delete the branch, or run without bootstrap." >&2 + exit 1 + fi + $FORCE_REPLAY && git branch -D submodulized 2>/dev/null || true + + git checkout -B submodulized + + submodulize_one_shot_apply_manifest + + if $DRY_RUN; then + exit 0 + fi + + unsub_args=(--repo "$REPO_ROOT" --fork-point "$vend_tip" --source submodulized --target unsubmodulized) + $FORCE_REPLAY && unsub_args+=(--force) + + bash "$SCRIPT_DIR/unsubmodulize.sh" "${unsub_args[@]}" + echo "Bootstrap complete: submodulized (submodules) and unsubmodulized (vendored replay) are ready." >&2 +} + submodulize_replay_mode() { git rev-parse --verify "$FORK_POINT^{commit}" >/dev/null git rev-parse --verify "$SOURCE_BRANCH^{commit}" >/dev/null @@ -495,102 +663,15 @@ submodulize_replay_mode() { echo "Done. Branch $TARGET_BRANCH -> $(git rev-parse "$TARGET_BRANCH")" } -if $REPLAY; then - submodulize_replay_mode +if $BOOTSTRAP; then + submodulize_bootstrap_pipeline exit 0 fi -manifest_entries=0 -while IFS='|' read -r raw_path raw_url raw_branch; do - path="${raw_path#"${raw_path%%[![:space:]]*}"}" - path="${path%"${path##*[![:space:]]}"}" - url="${raw_url#"${raw_url%%[![:space:]]*}"}" - url="${url%"${url##*[![:space:]]}"}" - branch="${raw_branch#"${raw_branch%%[![:space:]]*}"}" - branch="${branch%"${branch##*[![:space:]]}"}" - - [[ -z "$path" || "$path" =~ ^# ]] && continue - [[ -z "$url" ]] && { echo "Manifest: missing URL for path $path" >&2; exit 1; } - [[ -z "$branch" ]] && branch="main" - url="$(rewrite_github_url_to_ssh "$url")" - - ((++manifest_entries)) || true - - if [[ -f .gitmodules ]] && git config -f .gitmodules --get-regexp path 2>/dev/null | awk '{print $2}' | grep -Fxq "$path"; then - echo "Already a submodule (per .gitmodules): $path — skipping" - continue - fi - - if [[ -d "$path/.git" ]] || [[ -f "$path/.git" ]]; then - echo "Path already looks like a nested git repo: $path" >&2 - echo " Remove or convert it manually, or run unsubmodulize first." >&2 - exit 1 - fi - - if [[ -e "$path" ]] && ! $DRY_RUN; then - if ! git diff --quiet -- "$path" 2>/dev/null || ! git diff --cached --quiet -- "$path" 2>/dev/null; then - echo "Uncommitted changes under $path — commit or stash first." >&2 - exit 1 - fi - fi - - echo "Submodulizing: $path ← $url (branch $branch)" - - parent="$(dirname "$path")" - if [[ "$parent" != "." ]]; then - run mkdir -p "$parent" - fi - - # Use tracked-file listing, not --error-unmatch: for a directory, Git often has no - # single index entry named exactly $path, only files underneath — then plain rm - # would leave the path in the index and submodule add fails. - if [[ -n "$(git ls-files -- "$path" 2>/dev/null)" ]]; then - if $DRY_RUN; then - printf '[dry-run] git rm -rf [--sparse] -- %q\n' "$path" - else - git rm -rf --sparse -- "$path" 2>/dev/null || git rm -rf -- "$path" - fi - elif [[ -e "$path" ]]; then - run rm -rf -- "$path" - fi - - if $DRY_RUN; then - printf '[dry-run] git submodule add -f (-b %q if exists on remote, else default branch) -- %q %q\n' "$branch" "$url" "$path" - else - # -f: Moodle .gitignore often ignores plugin dirs; without it submodule add fails. - # -c overrides: sparse-checkout can still block the add otherwise. - # Many upstream Moodle plugins use master or MOODLE_*_STABLE, not main — omit -b to use remote HEAD. - submod_args=(-f) - if [[ -n "$branch" ]] && GIT_TERMINAL_PROMPT=0 git "${git_github_pat_c[@]}" ls-remote --heads "$url" "refs/heads/$branch" 2>/dev/null | grep -q .; then - submod_args+=(-b "$branch") - else - [[ -n "$branch" ]] && echo "Remote has no branch '$branch' for $path; using repository default branch." >&2 - fi - if ! GIT_TERMINAL_PROMPT=0 git "${git_github_pat_c[@]}" -c core.sparseCheckout=false -c index.sparse=false submodule add "${submod_args[@]}" -- "$url" "$path"; then - echo "submodulize: failed to add submodule $path ← $url" >&2 - echo " If the repo is private: configure HTTPS credentials, or re-run with --ssh (needs GitHub SSH access)." >&2 - echo " After a failed add you may need: git submodule deinit -f -- $path 2>/dev/null; rm -rf .git/modules/$path $path" >&2 - exit 1 - fi - fi -done < "$MANIFEST" - -if [[ "$manifest_entries" -eq 0 ]]; then - echo "No entries in manifest." >&2 - exit 1 -fi - -if $DRY_RUN; then - echo "Dry run complete." +if $REPLAY; then + submodulize_replay_mode exit 0 fi -if ! $NO_COMMIT; then - if git diff --cached --quiet 2>/dev/null; then - echo "Nothing staged; skipping commit." - else - git commit -m "chore: add plugin submodules per plugin-submodules.manifest" - fi -fi - -echo "Done. Submodule layout is ready (clean repo)." +submodulize_one_shot_apply_manifest +exit 0 diff --git a/cleandev/tests/test-bootstrap.sh b/cleandev/tests/test-bootstrap.sh new file mode 100644 index 0000000..90aa7a0 --- /dev/null +++ b/cleandev/tests/test-bootstrap.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +# Bootstrap: submodulize.sh with no submodulized branch → one-shot + unsub replay (file:// plugin, no network). +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=lib.sh +source "$SCRIPT_DIR/lib.sh" +require_cleandev + +TMP=$(mktemp -d) +export HOME="$TMP/_home" +mkdir -p "$HOME" +git config --global protocol.file.allow always +trap 'rm -rf "$TMP"' EXIT + +PLUGIN="$TMP/plugin-upstream" +MOODLE="$TMP/moodle" +mkdir -p "$PLUGIN" +git -C "$PLUGIN" init -b main +echo "plug=1" > "$PLUGIN/version.txt" +git -C "$PLUGIN" add version.txt +git -C "$PLUGIN" commit -q -m "init plugin" + +mkdir -p "$MOODLE/mod/testplugin" +git -C "$MOODLE" init -b master +echo "vendored-only" > "$MOODLE/mod/testplugin/local.txt" +git -C "$MOODLE" add mod/testplugin +git -C "$MOODLE" commit -q -m "vendored plugin" + +printf '%s\n' 'mod/testplugin|../plugin-upstream|main' > "$MOODLE/plugin-submodules.manifest" + +# No local submodulized branch → auto-bootstrap (same as explicit --bootstrap) +"$CLEANDEV/submodulize.sh" --repo "$MOODLE" --no-commit + +git -C "$MOODLE" show-ref --verify --quiet refs/heads/submodulized || fail "expected submodulized branch" +git -C "$MOODLE" show-ref --verify --quiet refs/heads/unsubmodulized || fail "expected unsubmodulized branch" +git -C "$MOODLE" ls-tree -r submodulized --name-only | grep -q '^mod/testplugin/version.txt$' || fail "expected submodule file on submodulized" +git -C "$MOODLE" ls-tree -r unsubmodulized --name-only | grep -q '^mod/testplugin/version.txt$' || fail "expected vendored file on unsubmodulized" + +ok "submodulize bootstrap (auto when submodulized missing)" From 32f46d6ca01ee2d486d09c70e43316a61e04489c Mon Sep 17 00:00:00 2001 From: Steve Mattsen Date: Tue, 14 Apr 2026 16:41:01 -0400 Subject: [PATCH 11/18] Update `submodulize.sh` and `unsubmodulize.sh` to improve handling of submodule commit states. Modify the `--no-commit` option in `submodulize.sh` to ensure necessary commits for unsub replay functionality. Adjust tree reference parsing to avoid issues with CRLF mismatches in `unsubmodulize.sh`. Enhance test cases in `test-bootstrap.sh` to validate submodule structure and ensure consistent behavior across platforms. --- cleandev/submodulize.sh | 17 ++++++++++++++--- cleandev/tests/test-bootstrap.sh | 7 +++++-- cleandev/unsubmodulize.sh | 13 +++++++++++-- 3 files changed, 30 insertions(+), 7 deletions(-) diff --git a/cleandev/submodulize.sh b/cleandev/submodulize.sh index b94c042..129f958 100644 --- a/cleandev/submodulize.sh +++ b/cleandev/submodulize.sh @@ -72,7 +72,7 @@ Examples: Options: --dry-run Print actions without changing the repo - --no-commit Stage submodule changes but do not commit + --no-commit Stage submodule changes but do not commit (bootstrap still commits the submodule layout so unsub replay can run) --ssh Use git@github.com URLs for github.com HTTPS entries --manifest PATH Plugin manifest (default: ROOT/plugin-submodules.manifest) --repo ROOT Moodle git root (explicit form of a bare ROOT; overrides an earlier bare ROOT; a bare path after --repo is an error) @@ -448,7 +448,18 @@ submodulize_bootstrap_pipeline() { git checkout -B submodulized + # Unsub replay reads commit trees on --source; submodule adds must be committed so + # submodulized records 160000 gitlinks, not only the pre-submodule vendored tree. + local _saved_no_commit=false + if $NO_COMMIT; then + _saved_no_commit=true + echo "submodulize: bootstrap commits submodule layout on submodulized (required for unsub replay); ignoring --no-commit for this step." >&2 + NO_COMMIT=false + fi submodulize_one_shot_apply_manifest + if $_saved_no_commit; then + NO_COMMIT=true + fi if $DRY_RUN; then exit 0 @@ -538,7 +549,7 @@ submodulize_replay_mode() { if [[ "$mode2" == "160000" && -n "$to_sha" ]]; then : elif [[ "$mode2" == "040000" ]]; then - tr_end="$(git rev-parse "$SOURCE_TIP:$P^{tree}" 2>/dev/null || true)" + tr_end="$(git rev-parse "$SOURCE_TIP:$P" 2>/dev/null || true)" if [[ -z "$tr_end" ]]; then echo "No tree at $SOURCE_BRANCH:$P (source tip)" >&2 exit 1 @@ -556,7 +567,7 @@ submodulize_replay_mode() { if [[ "$mode" == "160000" && -n "$from_sha" ]]; then : elif [[ "$mode" == "040000" ]]; then - tr_sha="$(git rev-parse "$FORK_POINT:$P^{tree}" 2>/dev/null || true)" + tr_sha="$(git rev-parse "$FORK_POINT:$P" 2>/dev/null || true)" if [[ -z "$tr_sha" ]]; then echo "No tree at $FORK_POINT:$P" >&2 exit 1 diff --git a/cleandev/tests/test-bootstrap.sh b/cleandev/tests/test-bootstrap.sh index 90aa7a0..49eee5e 100644 --- a/cleandev/tests/test-bootstrap.sh +++ b/cleandev/tests/test-bootstrap.sh @@ -16,13 +16,16 @@ PLUGIN="$TMP/plugin-upstream" MOODLE="$TMP/moodle" mkdir -p "$PLUGIN" git -C "$PLUGIN" init -b main +git -C "$PLUGIN" config core.autocrlf false echo "plug=1" > "$PLUGIN/version.txt" git -C "$PLUGIN" add version.txt git -C "$PLUGIN" commit -q -m "init plugin" mkdir -p "$MOODLE/mod/testplugin" git -C "$MOODLE" init -b master -echo "vendored-only" > "$MOODLE/mod/testplugin/local.txt" +git -C "$MOODLE" config core.autocrlf false +# Unpack from plugin so tree/blob IDs match plugin commits (avoids CRLF tree mismatch on Windows). +git -C "$PLUGIN" archive HEAD | tar -x -C "$MOODLE/mod/testplugin" git -C "$MOODLE" add mod/testplugin git -C "$MOODLE" commit -q -m "vendored plugin" @@ -33,7 +36,7 @@ printf '%s\n' 'mod/testplugin|../plugin-upstream|main' > "$MOODLE/plugin-submodu git -C "$MOODLE" show-ref --verify --quiet refs/heads/submodulized || fail "expected submodulized branch" git -C "$MOODLE" show-ref --verify --quiet refs/heads/unsubmodulized || fail "expected unsubmodulized branch" -git -C "$MOODLE" ls-tree -r submodulized --name-only | grep -q '^mod/testplugin/version.txt$' || fail "expected submodule file on submodulized" +git -C "$MOODLE" ls-tree submodulized mod/testplugin | grep -q '^160000' || fail "expected gitlink (submodule) at mod/testplugin on submodulized" git -C "$MOODLE" ls-tree -r unsubmodulized --name-only | grep -q '^mod/testplugin/version.txt$' || fail "expected vendored file on unsubmodulized" ok "submodulize bootstrap (auto when submodulized missing)" diff --git a/cleandev/unsubmodulize.sh b/cleandev/unsubmodulize.sh index 5a895d3..c91f06e 100644 --- a/cleandev/unsubmodulize.sh +++ b/cleandev/unsubmodulize.sh @@ -375,7 +375,7 @@ unsubmodulize_replay_mode() { if [[ "$mode2" == "160000" && -n "$to_sha" ]]; then : elif [[ "$mode2" == "040000" ]]; then - tr_end="$(git rev-parse "$SOURCE_TIP:$P^{tree}" 2>/dev/null || true)" + tr_end="$(git rev-parse "$SOURCE_TIP:$P" 2>/dev/null || true)" if [[ -z "$tr_end" ]]; then echo "No tree at $SOURCE_BRANCH:$P (source tip)" >&2 exit 1 @@ -394,7 +394,7 @@ unsubmodulize_replay_mode() { elif [[ "$mode" == "160000" && -n "$from_sha" ]]; then : elif [[ "$mode" == "040000" ]]; then - tr_sha="$(git rev-parse "$FORK_POINT:$P^{tree}" 2>/dev/null || true)" + tr_sha="$(git rev-parse "$FORK_POINT:$P" 2>/dev/null || true)" if [[ -z "$tr_sha" ]]; then echo "No tree at $FORK_POINT:$P" >&2 exit 1 @@ -432,6 +432,15 @@ unsubmodulize_replay_mode() { if [[ ${#EVENT_LINES[@]} -eq 0 ]]; then echo "No plugin commits to replay." >&2 + if $DRY_RUN; then + echo "Planned: branch $TARGET_BRANCH at $FORK_POINT (fork already matches submodule plugin SHAs)" >&2 + exit 0 + fi + if $FORCE_REPLAY && git show-ref --verify --quiet "refs/heads/$TARGET_BRANCH"; then + git branch -D "$TARGET_BRANCH" 2>/dev/null || true + fi + git branch "$TARGET_BRANCH" "$FORK_POINT" + echo "Done. Branch $TARGET_BRANCH -> $(git rev-parse "$TARGET_BRANCH")" exit 0 fi From 4d1096102f61282b7b24f30af76105fc63faa10a Mon Sep 17 00:00:00 2001 From: Steve Mattsen Date: Wed, 15 Apr 2026 12:12:43 -0400 Subject: [PATCH 12/18] Enhance `submodulize.sh` to improve handling of the `plugin-submodules.manifest` file. Update the `--manifest` option description for clarity and add functions to stage the manifest on the submodule branch. Introduce hints for switching branches with vendored plugin files, ensuring better user guidance. Update tests in `test-bootstrap.sh` to validate the presence and absence of the manifest in the appropriate branches. --- cleandev/submodulize.sh | 43 ++++++++++++++++++++++++++++++-- cleandev/tests/test-bootstrap.sh | 4 +++ 2 files changed, 45 insertions(+), 2 deletions(-) diff --git a/cleandev/submodulize.sh b/cleandev/submodulize.sh index 129f958..c2e093f 100644 --- a/cleandev/submodulize.sh +++ b/cleandev/submodulize.sh @@ -74,7 +74,7 @@ Options: --dry-run Print actions without changing the repo --no-commit Stage submodule changes but do not commit (bootstrap still commits the submodule layout so unsub replay can run) --ssh Use git@github.com URLs for github.com HTTPS entries - --manifest PATH Plugin manifest (default: ROOT/plugin-submodules.manifest) + --manifest PATH Plugin manifest (default: ROOT/plugin-submodules.manifest). If that file sits at the repo root, one-shot/bootstrap also stages it on the submodule branch so master/unsubmodulized can keep it untracked. --repo ROOT Moodle git root (explicit form of a bare ROOT; overrides an earlier bare ROOT; a bare path after --repo is an error) Bootstrap (from vendored tree + manifest → submodulized + unsubmodulized): @@ -335,6 +335,36 @@ rewrite_github_url_to_ssh() { fi } +# Parent index only has gitlinks; files inside each plugin dir are the submodule checkout (not parent blobs). +# checkout to a vendored branch would replace those dirs unless submodule working trees are cleared first. +submodulize_hint_switch_from_submodule_branch() { + cat <<'EOF' >&2 +To switch to a branch with vendored plugin files (not gitlinks), clear submodule checkouts first, then checkout: + git submodule deinit -f --all + git checkout master # or main / unsubmodulized +If plugin-submodules.manifest was only committed on submodulized, copy it back in after checkout (keep it untracked on vendored branches). +EOF +} + +# Commit root plugin-submodules.manifest on the submodule branch only (vendored branches can keep a local untracked copy). +submodulize_stage_root_manifest() { + [[ -f "$MANIFEST" ]] || return 0 + [[ "$(basename -- "$MANIFEST")" == "plugin-submodules.manifest" ]] || return 0 + local rtop mtop + rtop="$(cd "$REPO_ROOT" && pwd -P 2>/dev/null)" || return 0 + mtop="$(cd "$(dirname -- "$MANIFEST")" && pwd -P 2>/dev/null)" || return 0 + [[ "$mtop" == "$rtop" ]] || return 0 + if $DRY_RUN; then + printf '[dry-run] git add %q\n' "$MANIFEST" + return 0 + fi + if git check-ignore -q -- "$MANIFEST" 2>/dev/null; then + git add -f -- "$MANIFEST" || true + else + git add -- "$MANIFEST" || true + fi +} + run() { if $DRY_RUN; then printf '[dry-run] %q\n' "$@" @@ -420,10 +450,13 @@ submodulize_one_shot_apply_manifest() { if $DRY_RUN; then echo "Dry run complete." + submodulize_stage_root_manifest $BOOTSTRAP && echo "Bootstrap: would commit on submodulized, then run unsubmodulize replay (omit --dry-run)." >&2 exit 0 fi + submodulize_stage_root_manifest + if ! $NO_COMMIT; then if git diff --cached --quiet 2>/dev/null; then echo "Nothing staged; skipping commit." @@ -432,7 +465,12 @@ submodulize_one_shot_apply_manifest() { fi fi - $BOOTSTRAP || echo "Done. Submodule layout is ready (clean repo)." + if $BOOTSTRAP; then + : + else + echo "Done. Submodule layout is ready (clean repo)." + submodulize_hint_switch_from_submodule_branch + fi } submodulize_bootstrap_pipeline() { @@ -470,6 +508,7 @@ submodulize_bootstrap_pipeline() { bash "$SCRIPT_DIR/unsubmodulize.sh" "${unsub_args[@]}" echo "Bootstrap complete: submodulized (submodules) and unsubmodulized (vendored replay) are ready." >&2 + submodulize_hint_switch_from_submodule_branch } submodulize_replay_mode() { diff --git a/cleandev/tests/test-bootstrap.sh b/cleandev/tests/test-bootstrap.sh index 49eee5e..aea5e90 100644 --- a/cleandev/tests/test-bootstrap.sh +++ b/cleandev/tests/test-bootstrap.sh @@ -37,6 +37,10 @@ printf '%s\n' 'mod/testplugin|../plugin-upstream|main' > "$MOODLE/plugin-submodu git -C "$MOODLE" show-ref --verify --quiet refs/heads/submodulized || fail "expected submodulized branch" git -C "$MOODLE" show-ref --verify --quiet refs/heads/unsubmodulized || fail "expected unsubmodulized branch" git -C "$MOODLE" ls-tree submodulized mod/testplugin | grep -q '^160000' || fail "expected gitlink (submodule) at mod/testplugin on submodulized" +git -C "$MOODLE" ls-tree -r submodulized --name-only | grep -q '^plugin-submodules.manifest$' || fail "expected tracked plugin-submodules.manifest on submodulized" git -C "$MOODLE" ls-tree -r unsubmodulized --name-only | grep -q '^mod/testplugin/version.txt$' || fail "expected vendored file on unsubmodulized" +if git -C "$MOODLE" ls-tree -r unsubmodulized --name-only | grep -q '^plugin-submodules.manifest$'; then + fail "did not expect plugin-submodules.manifest on unsubmodulized" +fi ok "submodulize bootstrap (auto when submodulized missing)" From 36480835b1b9758a0a2d33226ce31011dc564c81 Mon Sep 17 00:00:00 2001 From: Steve Mattsen Date: Wed, 15 Apr 2026 13:07:28 -0400 Subject: [PATCH 13/18] Refactor `unsubmodulize.sh` to enhance replay mode handling. Update descriptions for `--source` and `--force` options for clarity, ensuring users understand the implications of incremental updates. Adjust default behavior to prioritize the `submodulized` branch when omitted, and improve error messages for better user guidance during replay operations. --- cleandev/tests/test-unsub-incremental.sh | 86 ++++++++++++++++++++++++ cleandev/unsubmodulize.sh | 82 +++++++++++++++------- 2 files changed, 144 insertions(+), 24 deletions(-) create mode 100644 cleandev/tests/test-unsub-incremental.sh diff --git a/cleandev/tests/test-unsub-incremental.sh b/cleandev/tests/test-unsub-incremental.sh new file mode 100644 index 0000000..ad1718e --- /dev/null +++ b/cleandev/tests/test-unsub-incremental.sh @@ -0,0 +1,86 @@ +#!/usr/bin/env bash +# Incremental unsub replay: default --source submodulized; no --force when unsubmodulized exists. +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=lib.sh +source "$SCRIPT_DIR/lib.sh" +require_cleandev + +TMP=$(mktemp -d) +export HOME="$TMP/_home" +mkdir -p "$HOME" +git config --global protocol.file.allow always +trap 'rm -rf "$TMP"' EXIT + +PLUGIN="$TMP/plugin" +MOODLE="$TMP/moodle" +mkdir -p "$PLUGIN" "$MOODLE" + +git -C "$PLUGIN" init -b main +git -C "$PLUGIN" config core.autocrlf false +echo "v1" > "$PLUGIN/f.txt" +git -C "$PLUGIN" add f.txt +git -C "$PLUGIN" commit -q -m "p1" +echo "v2" >> "$PLUGIN/f.txt" +git -C "$PLUGIN" commit -q -am "p2" +echo "v3" >> "$PLUGIN/f.txt" +git -C "$PLUGIN" commit -q -am "p3" + +C0="$(git -C "$PLUGIN" rev-parse HEAD~2)" +C1="$(git -C "$PLUGIN" rev-parse HEAD~1)" +C2="$(git -C "$PLUGIN" rev-parse HEAD)" + +git -C "$MOODLE" init -b main +printf 'core\n' > "$MOODLE/README.md" +mkdir -p "$MOODLE/mod/foo" +git -C "$PLUGIN" archive "$C0" | tar -x -C "$MOODLE/mod/foo" +git -C "$MOODLE" add README.md mod/foo +git -C "$MOODLE" commit -q -m "vendored base" + +BASE="$(git -C "$MOODLE" rev-parse HEAD)" + +git -C "$MOODLE" config -f .gitmodules submodule.mod/foo.path mod/foo +git -C "$MOODLE" config -f .gitmodules submodule.mod/foo.url "$PLUGIN" +git -C "$MOODLE" add .gitmodules +git -C "$MOODLE" rm -rf --cached mod/foo +git -C "$MOODLE" update-index --add --cacheinfo "160000,$C0,mod/foo" +git -C "$MOODLE" commit -q -m "sub at c0" + +git -C "$MOODLE" checkout -q -b submodulized +git -C "$MOODLE" update-index --add --cacheinfo "160000,$C1,mod/foo" +git -C "$MOODLE" commit -q -m "sub at c1" + +MANIFEST="$MOODLE/plugin-submodules.manifest" +printf 'mod/foo|%s|main\n' "$PLUGIN" > "$MANIFEST" + +# Initial build (force ok); submodulized tip is still C1 so replay is c0..c1 only. +"$CLEANDEV/unsubmodulize.sh" --repo "$MOODLE" --fork-point "$BASE" --source submodulized \ + --target unsubmodulized --force --plugin-base "mod/foo=$C0" + +N1="$(git -C "$MOODLE" rev-list --count "${BASE}..unsubmodulized" 2>/dev/null || echo 0)" +[[ "$N1" -eq 1 ]] || fail "expected 1 replay commit past BASE after first unsub, got $N1" +git -C "$MOODLE" show "unsubmodulized:mod/foo/f.txt" | grep -q v2 || fail "expected v2 after first unsub" +if git -C "$MOODLE" show "unsubmodulized:mod/foo/f.txt" | grep -q v3; then + fail "did not expect v3 after first unsub (only c0..c1 replayed)" +fi + +# Advance submodulized to C2, then incremental unsub (no --force). +git -C "$MOODLE" checkout -q submodulized +git -C "$MOODLE" update-index --add --cacheinfo "160000,$C2,mod/foo" +git -C "$MOODLE" commit -q -m "sub at c2" + +"$CLEANDEV/unsubmodulize.sh" --repo "$MOODLE" --fork-point "$BASE" --source submodulized \ + --target unsubmodulized + +N2="$(git -C "$MOODLE" rev-list --count "${BASE}..unsubmodulized" 2>/dev/null || echo 0)" +[[ "$N2" -eq 2 ]] || fail "expected 2 replay commits past BASE after incremental unsub, got $N2" +git -C "$MOODLE" show "unsubmodulized:mod/foo/f.txt" | grep -q v3 || fail "expected v3 after incremental unsub" + +# Idempotent +"$CLEANDEV/unsubmodulize.sh" --repo "$MOODLE" --fork-point "$BASE" --source submodulized \ + --target unsubmodulized + +N3="$(git -C "$MOODLE" rev-list --count "${BASE}..unsubmodulized" 2>/dev/null || echo 0)" +[[ "$N3" -eq 2 ]] || fail "expected still 2 commits past BASE when already up to date, got $N3" + +ok "unsubmodulize incremental replay without --force" diff --git a/cleandev/unsubmodulize.sh b/cleandev/unsubmodulize.sh index c91f06e..eaea80c 100644 --- a/cleandev/unsubmodulize.sh +++ b/cleandev/unsubmodulize.sh @@ -76,10 +76,10 @@ Replay (default — one superproject commit per plugin-repo commit on --target): --no-replay One-shot mode: convert submodules per manifest (no branch replay) --replay Replay mode (default; explicit if you toggled --no-replay earlier on the command line) --fork-point BASE Superproject where replay starts (default: local master, else main, when omitted — see docs) - --source BR End state per path: gitlinks and/or vendored trees (default: unsubmodulized, else master, else main, else submodulized) + --source BR Submodule/vendored tip to replay from (default: submodulized, else unsubmodulized, else master, else main) --target BR Branch to create/update (default: unsubmodulized) --order NAME Only chronological (committer date, then path, then SHA) - --force Overwrite --target branch if it exists (replay only) + --force Replay only: delete and rebuild --target from --fork-point (omit for incremental update when --target exists) --plugin-base P=S Optional start SHA for manifest path P (e.g. when BASE has no gitlink) Requires: git, and clean submodule working trees for paths being converted (no uncommitted changes inside submodules). @@ -226,23 +226,23 @@ if $REPLAY && [[ -z "$FORK_POINT" ]]; then echo "Default --fork-point $FORK_POINT (HEAD is ${base_ref} or $TARGET_BRANCH)." >&2 else FORK_POINT="$base_ref" - echo "Default --fork-point $FORK_POINT (recreating $TARGET_BRANCH from ${base_ref}; use --force if the branch already exists)." >&2 + echo "Default --fork-point $FORK_POINT ($TARGET_BRANCH exists: incremental replay without --force, or --force to rebuild from ${base_ref})." >&2 fi fi fi -# When --source is omitted in replay mode: prefer unsubmodulized (vendored end state), else master/main/submodulized. +# When --source is omitted in replay mode: prefer submodulized (gitlink tip) for replaying into unsubmodulized. if $REPLAY && ! $REPLAY_SOURCE_EXPLICIT; then - if git show-ref --verify --quiet "refs/heads/unsubmodulized"; then + if git show-ref --verify --quiet refs/heads/submodulized; then + SOURCE_BRANCH=submodulized + elif git show-ref --verify --quiet "refs/heads/unsubmodulized"; then SOURCE_BRANCH=unsubmodulized elif git show-ref --verify --quiet refs/heads/master; then SOURCE_BRANCH=master elif git show-ref --verify --quiet refs/heads/main; then SOURCE_BRANCH=main - elif git show-ref --verify --quiet refs/heads/submodulized; then - SOURCE_BRANCH=submodulized else - echo "Replay: pass --source BR (no local unsubmodulized, master, main, or submodulized branch found)." >&2 + echo "Replay: pass --source BR (no local submodulized, unsubmodulized, master, or main branch found)." >&2 exit 1 fi echo "Default --source $SOURCE_BRANCH" >&2 @@ -289,11 +289,6 @@ unsubmodulize_replay_mode() { git rev-parse --verify "$FORK_POINT^{commit}" >/dev/null git rev-parse --verify "$SOURCE_BRANCH^{commit}" >/dev/null - if git show-ref --verify --quiet "refs/heads/$TARGET_BRANCH" && ! $FORCE_REPLAY; then - echo "Branch $TARGET_BRANCH exists; pass --force to overwrite" >&2 - exit 1 - fi - ls_path_mode_sha() { local commit="$1" path="$2" git ls-tree "$commit" -- "$path" 2>/dev/null | awk '{print $1 "\t" $3}' | head -n1 @@ -346,6 +341,24 @@ unsubmodulize_replay_mode() { SOURCE_TIP="$(git rev-parse "$SOURCE_BRANCH^{commit}")" + INCREMENTAL=false + FROM_REF="$FORK_POINT" + if git show-ref --verify --quiet "refs/heads/$TARGET_BRANCH" && ! $FORCE_REPLAY; then + INCREMENTAL=true + FROM_REF="$(git rev-parse "$TARGET_BRANCH^{commit}")" + for _xi in "${!M_PATHS[@]}"; do + _xp="${M_PATHS[_xi]}" + _ms="$(ls_path_mode_sha "$FROM_REF" "$_xp")" + _mode="${_ms%%$'\t'*}" + if [[ "$_mode" != "040000" ]]; then + echo "unsubmodulize: $TARGET_BRANCH at $_xp is not a vendored directory (tree mode $_mode)." >&2 + echo " Use --force to delete $TARGET_BRANCH and rebuild from --fork-point $FORK_POINT." >&2 + exit 1 + fi + done + echo "unsubmodulize: incremental update of $TARGET_BRANCH from $(git rev-parse --short "$FROM_REF") toward $SOURCE_BRANCH." >&2 + fi + TMPD="$(mktemp -d "${TMPDIR:-/tmp}/unsub-replay.XXXXXX")" trap 'rm -rf "$TMPD"' EXIT @@ -356,7 +369,7 @@ unsubmodulize_replay_mode() { URL="${M_URLS[$i]}" URL="$(rewrite_github_url_to_ssh "$URL")" - ms="$(ls_path_mode_sha "$FORK_POINT" "$P")" + ms="$(ls_path_mode_sha "$FROM_REF" "$P")" mode="${ms%%$'\t'*}" from_sha="${ms#*$'\t'}" @@ -394,17 +407,17 @@ unsubmodulize_replay_mode() { elif [[ "$mode" == "160000" && -n "$from_sha" ]]; then : elif [[ "$mode" == "040000" ]]; then - tr_sha="$(git rev-parse "$FORK_POINT:$P" 2>/dev/null || true)" + tr_sha="$(git rev-parse "$FROM_REF:$P" 2>/dev/null || true)" if [[ -z "$tr_sha" ]]; then - echo "No tree at $FORK_POINT:$P" >&2 + echo "No tree at $FROM_REF:$P" >&2 exit 1 fi from_sha="$(find_plugin_commit_for_tree "$PDIR" "$tr_sha")" || { - echo "Could not find plugin commit matching tree at $FORK_POINT:$P" >&2 + echo "Could not find plugin commit matching tree at $FROM_REF:$P" >&2 exit 1 } else - echo "Path $P at fork-point must be gitlink or directory; use --plugin-base ${P}=SHA" >&2 + echo "Path $P at replay base $FROM_REF must be gitlink or vendored directory; use --plugin-base ${P}=SHA" >&2 exit 1 fi @@ -433,7 +446,15 @@ unsubmodulize_replay_mode() { if [[ ${#EVENT_LINES[@]} -eq 0 ]]; then echo "No plugin commits to replay." >&2 if $DRY_RUN; then - echo "Planned: branch $TARGET_BRANCH at $FORK_POINT (fork already matches submodule plugin SHAs)" >&2 + if $INCREMENTAL; then + echo "Planned: no new commits ($TARGET_BRANCH already matches $SOURCE_BRANCH submodule tips)." >&2 + else + echo "Planned: branch $TARGET_BRANCH at $FORK_POINT (fork already matches submodule plugin SHAs)" >&2 + fi + exit 0 + fi + if $INCREMENTAL; then + echo "Branch $TARGET_BRANCH is already up to date with $SOURCE_BRANCH." >&2 exit 0 fi if $FORCE_REPLAY && git show-ref --verify --quiet "refs/heads/$TARGET_BRANCH"; then @@ -449,7 +470,11 @@ unsubmodulize_replay_mode() { unset IFS if $DRY_RUN; then - echo "Planned ${#EVENT_LINES[@]} commits on $TARGET_BRANCH (from $FORK_POINT)" + if $INCREMENTAL; then + echo "Planned ${#EVENT_LINES[@]} incremental commit(s) on $TARGET_BRANCH ($(git rev-parse --short "$FROM_REF") → $SOURCE_BRANCH)" >&2 + else + echo "Planned ${#EVENT_LINES[@]} commits on $TARGET_BRANCH (from $FORK_POINT)" >&2 + fi while IFS=$'\t' read -r ct p sha pdir; do [[ -z "${ct:-}" ]] && continue echo " $ct $p $sha ($(git -C "$pdir" show -s --format=%s "$sha"))" @@ -464,7 +489,12 @@ unsubmodulize_replay_mode() { WT="$TMPD/wt" rm -rf "$WT" - FORK_R="$(git rev-parse "$FORK_POINT^{commit}")" + REPLAY_ROOT="$FORK_POINT" + REPLAY_ROOT_REV="$(git rev-parse "$FORK_POINT^{commit}")" + if $INCREMENTAL; then + REPLAY_ROOT="$FROM_REF" + REPLAY_ROOT_REV="$(git rev-parse "$FROM_REF^{commit}")" + fi materialize_commit_at() { local commit="$1" dest="$2" @@ -489,8 +519,8 @@ unsubmodulize_replay_mode() { fill_worktree_from_parent() { local parent="$1" dest="$2" - if [[ "$(git rev-parse "$parent^{commit}")" == "$FORK_R" ]]; then - materialize_commit_at "$FORK_POINT" "$dest" + if [[ "$(git rev-parse "$parent^{commit}")" == "$REPLAY_ROOT_REV" ]]; then + materialize_commit_at "$REPLAY_ROOT" "$dest" else rm -rf "${dest:?}/"* mkdir -p "$dest" @@ -498,7 +528,11 @@ unsubmodulize_replay_mode() { fi } - GIT_TERMINAL_PROMPT=0 git "${git_github_pat_c[@]}" worktree add -B "$TARGET_BRANCH" "$WT" "$FORK_POINT" --force + if $INCREMENTAL; then + GIT_TERMINAL_PROMPT=0 git "${git_github_pat_c[@]}" worktree add -f "$WT" "$TARGET_BRANCH" + else + GIT_TERMINAL_PROMPT=0 git "${git_github_pat_c[@]}" worktree add -B "$TARGET_BRANCH" "$WT" "$FORK_POINT" --force + fi while IFS=$'\t' read -r ct P csha pdir; do [[ -z "${ct:-}" ]] && continue From 7be38a34d15a0781d1e4bfffe8a070d06c8aabb7 Mon Sep 17 00:00:00 2001 From: Steve Mattsen Date: Wed, 15 Apr 2026 15:24:35 -0400 Subject: [PATCH 14/18] Enhance `submodulize.sh` to improve handling of existing `--target` branches during replay operations. Update the `--force` option description for clarity, emphasizing its role in rebuilding branches from a fork point. Introduce logic to apply manifest updates without requiring a full replay when the target branch exists, improving user experience and guidance. --- cleandev/submodulize.sh | 16 ++++- .../tests/test-submodulize-extend-manifest.sh | 62 +++++++++++++++++++ 2 files changed, 77 insertions(+), 1 deletion(-) create mode 100644 cleandev/tests/test-submodulize-extend-manifest.sh diff --git a/cleandev/submodulize.sh b/cleandev/submodulize.sh index c2e093f..3fd20ff 100644 --- a/cleandev/submodulize.sh +++ b/cleandev/submodulize.sh @@ -88,7 +88,7 @@ Replay (default — one superproject commit per plugin-repo commit on --target): --source BR End state per path (gitlinks and/or vendored trees; default: submodulized, else master, else main) --target BR Branch to create/update (default: submodulized) --order NAME Only chronological - --force Overwrite --target branch if it exists (replay only) + --force Replay: delete and rebuild --target from --fork-point. Omit when --target exists to only apply manifest changes (new paths → submodule add). --plugin-base P=S Optional start SHA for manifest path P Requires: git, and a clean enough working tree (commit or stash if plugin paths are dirty). @@ -718,6 +718,20 @@ if $BOOTSTRAP; then exit 0 fi +# Default "submodulize ./" on an existing --target branch: extend layout from manifest (new submodule paths) +# instead of requiring --force. Full replay rebuild still needs explicit --fork-point/--source/--target intent + --force. +if $REPLAY && ! $FORCE_REPLAY && git show-ref --verify --quiet "refs/heads/$TARGET_BRANCH"; then + if ! $FORK_POINT_EXPLICIT && ! $REPLAY_SOURCE_EXPLICIT && ! $REPLAY_TARGET_EXPLICIT && ((${#PLUGIN_BASE_OVERRIDES[@]} == 0)); then + echo "submodulize: branch $TARGET_BRANCH exists — applying manifest updates (same as --no-replay). Use --force to replay-recreate $TARGET_BRANCH from --fork-point." >&2 + REPLAY=false + active_br="$(git symbolic-ref -q --short HEAD 2>/dev/null || true)" + if [[ "$active_br" != "$TARGET_BRANCH" ]]; then + echo "submodulize: checking out $TARGET_BRANCH" >&2 + git checkout "$TARGET_BRANCH" + fi + fi +fi + if $REPLAY; then submodulize_replay_mode exit 0 diff --git a/cleandev/tests/test-submodulize-extend-manifest.sh b/cleandev/tests/test-submodulize-extend-manifest.sh new file mode 100644 index 0000000..7756b58 --- /dev/null +++ b/cleandev/tests/test-submodulize-extend-manifest.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +# With submodulized present, default submodulize (replay) applies manifest without --force; new paths become submodules. +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=lib.sh +source "$SCRIPT_DIR/lib.sh" +require_cleandev + +TMP=$(mktemp -d) +export HOME="$TMP/_home" +mkdir -p "$HOME" +git config --global protocol.file.allow always +trap 'rm -rf "$TMP"' EXIT + +PLUGIN1="$TMP/plugin1" +PLUGIN2="$TMP/plugin2" +MOODLE="$TMP/moodle" + +mkdir -p "$PLUGIN1" +git -C "$PLUGIN1" init -b main +git -C "$PLUGIN1" config core.autocrlf false +echo "one" > "$PLUGIN1/a.txt" +git -C "$PLUGIN1" add a.txt +git -C "$PLUGIN1" commit -q -m "p1" + +mkdir -p "$PLUGIN2" +git -C "$PLUGIN2" init -b main +git -C "$PLUGIN2" config core.autocrlf false +echo "two" > "$PLUGIN2/b.txt" +git -C "$PLUGIN2" add b.txt +git -C "$PLUGIN2" commit -q -m "p2" + +mkdir -p "$MOODLE/mod/p1" +git -C "$MOODLE" init -b master +git -C "$MOODLE" config core.autocrlf false +git -C "$PLUGIN1" archive HEAD | tar -x -C "$MOODLE/mod/p1" +git -C "$MOODLE" add mod/p1 +git -C "$MOODLE" commit -q -m "vendored p1" + +printf '%s\n' "mod/p1|../plugin1|main" > "$MOODLE/plugin-submodules.manifest" + +"$CLEANDEV/submodulize.sh" --repo "$MOODLE" + +# Second run: no new manifest lines — should not require --force (one-shot no-op / skips). +"$CLEANDEV/submodulize.sh" --repo "$MOODLE" + +git -C "$MOODLE" checkout -q submodulized +mkdir -p "$MOODLE/mod/p2" +git -C "$PLUGIN2" archive HEAD | tar -x -C "$MOODLE/mod/p2" +git -C "$MOODLE" add mod/p2 +git -C "$MOODLE" commit -q -m "vendored p2 for submod" + +{ + echo "mod/p1|../plugin1|main" + echo "mod/p2|../plugin2|main" +} > "$MOODLE/plugin-submodules.manifest" + +"$CLEANDEV/submodulize.sh" --repo "$MOODLE" + +git -C "$MOODLE" ls-tree submodulized mod/p2 | grep -q '^160000' || fail "expected gitlink at mod/p2 after manifest extend" + +ok "submodulize extends submodulized from manifest without --force" From 0e8606c00f48e00ada339ae8f97b689a0ae0f620 Mon Sep 17 00:00:00 2001 From: Steve Mattsen Date: Wed, 15 Apr 2026 15:46:20 -0400 Subject: [PATCH 15/18] Enhance `submodulize.sh` with new options `--sync-unsub` and `--no-sync-from-unsub` to improve control over syncing between unsubmodulized and submodulized branches. Update error handling to prevent conflicting option combinations and clarify usage instructions. Introduce logic for automatic syncing when conditions are met, enhancing user experience during submodule management. --- cleandev/submodulize.sh | 297 +++++++++++++++++++++++++++++- cleandev/tests/test-sync-unsub.sh | 119 ++++++++++++ 2 files changed, 414 insertions(+), 2 deletions(-) create mode 100644 cleandev/tests/test-sync-unsub.sh diff --git a/cleandev/submodulize.sh b/cleandev/submodulize.sh index 3fd20ff..31c6f66 100644 --- a/cleandev/submodulize.sh +++ b/cleandev/submodulize.sh @@ -3,7 +3,8 @@ # Intended for the submodule-layout Moodle checkout: keep plugin-submodules.manifest at the superproject # root (same repo as .gitmodules will live in). Run from inside that clone, or pass ROOT / --repo. # Default mode is replay: builds branch submodulized with one superproject commit per plugin-repo commit. -# Use --no-replay for one-shot conversion (manifest loop only). +# Use --no-replay for one-shot conversion (manifest loop only). When submodulized and unsubmodulized both +# exist and unsub is ahead of merge-base(unsub, sub), the one-shot path also replays those commits onto gitlinks. # # Requires: git, a clean enough working tree (commit or stash first if paths are dirty). # @@ -46,6 +47,8 @@ REPLAY_ORDER_EXPLICIT=false FORK_POINT_EXPLICIT=false BOOTSTRAP=false BOOTSTRAP_EXPLICIT=false +SYNC_UNSUB=false +NO_SYNC_FROM_UNSUB=false declare -a PLUGIN_BASE_OVERRIDES=() usage() { @@ -69,6 +72,8 @@ Examples: submodulize.sh ./ # same as --bootstrap when submodulized branch does not exist yet submodulize.sh --dry-run --no-replay ~/workspace/moodle submodulize.sh --fork-point abc123 --source submodulized + submodulize.sh ./ # when unsubmodulized is ahead of merge-base(sub,unsub), syncs onto submodulized automatically + submodulize.sh --no-sync-from-unsub ./ # manifest one-shot only; skip unsub → submodulized replay Options: --dry-run Print actions without changing the repo @@ -90,6 +95,8 @@ Replay (default — one superproject commit per plugin-repo commit on --target): --order NAME Only chronological --force Replay: delete and rebuild --target from --fork-point. Omit when --target exists to only apply manifest changes (new paths → submodule add). --plugin-base P=S Optional start SHA for manifest path P + --sync-unsub Force the unsub→submodulized sync step (even if merge-base already equals unsub tip). Normally automatic when both branches exist and unsub is ahead. + --no-sync-from-unsub After manifest one-shot: do not replay unsubmodulized onto submodulized (overrides automatic sync). Requires: git, and a clean enough working tree (commit or stash if plugin paths are dirty). @@ -140,6 +147,15 @@ while [[ $# -gt 0 ]]; do PLUGIN_BASE_OVERRIDES+=("${2:?}") shift 2 ;; + --sync-unsub) + SYNC_UNSUB=true + REPLAY=false + shift + ;; + --no-sync-from-unsub) + NO_SYNC_FROM_UNSUB=true + shift + ;; --manifest) MANIFEST="${2:?}" MANIFEST_EXPLICIT=true @@ -180,7 +196,57 @@ if ! $MANIFEST_EXPLICIT; then fi if [[ ! -f "$MANIFEST" ]]; then - echo "Manifest not found: $MANIFEST" >&2 + if $MANIFEST_EXPLICIT; then + echo "Manifest not found: $MANIFEST" >&2 + exit 1 + elif $SYNC_UNSUB; then + : + elif git -C "$REPO_ROOT" show-ref --verify --quiet refs/heads/submodulized 2>/dev/null \ + && git -C "$REPO_ROOT" show-ref --verify --quiet refs/heads/unsubmodulized 2>/dev/null; then + : + else + echo "Manifest not found: $MANIFEST" >&2 + exit 1 + fi +fi + +if $SYNC_UNSUB; then + $BOOTSTRAP_EXPLICIT && { + echo "Do not combine --sync-unsub with --bootstrap." >&2 + exit 1 + } + $FORK_POINT_EXPLICIT && { + echo "Do not combine --sync-unsub with --fork-point." >&2 + exit 1 + } + $REPLAY_SOURCE_EXPLICIT && { + echo "Do not combine --sync-unsub with --source." >&2 + exit 1 + } + $REPLAY_TARGET_EXPLICIT && { + echo "Do not combine --sync-unsub with --target." >&2 + exit 1 + } + $REPLAY_ORDER_EXPLICIT && { + echo "Do not combine --sync-unsub with --order." >&2 + exit 1 + } + ((${#PLUGIN_BASE_OVERRIDES[@]} == 0)) || { + echo "--plugin-base is not used with --sync-unsub." >&2 + exit 1 + } + $FORCE_REPLAY && { + echo "--force is not used with --sync-unsub." >&2 + exit 1 + } + $NO_COMMIT && { + echo "--sync-unsub requires commits on $TARGET_BRANCH (omit --no-commit)." >&2 + exit 1 + } +fi + +if $NO_SYNC_FROM_UNSUB && $SYNC_UNSUB; then + echo "Do not combine --sync-unsub with --no-sync-from-unsub." >&2 exit 1 fi @@ -209,6 +275,17 @@ fi cd "$REPO_ROOT" +if $SYNC_UNSUB; then + git show-ref --verify --quiet "refs/heads/$TARGET_BRANCH" || { + echo "submodulize --sync-unsub: branch $TARGET_BRANCH does not exist yet." >&2 + exit 1 + } + git show-ref --verify --quiet refs/heads/unsubmodulized || { + echo "submodulize --sync-unsub: branch unsubmodulized not found." >&2 + exit 1 + } +fi + # Greenfield: no submodulized branch yet — run bootstrap (one-shot + unsub replay) instead of sub replay. if $REPLAY && ! $FORK_POINT_EXPLICIT && ! $BOOTSTRAP_EXPLICIT && ! git show-ref --verify --quiet refs/heads/submodulized; then BOOTSTRAP=true @@ -365,6 +442,181 @@ submodulize_stage_root_manifest() { fi } +# Replay commits on unsubmodulized (since merge-base with TARGET_BRANCH) onto TARGET_BRANCH: each commit +# may only touch paths under plugin-submodules.manifest roots; trees must match a plugin-repo commit. +submodulize_sync_from_unsubmodulized() { + if $DRY_RUN; then + echo "submodulize: sync from unsubmodulized — dry-run not supported; skipping." >&2 + return 0 + fi + local UNSUB_BRANCH=unsubmodulized + local active_br + active_br="$(git symbolic-ref -q --short HEAD 2>/dev/null || true)" + if [[ "$active_br" != "$TARGET_BRANCH" ]]; then + echo "submodulize: checking out $TARGET_BRANCH (sync from unsubmodulized)" >&2 + git checkout "$TARGET_BRANCH" + fi + if ! git diff --quiet 2>/dev/null || ! git diff --cached --quiet 2>/dev/null; then + echo "submodulize: sync from unsubmodulized — working tree or index is dirty; commit or stash first." >&2 + exit 1 + fi + + declare -a S_PATHS=() S_URLS=() + while IFS='|' read -r raw_path raw_url raw_branch; do + path="${raw_path#"${raw_path%%[![:space:]]*}"}" + path="${path%"${path##*[![:space:]]}"}" + url="${raw_url#"${raw_url%%[![:space:]]*}"}" + url="${url%"${url##*[![:space:]]}"}" + branch="${raw_branch#"${raw_branch%%[![:space:]]*}"}" + branch="${branch%"${branch##*[![:space:]]}"}" + [[ -z "$path" || "$path" =~ ^# ]] && continue + [[ -z "$url" ]] && { echo "Manifest: missing URL for path $path" >&2; exit 1; } + S_PATHS+=("$path") + S_URLS+=("$(rewrite_github_url_to_ssh "$url")") + done < "$MANIFEST" + + ((${#S_PATHS[@]} > 0)) || { echo "No manifest entries." >&2; exit 1; } + + manifest_root_for_path() { + local f="$1" p best="" + for p in "${S_PATHS[@]}"; do + if [[ "$f" == "$p" || "$f" == "$p"/* ]]; then + [[ ${#p} -gt ${#best} ]] && best="$p" + fi + done + printf '%s\n' "$best" + } + + find_plugin_commit_for_tree_sync() { + local pdir="$1" want_tree="$2" c t + while IFS= read -r c; do + t="$(git -C "$pdir" rev-parse "$c^{tree}" 2>/dev/null || true)" + [[ "$t" == "$want_tree" ]] && { echo "$c"; return 0; } + done < <(git -C "$pdir" rev-list --first-parent --reverse --all) + return 1 + } + + local BASE unsub_tip TMPD i url pdir U parent_u changed fn r ms mode newsha idx new_tree cur_tree nc msgf + BASE="$(git merge-base "$UNSUB_BRANCH" "$TARGET_BRANCH")" || { + echo "submodulize: sync from unsubmodulized — could not compute merge-base($UNSUB_BRANCH, $TARGET_BRANCH)." >&2 + exit 1 + } + unsub_tip="$(git rev-parse "$UNSUB_BRANCH^{commit}")" + if [[ "$BASE" == "$unsub_tip" ]]; then + echo "submodulize: sync from unsubmodulized — no new commits on $UNSUB_BRANCH since merge-base." >&2 + return 0 + fi + + TMPD="$(mktemp -d "${TMPDIR:-/tmp}/sub-sync-unsub.XXXXXX")" + declare -a PDIRS=() + for i in "${!S_PATHS[@]}"; do + url="${S_URLS[$i]}" + pdir="$TMPD/plugin_${i}_$(echo "${S_PATHS[$i]}" | tr '/' '_')" + PDIRS+=("$pdir") + if [[ ! -d "$pdir/.git" ]]; then + GIT_TERMINAL_PROMPT=0 git "${git_github_pat_c[@]}" clone --bare "$url" "$pdir" + fi + GIT_TERMINAL_PROMPT=0 git -C "$pdir" fetch -q origin 2>/dev/null || true + GIT_TERMINAL_PROMPT=0 git -C "$pdir" fetch -q "$url" "+refs/*:refs/remotes/import/*" 2>/dev/null || true + done + + git_write_gitmodules_from_index() { + local gm="$TMPD/gitmodules.tmp" blob _i _p _u _line _mode _sha + rm -f "$gm" + for _i in "${!S_PATHS[@]}"; do + _p="${S_PATHS[_i]}" + _u="${S_URLS[$_i]}" + _line="$(git ls-files --stage -- "$_p" 2>/dev/null | head -n1 || true)" + _mode="$(awk '{print $1}' <<< "$_line")" + _sha="$(awk '{print $2}' <<< "$_line")" + if [[ "$_mode" == "160000" ]]; then + printf '[submodule "%s"]\n\tpath = %s\n\turl = %s\n' "$_p" "$_p" "$_u" >>"$gm" + fi + done + if [[ -f "$gm" ]]; then + blob="$(git hash-object -w "$gm")" + git update-index --add --cacheinfo "100644,$blob,.gitmodules" + else + git rm --cached -f --ignore-unmatch .gitmodules 2>/dev/null || true + fi + } + + while IFS= read -r U; do + [[ -z "${U:-}" ]] && continue + if git rev-parse --verify -q "$U^2" >/dev/null 2>&1; then + echo "submodulize: sync from unsubmodulized — skipping merge commit $(git rev-parse --short "$U")" >&2 + continue + fi + parent_u="$(git rev-parse "$U^")" + changed="$(git diff-tree --no-commit-id --name-only -r "$parent_u" "$U")" + + declare -A seen_roots=() + while IFS= read -r fn; do + [[ -z "${fn:-}" ]] && continue + r="$(manifest_root_for_path "$fn")" + if [[ -z "$r" ]]; then + echo "submodulize: sync from unsubmodulized — commit $(git rev-parse --short "$U") touches $fn (outside manifest paths)." >&2 + exit 1 + fi + seen_roots[$r]=1 + done <<< "$changed" + + idx="$TMPD/index.${U}" + rm -f "$idx" + export GIT_INDEX_FILE="$idx" + git read-tree "$(git rev-parse HEAD)" + + for r in "${!seen_roots[@]}"; do + newsha="" + for j in "${!S_PATHS[@]}"; do + [[ "${S_PATHS[$j]}" == "$r" ]] || continue + pdir="${PDIRS[$j]}" + ms="$(git ls-tree "$U" -- "$r" 2>/dev/null | awk '{print $1 "\t" $3}' | head -n1)" + mode="${ms%%$'\t'*}" + if [[ "$mode" != "040000" ]]; then + echo "submodulize: sync from unsubmodulized — at $U path $r is not a vendored directory (mode $mode)." >&2 + exit 1 + fi + newsha="$(find_plugin_commit_for_tree_sync "$pdir" "$(git rev-parse "$U:$r" 2>/dev/null)")" || { + echo "submodulize: sync from unsubmodulized — no plugin commit matches tree at $U:$r (push the same tree to the plugin remote first)." >&2 + exit 1 + } + break + done + [[ -n "$newsha" ]] || continue + git rm -rf --cached --ignore-unmatch -q -- "$r" 2>/dev/null || true + git update-index --add --cacheinfo "160000,$newsha,$r" + done + + git_write_gitmodules_from_index + + new_tree="$(git write-tree)" + cur_tree="$(git rev-parse 'HEAD^{tree}')" + if [[ "$new_tree" == "$cur_tree" ]]; then + echo "submodulize: sync from unsubmodulized — $(git rev-parse --short "$U") → no gitlink change (skip)" >&2 + unset GIT_INDEX_FILE + rm -f "$idx" + continue + fi + + msgf="$TMPD/commitmsg.$U.txt" + { + git show -s --format=%B "$U" + printf '\nSynced-from-unsub: %s\n' "$U" + } >"$msgf" + nc="$(git commit-tree "$new_tree" -p HEAD -F "$msgf")" + git update-ref "refs/heads/$TARGET_BRANCH" "$nc" + git reset --hard -q "$nc" + echo "submodulize: sync from unsubmodulized — applied $(git rev-parse --short "$U") → $TARGET_BRANCH $(git rev-parse --short "$nc")" >&2 + + unset GIT_INDEX_FILE + rm -f "$idx" + done < <(git rev-list --reverse "${BASE}..${UNSUB_BRANCH}") + + unset GIT_INDEX_FILE + rm -rf "$TMPD" +} + run() { if $DRY_RUN; then printf '[dry-run] %q\n' "$@" @@ -737,5 +989,46 @@ if $REPLAY; then exit 0 fi +# After manifest one-shot: replay unsubmodulized → submodulized when unsub is ahead (or when --sync-unsub). +RUN_SYNC_FROM_UNSUB=false +if ! $NO_SYNC_FROM_UNSUB && ! $NO_COMMIT; then + if git show-ref --verify --quiet "refs/heads/$TARGET_BRANCH" && git show-ref --verify --quiet refs/heads/unsubmodulized; then + if $SYNC_UNSUB; then + RUN_SYNC_FROM_UNSUB=true + else + _mb="$(git merge-base unsubmodulized "$TARGET_BRANCH" 2>/dev/null || true)" + _ut="$(git rev-parse unsubmodulized^{commit} 2>/dev/null || true)" + if [[ -n "$_mb" && -n "$_ut" && "$_mb" != "$_ut" ]]; then + RUN_SYNC_FROM_UNSUB=true + fi + fi + fi +fi + +if $RUN_SYNC_FROM_UNSUB && ! $SYNC_UNSUB; then + echo "submodulize: unsubmodulized is ahead of merge-base with $TARGET_BRANCH — syncing plugin gitlinks." >&2 +fi + +if $RUN_SYNC_FROM_UNSUB; then + sb="$(git symbolic-ref -q --short HEAD 2>/dev/null || true)" + if [[ "$sb" != "$TARGET_BRANCH" ]]; then + echo "submodulize: checking out $TARGET_BRANCH (sync from unsubmodulized)" >&2 + git checkout "$TARGET_BRANCH" + fi + if [[ ! -f "$MANIFEST" ]]; then + if git cat-file -e "$TARGET_BRANCH:plugin-submodules.manifest" 2>/dev/null; then + git show "$TARGET_BRANCH:plugin-submodules.manifest" >"$MANIFEST" + echo "submodulize: wrote plugin-submodules.manifest from $TARGET_BRANCH (was missing at repo root)." >&2 + else + echo "Manifest not found: $MANIFEST (and not in $TARGET_BRANCH:plugin-submodules.manifest)" >&2 + exit 1 + fi + fi +fi + submodulize_one_shot_apply_manifest + +if $RUN_SYNC_FROM_UNSUB; then + submodulize_sync_from_unsubmodulized +fi exit 0 diff --git a/cleandev/tests/test-sync-unsub.sh b/cleandev/tests/test-sync-unsub.sh new file mode 100644 index 0000000..82a2977 --- /dev/null +++ b/cleandev/tests/test-sync-unsub.sh @@ -0,0 +1,119 @@ +#!/usr/bin/env bash +# Auto sync: when unsubmodulized is ahead of merge-base with submodulized, submodulize replays onto gitlinks. +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=lib.sh +source "$SCRIPT_DIR/lib.sh" +require_cleandev + +TMP=$(mktemp -d) +export HOME="$TMP/_home" +mkdir -p "$HOME" +git config --global protocol.file.allow always +trap 'rm -rf "$TMP"' EXIT + +PLUGIN="$TMP/plugin" +MOODLE="$TMP/moodle" +mkdir -p "$PLUGIN" "$MOODLE" + +git -C "$PLUGIN" init -b main +echo "v1" > "$PLUGIN/f.txt" +git -C "$PLUGIN" add f.txt +git -C "$PLUGIN" commit -q -m "p1" +echo "v2" >> "$PLUGIN/f.txt" +git -C "$PLUGIN" commit -q -am "p2" + +C0="$(git -C "$PLUGIN" rev-parse HEAD~1)" +C1="$(git -C "$PLUGIN" rev-parse HEAD)" + +git -C "$MOODLE" init -b main +printf 'core\n' > "$MOODLE/README.md" +mkdir -p "$MOODLE/mod/foo" +git -C "$PLUGIN" archive "$C0" | tar -x -C "$MOODLE/mod/foo" +git -C "$MOODLE" add README.md mod/foo +git -C "$MOODLE" commit -q -m "vendored base" + +M0="$(git -C "$MOODLE" rev-parse HEAD)" + +git -C "$MOODLE" checkout -q -b submodulized +printf 'mod/foo|%s|main\n' "$PLUGIN" > "$MOODLE/plugin-submodules.manifest" +git -C "$MOODLE" config -f .gitmodules submodule.mod/foo.path mod/foo +git -C "$MOODLE" config -f .gitmodules submodule.mod/foo.url "$PLUGIN" +git -C "$MOODLE" add .gitmodules plugin-submodules.manifest +git -C "$MOODLE" rm -rf --cached mod/foo +git -C "$MOODLE" update-index --add --cacheinfo "160000,$C0,mod/foo" +git -C "$MOODLE" commit -q -m "sub at c0" +# Manual gitlink commit can leave vendored files on disk (untracked); remove before branch hops. +git -C "$MOODLE" reset --hard -q HEAD +rm -rf "$MOODLE/mod/foo" + +git -C "$MOODLE" branch -q unsubmodulized "$M0" + +git -C "$MOODLE" checkout -q unsubmodulized +find "$MOODLE/mod/foo" -mindepth 1 -delete +git -C "$PLUGIN" archive "$C1" | tar -x -C "$MOODLE/mod/foo" +git -C "$MOODLE" add mod/foo +git -C "$MOODLE" commit -q -m "vendor bump to c1" + +# Stay on unsubmodulized; default one-shot path checks out submodulized and syncs when unsub is ahead. +"$CLEANDEV/submodulize.sh" --repo "$MOODLE" + +link="$(git -C "$MOODLE" ls-tree submodulized mod/foo | awk '{print $3}')" +[[ "$link" == "$C1" ]] || fail "expected gitlink $C1 on submodulized, got ${link:-empty}" + +git -C "$MOODLE" log -1 --format=%B submodulized | grep -q '^Synced-from-unsub:' || fail "expected Synced-from-unsub footer" + +ok "submodulize auto sync from unsubmodulized (fixture)" + +# --- opt-out: same vendor bump but --no-sync-from-unsub leaves gitlink at C0 --- +TMP2=$(mktemp -d) +export HOME="$TMP2/_home" +mkdir -p "$HOME" +git config --global protocol.file.allow always +trap 'rm -rf "$TMP" "$TMP2"' EXIT + +PLUGIN2="$TMP2/plugin" +MOODLE2="$TMP2/moodle" +mkdir -p "$PLUGIN2" "$MOODLE2" + +git -C "$PLUGIN2" init -b main +echo "v1" > "$PLUGIN2/f.txt" +git -C "$PLUGIN2" add f.txt +git -C "$PLUGIN2" commit -q -m "p1" +echo "v2" >> "$PLUGIN2/f.txt" +git -C "$PLUGIN2" commit -q -am "p2" +C0b="$(git -C "$PLUGIN2" rev-parse HEAD~1)" +C1b="$(git -C "$PLUGIN2" rev-parse HEAD)" + +git -C "$MOODLE2" init -b main +printf 'core\n' > "$MOODLE2/README.md" +mkdir -p "$MOODLE2/mod/foo" +git -C "$PLUGIN2" archive "$C0b" | tar -x -C "$MOODLE2/mod/foo" +git -C "$MOODLE2" add README.md mod/foo +git -C "$MOODLE2" commit -q -m "vendored base" +M0b="$(git -C "$MOODLE2" rev-parse HEAD)" + +git -C "$MOODLE2" checkout -q -b submodulized +printf 'mod/foo|%s|main\n' "$PLUGIN2" > "$MOODLE2/plugin-submodules.manifest" +git -C "$MOODLE2" config -f .gitmodules submodule.mod/foo.path mod/foo +git -C "$MOODLE2" config -f .gitmodules submodule.mod/foo.url "$PLUGIN2" +git -C "$MOODLE2" add .gitmodules plugin-submodules.manifest +git -C "$MOODLE2" rm -rf --cached mod/foo +git -C "$MOODLE2" update-index --add --cacheinfo "160000,$C0b,mod/foo" +git -C "$MOODLE2" commit -q -m "sub at c0" +git -C "$MOODLE2" reset --hard -q HEAD +rm -rf "$MOODLE2/mod/foo" + +git -C "$MOODLE2" branch -q unsubmodulized "$M0b" +git -C "$MOODLE2" checkout -q unsubmodulized +find "$MOODLE2/mod/foo" -mindepth 1 -delete +git -C "$PLUGIN2" archive "$C1b" | tar -x -C "$MOODLE2/mod/foo" +git -C "$MOODLE2" add mod/foo +git -C "$MOODLE2" commit -q -m "vendor bump to c1" + +"$CLEANDEV/submodulize.sh" --repo "$MOODLE2" --no-sync-from-unsub + +linkb="$(git -C "$MOODLE2" ls-tree submodulized mod/foo | awk '{print $3}')" +[[ "$linkb" == "$C0b" ]] || fail "expected gitlink unchanged at $C0b with --no-sync-from-unsub, got ${linkb:-empty}" + +ok "submodulize --no-sync-from-unsub skips sync (fixture)" From 7a3669cf9496ea0de74bb01c93ecc896ba26403b Mon Sep 17 00:00:00 2001 From: Steve Mattsen Date: Mon, 1 Jun 2026 16:55:49 -0500 Subject: [PATCH 16/18] Refactor project structure to integrate `submodulizer` as a Git submodule, replacing `cleandev` references. Update `.gitattributes` and `.gitignore` for new paths, and enhance `build.sh` to manage GitHub token resolution for private repositories. Revise `docker-compose.yml` to utilize secrets for token handling, and update `README.md` to reflect changes in submodule management. Remove obsolete `cleandev` test scripts and workflows, introducing new tests for `submodulizer`. This restructuring improves clarity and maintainability of the project. --- .gitattributes | 4 +- .github/workflows/cleandev-tests.yml | 23 - .github/workflows/submodulizer-tests.yml | 29 + .gitignore | 4 +- .gitmodules | 4 + Dockerfile | 25 +- Dockerfile.txt | 20 +- README.md | 32 +- ROADMAP.md | 14 +- build.sh | 19 +- cleandev/submodulize.sh | 1034 ----------------- cleandev/tests/lib.sh | 29 - cleandev/tests/run.sh | 32 - cleandev/tests/test-bootstrap.sh | 46 - cleandev/tests/test-manifest-default-repo.sh | 40 - cleandev/tests/test-manifest-lint.sh | 86 -- .../tests/test-manifest-submod-redundant.sh | 119 -- cleandev/tests/test-pat-parity.sh | 22 - cleandev/tests/test-replay-both.sh | 70 -- cleandev/tests/test-shellcheck.sh | 19 - cleandev/tests/test-sparse-submod.sh | 45 - cleandev/tests/test-submod-roundtrip.sh | 60 - .../tests/test-submodulize-extend-manifest.sh | 62 - cleandev/tests/test-sync-unsub.sh | 119 -- cleandev/tests/test-unsub-incremental.sh | 86 -- cleandev/unsubmodulize.sh | 649 ----------- docker-compose.yml | 15 +- new.sh | 63 +- submodulizer | 1 + .../TEAM-PROCESS.md | 6 +- .../docker-compose.github-runner.yml | 0 .../github-actions.workflow.example.yml | 0 .../github-runner.env.example | 0 submodulizer-local/lib-github-token.sh | 67 ++ .../manifest-submodulize-redundant.sh | 4 +- .../plugin-submodules.manifest | 2 +- 36 files changed, 242 insertions(+), 2608 deletions(-) delete mode 100644 .github/workflows/cleandev-tests.yml create mode 100644 .github/workflows/submodulizer-tests.yml create mode 100644 .gitmodules delete mode 100644 cleandev/submodulize.sh delete mode 100644 cleandev/tests/lib.sh delete mode 100644 cleandev/tests/run.sh delete mode 100644 cleandev/tests/test-bootstrap.sh delete mode 100644 cleandev/tests/test-manifest-default-repo.sh delete mode 100644 cleandev/tests/test-manifest-lint.sh delete mode 100644 cleandev/tests/test-manifest-submod-redundant.sh delete mode 100644 cleandev/tests/test-pat-parity.sh delete mode 100644 cleandev/tests/test-replay-both.sh delete mode 100644 cleandev/tests/test-shellcheck.sh delete mode 100644 cleandev/tests/test-sparse-submod.sh delete mode 100644 cleandev/tests/test-submod-roundtrip.sh delete mode 100644 cleandev/tests/test-submodulize-extend-manifest.sh delete mode 100644 cleandev/tests/test-sync-unsub.sh delete mode 100644 cleandev/tests/test-unsub-incremental.sh delete mode 100644 cleandev/unsubmodulize.sh create mode 160000 submodulizer rename {cleandev => submodulizer-local}/TEAM-PROCESS.md (94%) rename {cleandev => submodulizer-local}/docker-compose.github-runner.yml (100%) rename {cleandev => submodulizer-local}/github-actions.workflow.example.yml (100%) rename {cleandev => submodulizer-local}/github-runner.env.example (100%) create mode 100644 submodulizer-local/lib-github-token.sh rename {cleandev => submodulizer-local}/manifest-submodulize-redundant.sh (91%) rename {cleandev => submodulizer-local}/plugin-submodules.manifest (99%) diff --git a/.gitattributes b/.gitattributes index 6a2d3c2..1e6d3fd 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,3 +1,3 @@ # Shell scripts copied into Linux containers must use LF only (CRLF breaks bash). -cleandev/*.sh text eol=lf -cleandev/tests/*.sh text eol=lf +# (Scripts in the submodulizer/ submodule are governed by that repo's own .gitattributes.) +submodulizer-local/*.sh text eol=lf diff --git a/.github/workflows/cleandev-tests.yml b/.github/workflows/cleandev-tests.yml deleted file mode 100644 index 7f9808c..0000000 --- a/.github/workflows/cleandev-tests.yml +++ /dev/null @@ -1,23 +0,0 @@ -name: cleandev tests - -on: - push: - paths: - - "cleandev/**" - - ".github/workflows/cleandev-tests.yml" - pull_request: - paths: - - "cleandev/**" - - ".github/workflows/cleandev-tests.yml" - -jobs: - test: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Install shellcheck - run: sudo apt-get update && sudo apt-get install -y shellcheck - - - name: Run cleandev test suite - run: bash cleandev/tests/run.sh diff --git a/.github/workflows/submodulizer-tests.yml b/.github/workflows/submodulizer-tests.yml new file mode 100644 index 0000000..42c725d --- /dev/null +++ b/.github/workflows/submodulizer-tests.yml @@ -0,0 +1,29 @@ +name: submodulizer tests + +on: + push: + paths: + - ".gitmodules" + - "submodulizer" + - "submodulizer-local/**" + - ".github/workflows/submodulizer-tests.yml" + pull_request: + paths: + - ".gitmodules" + - "submodulizer" + - "submodulizer-local/**" + - ".github/workflows/submodulizer-tests.yml" + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Install shellcheck + run: sudo apt-get update && sudo apt-get install -y shellcheck + + - name: Run submodulizer test suite (upstream) + run: bash submodulizer/tests/run.sh diff --git a/.gitignore b/.gitignore index e1f8b9b..7687fab 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,3 @@ confidential -cleandev/github-runner.env -cleandev/.github-token \ No newline at end of file +submodulizer-local/github-runner.env +submodulizer-local/.github-token \ No newline at end of file diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..8b53c38 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,4 @@ +[submodule "submodulizer"] + path = submodulizer + url = https://github.com/smatts3/submodulizer.git + branch = master diff --git a/Dockerfile b/Dockerfile index 0c1b010..63cce40 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,11 +3,28 @@ FROM php:8.3-apache # Install basic tools RUN apt update && apt install git -y -# Import dev source code -RUN su -g www-data -c "git clone --branch develop --single-branch https://github.com/lsuonline/lsuce-moodle.git /var/www/html/" +# Import dev source code. lsuonline/moodleus is private; the token comes in via +# a BuildKit secret (id=github_token). docker-compose.yml maps it from the +# GITHUB_TOKEN env var; build.sh passes --secret id=github_token,env=GITHUB_TOKEN. +# Both entry points use submodulizer-local/lib-github-token.sh to populate +# GITHUB_TOKEN. The token is consumed via `git -c url...insteadOf`, so neither +# the token nor the username persist in image layers or /var/www/html/.git/config. +# Falls back to an anonymous clone when no secret is provided (useful for testing +# against any future public mirror; fails for the real private repo). +RUN --mount=type=secret,id=github_token \ + su -g www-data -c 'URL=https://github.com/lsuonline/moodleus.git; \ + if [ -s /run/secrets/github_token ]; then \ + TOKEN=$(cat /run/secrets/github_token); \ + git -c "url.https://smatts3%40lsu.edu:${TOKEN}@github.com/.insteadOf=https://github.com/" \ + clone --branch MOODLE_405_MAIN --single-branch "$URL" /var/www/html/; \ + else \ + git clone --branch MOODLE_405_MAIN --single-branch "$URL" /var/www/html/; \ + fi' -# Install dependencies -RUN apt-get install -y --fix-missing \ +# Install dependencies. `apt-get update` must run in the same layer as install: +# the base image ships a cached apt index that pins -security package versions +# Debian removes over time (e.g. libpng1.6 .deb 404s on stale indexes). +RUN apt-get update && apt-get install -y --fix-missing \ libpng-dev \ libonig-dev \ libjpeg-dev \ diff --git a/Dockerfile.txt b/Dockerfile.txt index 6e30cb7..7fcf790 100644 --- a/Dockerfile.txt +++ b/Dockerfile.txt @@ -3,11 +3,23 @@ FROM php:8.3-apache # Install basic tools RUN apt update && apt install git -y -# Import dev source code -RUN su -g www-data -c "git clone --branch develop --single-branch https://github.com/lsuonline/lsuce-moodle.git /var/www/html/" +# Import dev source code. See Dockerfile for full notes on auth: the token +# comes in via a BuildKit secret (id=github_token) supplied by build.sh / +# docker-compose.yml, consumed via `git -c url...insteadOf` so it never +# persists in image layers or .git/config. +RUN --mount=type=secret,id=github_token \ + su -g www-data -c 'URL=https://github.com/lsuonline/moodleus.git; \ + if [ -s /run/secrets/github_token ]; then \ + TOKEN=$(cat /run/secrets/github_token); \ + git -c "url.https://smatts3%40lsu.edu:${TOKEN}@github.com/.insteadOf=https://github.com/" \ + clone --branch MOODLE_405_MAIN --single-branch "$URL" /var/www/html/; \ + else \ + git clone --branch MOODLE_405_MAIN --single-branch "$URL" /var/www/html/; \ + fi' -# Install dependencies -RUN apt-get install -y \ +# Install dependencies. apt-get update must be in the same layer as install +# (stale base-image apt index causes -security 404s on removed packages). +RUN apt-get update && apt-get install -y \ libpng-dev \ libonig-dev \ libjpeg-dev \ diff --git a/README.md b/README.md index ab8e23d..617b937 100644 --- a/README.md +++ b/README.md @@ -62,7 +62,7 @@ To convert vendored plugin directories in the container to Git submodules (see * ./new.sh NAME --submodulize ``` -GitHub auth for private `lsuonline/*` repos: `GITHUB_TOKEN` or `GH_TOKEN`, file `cleandev/.github-token` (gitignored), interactive prompt when run in a TTY, or `SUBMODULIZE_SSH=1` if SSH works inside the container. +GitHub auth for private `lsuonline/*` repos: `GITHUB_TOKEN` or `GH_TOKEN`, file `submodulizer-local/.github-token` (gitignored), interactive prompt when run in a TTY, or `SUBMODULIZE_SSH=1` if SSH works inside the container. # Build @@ -90,13 +90,21 @@ These are **branches on the Moodle repo** (`lsuce-moodle`), not branch names in **Current repo state:** This project ships **layout converters** (`submodulize.sh` / `unsubmodulize.sh`). **Replay is the default:** they build `submodulized` / `unsubmodulized` branches with **one superproject commit per plugin-repo commit** (chronological ordering, carry-forward). **`--fork-point`** defaults to local **`master`** (else **`main`**) when omitted, if a default can be chosen safely; **`--source`** defaults for **unsub** to **`unsubmodulized`**, else **`master`**, else **`main`**, else **`submodulized`** (for **sub**: **`submodulized`**, **`master`**, **`main`**). Use **`--no-replay`** for one-shot conversion over the manifest only. A `new.sh` mode that detects vendored vs submodulized state without manual choice is still not implemented. -**Branch policy, Docker image decision, and updating the manifest from the CSV:** [cleandev/TEAM-PROCESS.md](cleandev/TEAM-PROCESS.md). +**Branch policy, Docker image decision, and updating the manifest from the CSV:** [submodulizer-local/TEAM-PROCESS.md](submodulizer-local/TEAM-PROCESS.md). -# Submodule tooling (`cleandev/`) +# Submodule tooling (`submodulizer/` + `submodulizer-local/`) -## Manifest (`cleandev/plugin-submodules.manifest`) +The conversion scripts (`submodulize.sh`, `unsubmodulize.sh`, their tests) live in **[smatts3/submodulizer](https://github.com/smatts3/submodulizer)**, vendored here as a Git submodule at `submodulizer/`. Project-owned glue (manifest, team process, runner config) lives next to it in `submodulizer-local/`. -This repo keeps a **canonical copy** under `cleandev/` for linting and for `new.sh --submodulize` (copied into the Moodle tree at runtime). When you run `submodulize.sh` / `unsubmodulize.sh` against a checkout, the default manifest path is **`plugin-submodules.manifest` at the Moodle superproject root** (`--repo` / current directory), not next to the scripts; use `--manifest PATH` to override. +After cloning this repo: + +```bash +git submodule update --init --recursive +``` + +## Manifest (`submodulizer-local/plugin-submodules.manifest`) + +This repo keeps a **canonical copy** under `submodulizer-local/` for linting and for `new.sh --submodulize` (copied into the Moodle tree at runtime). When you run `submodulize.sh` / `unsubmodulize.sh` against a checkout, the default manifest path is **`plugin-submodules.manifest` at the Moodle superproject root** (`--repo` / current directory), not next to the scripts; use `--manifest PATH` to override. - Format: `relative_path|clone_url|branch` (e.g. `mod/hvp|https://github.com/...|main`). Lines starting with `#` are ignored. - If the third field is empty, scripts default the branch to `main`; if `refs/heads/` is missing on the remote, they omit `-b` and use the remote’s default branch. @@ -109,20 +117,20 @@ This repo keeps a **canonical copy** under `cleandev/` for linting and for `new. | Script | Role | |--------|------| -| `cleandev/submodulize.sh` | Default **replay**; **`--fork-point`** defaults to `master`/`main` when omitted (see script help). **`--no-replay`**: one-shot vendored → submodules (sparse-checkout disabled first; skips paths already in `.gitmodules`; `GITHUB_TOKEN` via `-c url...insteadOf` for `ls-remote` / `submodule add`). | -| `cleandev/unsubmodulize.sh` | Default **replay**; same **`--fork-point`** defaulting. **`--no-replay`**: one-shot submodules → vendored (clone depth 1, drop nested `.git`, `git add`). Same `GITHUB_TOKEN` / `--ssh` as `submodulize.sh`. | +| `submodulizer/submodulize.sh` | Default **replay**; **`--fork-point`** defaults to `master`/`main` when omitted (see script help). **`--no-replay`**: one-shot vendored → submodules (sparse-checkout disabled first; skips paths already in `.gitmodules`; `GITHUB_TOKEN` via `-c url...insteadOf` for `ls-remote` / `submodule add`). | +| `submodulizer/unsubmodulize.sh` | Default **replay**; same **`--fork-point`** defaulting. **`--no-replay`**: one-shot submodules → vendored (clone depth 1, drop nested `.git`, `git add`). Same `GITHUB_TOKEN` / `--ssh` as `submodulize.sh`. | Run manually from a Moodle clone: ```bash -./cleandev/submodulize.sh [--no-replay] [--dry-run] [--no-commit] [--ssh] [--manifest PATH] [--repo ROOT] -./cleandev/unsubmodulize.sh [--no-replay] [--dry-run] [--no-commit] [--ssh] [--manifest PATH] [--repo ROOT] +./submodulizer/submodulize.sh [--no-replay] [--dry-run] [--no-commit] [--ssh] [--manifest PATH] [--repo ROOT] +./submodulizer/unsubmodulize.sh [--no-replay] [--dry-run] [--no-commit] [--ssh] [--manifest PATH] [--repo ROOT] ``` Automated tests (manifest lint, PAT wiring checks, `submodulize`/`unsubmodulize` round-trip in temp repos—no changes to your working tree): ```bash -bash cleandev/tests/run.sh +bash submodulizer/tests/run.sh ``` ## Container startup (`new.sh`) @@ -130,9 +138,9 @@ bash cleandev/tests/run.sh - Compose project name = first argument (containers `{NAME}-moodle`, etc.). - Web service builds from `.` per `docker-compose.yml`. - As `www-data`: `git fetch` / `git merge origin/develop`; removes `blocks/ues_people` and uses `skip-worktree` so it does not clash with `block_lsu_people` (see `config/moodle-pull` for the same idea on `git pull`). -- With `--submodulize`: copies scripts into the container, stages `cleandev/plugin-submodules.manifest` into `/var/www/html/plugin-submodules.manifest` after the merge, resolves GitHub token, sets local `url...insteadOf` when using HTTPS token, runs `manifest-submodulize-redundant.sh` / `submodulize.sh --no-replay` with default manifest paths (`--repo /var/www/html`, optional SSH via `SUBMODULIZE_SSH=1`). +- With `--submodulize`: copies scripts into the container, stages `submodulizer-local/plugin-submodules.manifest` into `/var/www/html/plugin-submodules.manifest` after the merge, resolves GitHub token, sets local `url...insteadOf` when using HTTPS token, runs `manifest-submodulize-redundant.sh` / `submodulize.sh --no-replay` with default manifest paths (`--repo /var/www/html`, optional SSH via `SUBMODULIZE_SSH=1`). -Secrets: `cleandev/.github-token` is listed in `.gitignore`. +Secrets: `submodulizer-local/.github-token` is listed in `.gitignore`. # Roadmap diff --git a/ROADMAP.md b/ROADMAP.md index 111a206..68010c7 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -6,25 +6,25 @@ Progress toward **seamless `develop` ↔ `cleandev`** (on [lsuce-moodle](https:/ - [x] Docker Compose + Traefik-oriented local stack via `new.sh` - [x] Web image built from repo `Dockerfile` (PHP 8.3) to avoid Moodle 4.5 + PHP 8.4 CLI `SerializableClosure` install failure (`docker-compose.yml` documents reverting to a pre-built image if desired) -- [x] `cleandev/plugin-submodules.manifest` + `submodulize.sh` + `unsubmodulize.sh` +- [x] `submodulizer-local/plugin-submodules.manifest` + `submodulizer/submodulize.sh` + `submodulizer/unsubmodulize.sh` (the scripts vendored as a Git submodule from [smatts3/submodulizer](https://github.com/smatts3/submodulizer)) - [x] `new.sh --submodulize` with token file / env / prompt and optional SSH - [x] Sparse-checkout disabled in both scripts before mutating plugin paths - [x] `blocks/ues_people` handling in `new.sh` and `config/moodle-pull` (merge / pull alias) -- [x] Git ignore for `cleandev/.github-token` -- [x] **Team process, image strategy, manifest-from-CSV docs:** [cleandev/TEAM-PROCESS.md](cleandev/TEAM-PROCESS.md) +- [x] Git ignore for `submodulizer-local/.github-token` +- [x] **Team process, image strategy, manifest-from-CSV docs:** [submodulizer-local/TEAM-PROCESS.md](submodulizer-local/TEAM-PROCESS.md) ## TODO - [ ] **Commit portability:** Script or documented workflow to **replay or port commits** across layouts with a **header** noting the original commit (and branch/repo). *Not in repo today—only bulk layout conversion exists.* - [x] **`new.sh` / tooling:** With `--submodulize`, **detect** when every manifest path is already a submodule (cleandev-style) and **skip** `submodulize.sh` instead of a no-op pass. *Not done: auto-run conversion without passing `--submodulize`.* - [x] **`unsubmodulize.sh`:** Same `GITHUB_TOKEN` `-c url.insteadOf` pattern as `submodulize.sh` for private HTTPS (parity with submodule add). -- [x] **Monorepo manifest + scripts:** **One-shot manual procedures** for commented monorepo lines: [cleandev/TEAM-PROCESS.md](cleandev/TEAM-PROCESS.md) (manifest regeneration → Monorepos). *Not done: automated “clone once, map subpaths” in `submodulize.sh`.* +- [x] **Monorepo manifest + scripts:** **One-shot manual procedures** for commented monorepo lines: [submodulizer-local/TEAM-PROCESS.md](submodulizer-local/TEAM-PROCESS.md) (manifest regeneration → Monorepos). *Not done: automated “clone once, map subpaths” in `submodulize.sh`.* - [ ] **`local/ml`:** Point at the correct Git remote when known; uncomment or add manifest line. -- [x] **CI (cleandev):** GitHub Actions runs `cleandev/tests/run.sh` (manifest lint, round-trip integration tests, shellcheck on Ubuntu). +- [x] **CI (submodulizer):** GitHub Actions runs `submodulizer/tests/run.sh` (manifest lint, round-trip integration tests, shellcheck on Ubuntu). - [ ] **Optional CI:** Check that manifest paths still match **lsuce-moodle** `develop` plugin layout when plugins move. --- -For branch policy, when to use `new.sh --submodulize`, image strategy, and refreshing `plugin-submodules.manifest` from the CSV, see [cleandev/TEAM-PROCESS.md](cleandev/TEAM-PROCESS.md). +For branch policy, when to use `new.sh --submodulize`, image strategy, and refreshing `plugin-submodules.manifest` from the CSV, see [submodulizer-local/TEAM-PROCESS.md](submodulizer-local/TEAM-PROCESS.md). -For a concise summary of flags and auth behavior, see comments at the top of `cleandev/submodulize.sh` and `cleandev/unsubmodulize.sh`. +For a concise summary of flags and auth behavior, see comments at the top of `submodulizer/submodulize.sh` and `submodulizer/unsubmodulize.sh`. diff --git a/build.sh b/build.sh index bc25b93..caaabd9 100644 --- a/build.sh +++ b/build.sh @@ -1,5 +1,16 @@ -#!/bin/sh +#!/usr/bin/env bash +set -euo pipefail -SCRIPT_DIR="$(dirname "$(readlink -f "$0")")"; -cd $SCRIPT_DIR; -docker build -t lsuonline/moodle-dev:latest "$@" . \ No newline at end of file +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +cd "$SCRIPT_DIR" + +# lsuonline/moodleus is a private repo; the Dockerfile clones it via a BuildKit +# secret. Resolve+export GITHUB_TOKEN, then pass --secret so docker build can +# mount it. See submodulizer-local/lib-github-token.sh for sources/prompts. +# shellcheck source=submodulizer-local/lib-github-token.sh +. "${SCRIPT_DIR}/submodulizer-local/lib-github-token.sh" +resolve_github_token || exit 1 + +DOCKER_BUILDKIT=1 docker build \ + --secret id=github_token,env=GITHUB_TOKEN \ + -t lsuonline/moodle-dev:latest "$@" . diff --git a/cleandev/submodulize.sh b/cleandev/submodulize.sh deleted file mode 100644 index 31c6f66..0000000 --- a/cleandev/submodulize.sh +++ /dev/null @@ -1,1034 +0,0 @@ -#!/usr/bin/env bash -# Convert vendored plugin directories (plain files in the Moodle superproject) into git submodules. -# Intended for the submodule-layout Moodle checkout: keep plugin-submodules.manifest at the superproject -# root (same repo as .gitmodules will live in). Run from inside that clone, or pass ROOT / --repo. -# Default mode is replay: builds branch submodulized with one superproject commit per plugin-repo commit. -# Use --no-replay for one-shot conversion (manifest loop only). When submodulized and unsubmodulized both -# exist and unsub is ahead of merge-base(unsub, sub), the one-shot path also replays those commits onto gitlinks. -# -# Requires: git, a clean enough working tree (commit or stash first if paths are dirty). -# -# Usage: -# ./cleandev/submodulize.sh [ROOT] [--dry-run] [--no-commit] [--ssh] [--manifest PATH] [--repo ROOT] -# ./cleandev/submodulize.sh --fork-point BASE ... # replay (default) -# ./cleandev/submodulize.sh --no-replay ... # one-shot -# Bare ROOT is the same as --repo (optional; may appear before or after flags). -# -# Private GitHub repos over HTTPS need credentials. Set GITHUB_TOKEN (PAT) so HTTPS URLs are rewritten -# for ls-remote / submodule add (parent repo url.insteadOf is not always applied to submodule clone). -# Or use --ssh. In Docker with no TTY you see: "could not read Username for 'https://github.com'". -# -# Requires bash (arrays, pipefail). Do not run as `sh this-script.sh`; use `bash` or execute directly. - -if [ -z "${BASH_VERSION:-}" ]; then - printf '%s: requires bash, not sh. Example: bash "%s" ./ ...your args...\n' "${0##*/}" "$0" >&2 - exit 1 -fi - -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" - -MANIFEST="" -MANIFEST_EXPLICIT=false -DRY_RUN=false -NO_COMMIT=false -USE_SSH=false -REPO_ROOT="" -REPLAY=true -FORK_POINT="" -SOURCE_BRANCH="" -TARGET_BRANCH="submodulized" -REPLAY_ORDER="chronological" -FORCE_REPLAY=false -REPLAY_SOURCE_EXPLICIT=false -REPLAY_TARGET_EXPLICIT=false -REPLAY_ORDER_EXPLICIT=false -FORK_POINT_EXPLICIT=false -BOOTSTRAP=false -BOOTSTRAP_EXPLICIT=false -SYNC_UNSUB=false -NO_SYNC_FROM_UNSUB=false -declare -a PLUGIN_BASE_OVERRIDES=() - -usage() { - cat <<'EOF' -Convert vendored plugin directories in the Moodle superproject into git submodules (cleandev-style). -The manifest lists paths and clone URLs; it normally lives in the superproject root as -plugin-submodules.manifest (not under cleandev/). Moodle root defaults to the current directory’s -git superproject (git rev-parse --show-toplevel), unless you set it explicitly. - -Usage: - submodulize.sh [ROOT] [OPTIONS...] - submodulize.sh [OPTIONS...] [ROOT] - - ROOT — optional path to the Moodle git checkout. Give it as a single bare argument (no leading -), - anywhere among the flags; same meaning as --repo. Only one repo path: do not pass a second - bare path, and do not put a bare path after --repo (that is rejected). - -Examples: - submodulize.sh --no-replay ~/workspace/moodle - submodulize.sh ./ --bootstrap - submodulize.sh ./ # same as --bootstrap when submodulized branch does not exist yet - submodulize.sh --dry-run --no-replay ~/workspace/moodle - submodulize.sh --fork-point abc123 --source submodulized - submodulize.sh ./ # when unsubmodulized is ahead of merge-base(sub,unsub), syncs onto submodulized automatically - submodulize.sh --no-sync-from-unsub ./ # manifest one-shot only; skip unsub → submodulized replay - -Options: - --dry-run Print actions without changing the repo - --no-commit Stage submodule changes but do not commit (bootstrap still commits the submodule layout so unsub replay can run) - --ssh Use git@github.com URLs for github.com HTTPS entries - --manifest PATH Plugin manifest (default: ROOT/plugin-submodules.manifest). If that file sits at the repo root, one-shot/bootstrap also stages it on the submodule branch so master/unsubmodulized can keep it untracked. - --repo ROOT Moodle git root (explicit form of a bare ROOT; overrides an earlier bare ROOT; a bare path after --repo is an error) - -Bootstrap (from vendored tree + manifest → submodulized + unsubmodulized): - --bootstrap Add submodules on branch submodulized, then run unsubmodulize replay to create unsubmodulized. - If local branch submodulized is missing, this runs automatically (same as passing --bootstrap). - -Replay (default — one superproject commit per plugin-repo commit on --target): - --no-replay One-shot mode: add submodules per manifest (no branch replay) - --replay Replay mode (default; explicit if you toggled --no-replay earlier on the command line) - --fork-point BASE Start of replay (default: local master, else main, when omitted — see docs) - --source BR End state per path (gitlinks and/or vendored trees; default: submodulized, else master, else main) - --target BR Branch to create/update (default: submodulized) - --order NAME Only chronological - --force Replay: delete and rebuild --target from --fork-point. Omit when --target exists to only apply manifest changes (new paths → submodule add). - --plugin-base P=S Optional start SHA for manifest path P - --sync-unsub Force the unsub→submodulized sync step (even if merge-base already equals unsub tip). Normally automatic when both branches exist and unsub is ahead. - --no-sync-from-unsub After manifest one-shot: do not replay unsubmodulized onto submodulized (overrides automatic sync). - -Requires: git, and a clean enough working tree (commit or stash if plugin paths are dirty). - -Private GitHub over HTTPS: set GITHUB_TOKEN (PAT) so ls-remote / submodule add can authenticate, -or use --ssh. Without credentials in non-interactive environments you may see: - could not read Username for 'https://github.com' -EOF -} - -while [[ $# -gt 0 ]]; do - case "$1" in - --dry-run) DRY_RUN=true; shift ;; - --no-commit) NO_COMMIT=true; shift ;; - --ssh) USE_SSH=true; shift ;; - --no-replay) REPLAY=false; shift ;; - --replay) REPLAY=true; shift ;; - --bootstrap) - BOOTSTRAP=true - BOOTSTRAP_EXPLICIT=true - REPLAY=false - shift - ;; - --fork-point) - FORK_POINT="${2:?}" - FORK_POINT_EXPLICIT=true - shift 2 - ;; - --source) - SOURCE_BRANCH="${2:?}" - REPLAY_SOURCE_EXPLICIT=true - shift 2 - ;; - --target) - TARGET_BRANCH="${2:?}" - REPLAY_TARGET_EXPLICIT=true - shift 2 - ;; - --order) - REPLAY_ORDER="${2:?}" - REPLAY_ORDER_EXPLICIT=true - shift 2 - ;; - --force) - FORCE_REPLAY=true - shift - ;; - --plugin-base) - PLUGIN_BASE_OVERRIDES+=("${2:?}") - shift 2 - ;; - --sync-unsub) - SYNC_UNSUB=true - REPLAY=false - shift - ;; - --no-sync-from-unsub) - NO_SYNC_FROM_UNSUB=true - shift - ;; - --manifest) - MANIFEST="${2:?}" - MANIFEST_EXPLICIT=true - shift 2 - ;; - --repo) - REPO_ROOT="${2:?}" - shift 2 - ;; - -h|--help) - usage - exit 0 - ;; - -*) - echo "Unknown option: $1" >&2 - exit 1 - ;; - *) - if [[ -n "$REPO_ROOT" ]]; then - echo "Unexpected argument: $1 (repo already set via --repo or positional path)" >&2 - exit 1 - fi - REPO_ROOT="$1" - shift - ;; - esac -done - -if [[ -z "$REPO_ROOT" ]]; then - REPO_ROOT="$(git rev-parse --show-toplevel 2>/dev/null)" || { - echo "Not inside a git repository. Pass the Moodle root as a bare path or use --repo /path/to/moodle" >&2 - exit 1 - } -fi - -if ! $MANIFEST_EXPLICIT; then - MANIFEST="${REPO_ROOT%/}/plugin-submodules.manifest" -fi - -if [[ ! -f "$MANIFEST" ]]; then - if $MANIFEST_EXPLICIT; then - echo "Manifest not found: $MANIFEST" >&2 - exit 1 - elif $SYNC_UNSUB; then - : - elif git -C "$REPO_ROOT" show-ref --verify --quiet refs/heads/submodulized 2>/dev/null \ - && git -C "$REPO_ROOT" show-ref --verify --quiet refs/heads/unsubmodulized 2>/dev/null; then - : - else - echo "Manifest not found: $MANIFEST" >&2 - exit 1 - fi -fi - -if $SYNC_UNSUB; then - $BOOTSTRAP_EXPLICIT && { - echo "Do not combine --sync-unsub with --bootstrap." >&2 - exit 1 - } - $FORK_POINT_EXPLICIT && { - echo "Do not combine --sync-unsub with --fork-point." >&2 - exit 1 - } - $REPLAY_SOURCE_EXPLICIT && { - echo "Do not combine --sync-unsub with --source." >&2 - exit 1 - } - $REPLAY_TARGET_EXPLICIT && { - echo "Do not combine --sync-unsub with --target." >&2 - exit 1 - } - $REPLAY_ORDER_EXPLICIT && { - echo "Do not combine --sync-unsub with --order." >&2 - exit 1 - } - ((${#PLUGIN_BASE_OVERRIDES[@]} == 0)) || { - echo "--plugin-base is not used with --sync-unsub." >&2 - exit 1 - } - $FORCE_REPLAY && { - echo "--force is not used with --sync-unsub." >&2 - exit 1 - } - $NO_COMMIT && { - echo "--sync-unsub requires commits on $TARGET_BRANCH (omit --no-commit)." >&2 - exit 1 - } -fi - -if $NO_SYNC_FROM_UNSUB && $SYNC_UNSUB; then - echo "Do not combine --sync-unsub with --no-sync-from-unsub." >&2 - exit 1 -fi - -if $BOOTSTRAP_EXPLICIT; then - $FORK_POINT_EXPLICIT && { - echo "Do not combine --bootstrap with --fork-point (bootstrap records the current commit as the fork)." >&2 - exit 1 - } - $REPLAY_SOURCE_EXPLICIT && { - echo "Do not combine --bootstrap with --source (bootstrap uses submodulized → unsubmodulized)." >&2 - exit 1 - } - $REPLAY_TARGET_EXPLICIT && { - echo "Do not combine --bootstrap with --target." >&2 - exit 1 - } - $REPLAY_ORDER_EXPLICIT && { - echo "Do not combine --bootstrap with --order." >&2 - exit 1 - } - ((${#PLUGIN_BASE_OVERRIDES[@]} == 0)) || { - echo "--plugin-base is not used with --bootstrap." >&2 - exit 1 - } -fi - -cd "$REPO_ROOT" - -if $SYNC_UNSUB; then - git show-ref --verify --quiet "refs/heads/$TARGET_BRANCH" || { - echo "submodulize --sync-unsub: branch $TARGET_BRANCH does not exist yet." >&2 - exit 1 - } - git show-ref --verify --quiet refs/heads/unsubmodulized || { - echo "submodulize --sync-unsub: branch unsubmodulized not found." >&2 - exit 1 - } -fi - -# Greenfield: no submodulized branch yet — run bootstrap (one-shot + unsub replay) instead of sub replay. -if $REPLAY && ! $FORK_POINT_EXPLICIT && ! $BOOTSTRAP_EXPLICIT && ! git show-ref --verify --quiet refs/heads/submodulized; then - BOOTSTRAP=true - REPLAY=false - echo "submodulize: no local branch submodulized — bootstrap (add submodules, then build unsubmodulized)." >&2 -fi - -if $REPLAY; then - [[ "$REPLAY_ORDER" == "chronological" ]] || { - echo "Only --order chronological is supported: $REPLAY_ORDER" >&2 - exit 1 - } -elif $BOOTSTRAP; then - : -else - [[ -z "$FORK_POINT" ]] || { - echo "--fork-point is only used in replay mode (omit --no-replay)" >&2 - exit 1 - } - ((${#PLUGIN_BASE_OVERRIDES[@]} == 0)) || { - echo "--plugin-base is only valid in replay mode (omit --no-replay)" >&2 - exit 1 - } - $FORCE_REPLAY && { - echo "--force is only valid in replay mode or bootstrap (omit --no-replay)" >&2 - exit 1 - } - $REPLAY_SOURCE_EXPLICIT && { - echo "--source is only valid in replay mode (omit --no-replay)" >&2 - exit 1 - } - $REPLAY_TARGET_EXPLICIT && { - echo "--target is only valid in replay mode (omit --no-replay)" >&2 - exit 1 - } - $REPLAY_ORDER_EXPLICIT && { - echo "--order is only valid in replay mode (omit --no-replay)" >&2 - exit 1 - } -fi - -# When --fork-point is omitted in replay mode, default to local master (else main) if safe. -if $REPLAY && [[ -z "$FORK_POINT" ]]; then - base_ref="" - if git show-ref --verify --quiet refs/heads/master; then - base_ref=master - elif git show-ref --verify --quiet refs/heads/main; then - base_ref=main - else - echo "Replay needs --fork-point (no local master or main branch to use as default)." >&2 - exit 1 - fi - head_sha="$(git rev-parse HEAD)" - master_sha="$(git rev-parse "${base_ref}^{commit}")" - target_exists=false - git show-ref --verify --quiet "refs/heads/$TARGET_BRANCH" && target_exists=true - if ! $target_exists; then - FORK_POINT="$base_ref" - echo "Default --fork-point $FORK_POINT ($TARGET_BRANCH does not exist yet)." >&2 - else - target_sha="$(git rev-parse "$TARGET_BRANCH^{commit}")" - if [[ "$head_sha" == "$master_sha" ]] || [[ "$head_sha" == "$target_sha" ]]; then - FORK_POINT="$base_ref" - echo "Default --fork-point $FORK_POINT (HEAD is ${base_ref} or $TARGET_BRANCH)." >&2 - else - FORK_POINT="$base_ref" - echo "Default --fork-point $FORK_POINT (recreating $TARGET_BRANCH from ${base_ref}; use --force if the branch already exists)." >&2 - fi - fi -fi - -# When --source is omitted in replay mode: prefer submodulized, else master, else main. -if $REPLAY && ! $REPLAY_SOURCE_EXPLICIT; then - if git show-ref --verify --quiet refs/heads/submodulized; then - SOURCE_BRANCH=submodulized - elif git show-ref --verify --quiet refs/heads/master; then - SOURCE_BRANCH=master - elif git show-ref --verify --quiet refs/heads/main; then - SOURCE_BRANCH=main - else - echo "Replay: pass --source BR (no local submodulized, master, or main branch found)." >&2 - exit 1 - fi - echo "Default --source $SOURCE_BRANCH" >&2 -fi - -# Moodle dev images often use sparse-checkout (cone, index.sparse, or only .git/info/sparse-checkout). -# If any of that is active, git rm / submodule add can refuse paths "outside" the cone. -disable_sparse_checkout_if_needed() { - $DRY_RUN && return 0 - local active= - [[ -f .git/info/sparse-checkout ]] && active=1 - [[ "$(git config --bool core.sparseCheckout 2>/dev/null)" == "true" ]] && active=1 - [[ "$(git config --bool index.sparse 2>/dev/null)" == "true" ]] && active=1 - if [[ -z "$active" ]] && command -v git >/dev/null; then - local listed - listed="$(git sparse-checkout list 2>/dev/null | head -n 1 || true)" - [[ -n "${listed// }" ]] && active=1 - fi - [[ -z "$active" ]] && return 0 - echo "Disabling sparse-checkout so plugin paths can be converted to submodules." >&2 - git sparse-checkout disable 2>/dev/null || true - git config core.sparseCheckout false 2>/dev/null || true - git config --unset-all core.sparseCheckoutCone 2>/dev/null || true - git config index.sparse false 2>/dev/null || true - rm -f .git/info/sparse-checkout -} - -disable_sparse_checkout_if_needed - -# Extra -c flags so submodule clone honors GitHub PAT (superproject local config is skipped by some git versions). -git_github_pat_c=() -if [[ -n "${GITHUB_TOKEN:-}" ]]; then - git_github_pat_c+=(-c "url.https://${GITHUB_TOKEN}@github.com/.insteadOf=https://github.com/") -fi - -# github.com HTTPS → SSH (git@github.com:org/repo.git) when --ssh is set. -rewrite_github_url_to_ssh() { - local u="$1" - if $USE_SSH && [[ "$u" == https://github.com/* ]]; then - printf '%s\n' "git@github.com:${u#https://github.com/}" - else - printf '%s\n' "$u" - fi -} - -# Parent index only has gitlinks; files inside each plugin dir are the submodule checkout (not parent blobs). -# checkout to a vendored branch would replace those dirs unless submodule working trees are cleared first. -submodulize_hint_switch_from_submodule_branch() { - cat <<'EOF' >&2 -To switch to a branch with vendored plugin files (not gitlinks), clear submodule checkouts first, then checkout: - git submodule deinit -f --all - git checkout master # or main / unsubmodulized -If plugin-submodules.manifest was only committed on submodulized, copy it back in after checkout (keep it untracked on vendored branches). -EOF -} - -# Commit root plugin-submodules.manifest on the submodule branch only (vendored branches can keep a local untracked copy). -submodulize_stage_root_manifest() { - [[ -f "$MANIFEST" ]] || return 0 - [[ "$(basename -- "$MANIFEST")" == "plugin-submodules.manifest" ]] || return 0 - local rtop mtop - rtop="$(cd "$REPO_ROOT" && pwd -P 2>/dev/null)" || return 0 - mtop="$(cd "$(dirname -- "$MANIFEST")" && pwd -P 2>/dev/null)" || return 0 - [[ "$mtop" == "$rtop" ]] || return 0 - if $DRY_RUN; then - printf '[dry-run] git add %q\n' "$MANIFEST" - return 0 - fi - if git check-ignore -q -- "$MANIFEST" 2>/dev/null; then - git add -f -- "$MANIFEST" || true - else - git add -- "$MANIFEST" || true - fi -} - -# Replay commits on unsubmodulized (since merge-base with TARGET_BRANCH) onto TARGET_BRANCH: each commit -# may only touch paths under plugin-submodules.manifest roots; trees must match a plugin-repo commit. -submodulize_sync_from_unsubmodulized() { - if $DRY_RUN; then - echo "submodulize: sync from unsubmodulized — dry-run not supported; skipping." >&2 - return 0 - fi - local UNSUB_BRANCH=unsubmodulized - local active_br - active_br="$(git symbolic-ref -q --short HEAD 2>/dev/null || true)" - if [[ "$active_br" != "$TARGET_BRANCH" ]]; then - echo "submodulize: checking out $TARGET_BRANCH (sync from unsubmodulized)" >&2 - git checkout "$TARGET_BRANCH" - fi - if ! git diff --quiet 2>/dev/null || ! git diff --cached --quiet 2>/dev/null; then - echo "submodulize: sync from unsubmodulized — working tree or index is dirty; commit or stash first." >&2 - exit 1 - fi - - declare -a S_PATHS=() S_URLS=() - while IFS='|' read -r raw_path raw_url raw_branch; do - path="${raw_path#"${raw_path%%[![:space:]]*}"}" - path="${path%"${path##*[![:space:]]}"}" - url="${raw_url#"${raw_url%%[![:space:]]*}"}" - url="${url%"${url##*[![:space:]]}"}" - branch="${raw_branch#"${raw_branch%%[![:space:]]*}"}" - branch="${branch%"${branch##*[![:space:]]}"}" - [[ -z "$path" || "$path" =~ ^# ]] && continue - [[ -z "$url" ]] && { echo "Manifest: missing URL for path $path" >&2; exit 1; } - S_PATHS+=("$path") - S_URLS+=("$(rewrite_github_url_to_ssh "$url")") - done < "$MANIFEST" - - ((${#S_PATHS[@]} > 0)) || { echo "No manifest entries." >&2; exit 1; } - - manifest_root_for_path() { - local f="$1" p best="" - for p in "${S_PATHS[@]}"; do - if [[ "$f" == "$p" || "$f" == "$p"/* ]]; then - [[ ${#p} -gt ${#best} ]] && best="$p" - fi - done - printf '%s\n' "$best" - } - - find_plugin_commit_for_tree_sync() { - local pdir="$1" want_tree="$2" c t - while IFS= read -r c; do - t="$(git -C "$pdir" rev-parse "$c^{tree}" 2>/dev/null || true)" - [[ "$t" == "$want_tree" ]] && { echo "$c"; return 0; } - done < <(git -C "$pdir" rev-list --first-parent --reverse --all) - return 1 - } - - local BASE unsub_tip TMPD i url pdir U parent_u changed fn r ms mode newsha idx new_tree cur_tree nc msgf - BASE="$(git merge-base "$UNSUB_BRANCH" "$TARGET_BRANCH")" || { - echo "submodulize: sync from unsubmodulized — could not compute merge-base($UNSUB_BRANCH, $TARGET_BRANCH)." >&2 - exit 1 - } - unsub_tip="$(git rev-parse "$UNSUB_BRANCH^{commit}")" - if [[ "$BASE" == "$unsub_tip" ]]; then - echo "submodulize: sync from unsubmodulized — no new commits on $UNSUB_BRANCH since merge-base." >&2 - return 0 - fi - - TMPD="$(mktemp -d "${TMPDIR:-/tmp}/sub-sync-unsub.XXXXXX")" - declare -a PDIRS=() - for i in "${!S_PATHS[@]}"; do - url="${S_URLS[$i]}" - pdir="$TMPD/plugin_${i}_$(echo "${S_PATHS[$i]}" | tr '/' '_')" - PDIRS+=("$pdir") - if [[ ! -d "$pdir/.git" ]]; then - GIT_TERMINAL_PROMPT=0 git "${git_github_pat_c[@]}" clone --bare "$url" "$pdir" - fi - GIT_TERMINAL_PROMPT=0 git -C "$pdir" fetch -q origin 2>/dev/null || true - GIT_TERMINAL_PROMPT=0 git -C "$pdir" fetch -q "$url" "+refs/*:refs/remotes/import/*" 2>/dev/null || true - done - - git_write_gitmodules_from_index() { - local gm="$TMPD/gitmodules.tmp" blob _i _p _u _line _mode _sha - rm -f "$gm" - for _i in "${!S_PATHS[@]}"; do - _p="${S_PATHS[_i]}" - _u="${S_URLS[$_i]}" - _line="$(git ls-files --stage -- "$_p" 2>/dev/null | head -n1 || true)" - _mode="$(awk '{print $1}' <<< "$_line")" - _sha="$(awk '{print $2}' <<< "$_line")" - if [[ "$_mode" == "160000" ]]; then - printf '[submodule "%s"]\n\tpath = %s\n\turl = %s\n' "$_p" "$_p" "$_u" >>"$gm" - fi - done - if [[ -f "$gm" ]]; then - blob="$(git hash-object -w "$gm")" - git update-index --add --cacheinfo "100644,$blob,.gitmodules" - else - git rm --cached -f --ignore-unmatch .gitmodules 2>/dev/null || true - fi - } - - while IFS= read -r U; do - [[ -z "${U:-}" ]] && continue - if git rev-parse --verify -q "$U^2" >/dev/null 2>&1; then - echo "submodulize: sync from unsubmodulized — skipping merge commit $(git rev-parse --short "$U")" >&2 - continue - fi - parent_u="$(git rev-parse "$U^")" - changed="$(git diff-tree --no-commit-id --name-only -r "$parent_u" "$U")" - - declare -A seen_roots=() - while IFS= read -r fn; do - [[ -z "${fn:-}" ]] && continue - r="$(manifest_root_for_path "$fn")" - if [[ -z "$r" ]]; then - echo "submodulize: sync from unsubmodulized — commit $(git rev-parse --short "$U") touches $fn (outside manifest paths)." >&2 - exit 1 - fi - seen_roots[$r]=1 - done <<< "$changed" - - idx="$TMPD/index.${U}" - rm -f "$idx" - export GIT_INDEX_FILE="$idx" - git read-tree "$(git rev-parse HEAD)" - - for r in "${!seen_roots[@]}"; do - newsha="" - for j in "${!S_PATHS[@]}"; do - [[ "${S_PATHS[$j]}" == "$r" ]] || continue - pdir="${PDIRS[$j]}" - ms="$(git ls-tree "$U" -- "$r" 2>/dev/null | awk '{print $1 "\t" $3}' | head -n1)" - mode="${ms%%$'\t'*}" - if [[ "$mode" != "040000" ]]; then - echo "submodulize: sync from unsubmodulized — at $U path $r is not a vendored directory (mode $mode)." >&2 - exit 1 - fi - newsha="$(find_plugin_commit_for_tree_sync "$pdir" "$(git rev-parse "$U:$r" 2>/dev/null)")" || { - echo "submodulize: sync from unsubmodulized — no plugin commit matches tree at $U:$r (push the same tree to the plugin remote first)." >&2 - exit 1 - } - break - done - [[ -n "$newsha" ]] || continue - git rm -rf --cached --ignore-unmatch -q -- "$r" 2>/dev/null || true - git update-index --add --cacheinfo "160000,$newsha,$r" - done - - git_write_gitmodules_from_index - - new_tree="$(git write-tree)" - cur_tree="$(git rev-parse 'HEAD^{tree}')" - if [[ "$new_tree" == "$cur_tree" ]]; then - echo "submodulize: sync from unsubmodulized — $(git rev-parse --short "$U") → no gitlink change (skip)" >&2 - unset GIT_INDEX_FILE - rm -f "$idx" - continue - fi - - msgf="$TMPD/commitmsg.$U.txt" - { - git show -s --format=%B "$U" - printf '\nSynced-from-unsub: %s\n' "$U" - } >"$msgf" - nc="$(git commit-tree "$new_tree" -p HEAD -F "$msgf")" - git update-ref "refs/heads/$TARGET_BRANCH" "$nc" - git reset --hard -q "$nc" - echo "submodulize: sync from unsubmodulized — applied $(git rev-parse --short "$U") → $TARGET_BRANCH $(git rev-parse --short "$nc")" >&2 - - unset GIT_INDEX_FILE - rm -f "$idx" - done < <(git rev-list --reverse "${BASE}..${UNSUB_BRANCH}") - - unset GIT_INDEX_FILE - rm -rf "$TMPD" -} - -run() { - if $DRY_RUN; then - printf '[dry-run] %q\n' "$@" - else - "$@" - fi -} - -submodulize_one_shot_apply_manifest() { - manifest_entries=0 - while IFS='|' read -r raw_path raw_url raw_branch; do - path="${raw_path#"${raw_path%%[![:space:]]*}"}" - path="${path%"${path##*[![:space:]]}"}" - url="${raw_url#"${raw_url%%[![:space:]]*}"}" - url="${url%"${url##*[![:space:]]}"}" - branch="${raw_branch#"${raw_branch%%[![:space:]]*}"}" - branch="${branch%"${branch##*[![:space:]]}"}" - - [[ -z "$path" || "$path" =~ ^# ]] && continue - [[ -z "$url" ]] && { echo "Manifest: missing URL for path $path" >&2; exit 1; } - [[ -z "$branch" ]] && branch="main" - url="$(rewrite_github_url_to_ssh "$url")" - - ((++manifest_entries)) || true - - if [[ -f .gitmodules ]] && git config -f .gitmodules --get-regexp path 2>/dev/null | awk '{print $2}' | grep -Fxq "$path"; then - echo "Already a submodule (per .gitmodules): $path — skipping" - continue - fi - - if [[ -d "$path/.git" ]] || [[ -f "$path/.git" ]]; then - echo "Path already looks like a nested git repo: $path" >&2 - echo " Remove or convert it manually, or run unsubmodulize first." >&2 - exit 1 - fi - - if [[ -e "$path" ]] && ! $DRY_RUN; then - if ! git diff --quiet -- "$path" 2>/dev/null || ! git diff --cached --quiet -- "$path" 2>/dev/null; then - echo "Uncommitted changes under $path — commit or stash first." >&2 - exit 1 - fi - fi - - echo "Submodulizing: $path ← $url (branch $branch)" - - parent="$(dirname "$path")" - if [[ "$parent" != "." ]]; then - run mkdir -p "$parent" - fi - - if [[ -n "$(git ls-files -- "$path" 2>/dev/null)" ]]; then - if $DRY_RUN; then - printf '[dry-run] git rm -rf [--sparse] -- %q\n' "$path" - else - git rm -rf --sparse -- "$path" 2>/dev/null || git rm -rf -- "$path" - fi - elif [[ -e "$path" ]]; then - run rm -rf -- "$path" - fi - - if $DRY_RUN; then - printf '[dry-run] git submodule add -f (-b %q if exists on remote, else default branch) -- %q %q\n' "$branch" "$url" "$path" - else - submod_args=(-f) - if [[ -n "$branch" ]] && GIT_TERMINAL_PROMPT=0 git "${git_github_pat_c[@]}" ls-remote --heads "$url" "refs/heads/$branch" 2>/dev/null | grep -q .; then - submod_args+=(-b "$branch") - else - [[ -n "$branch" ]] && echo "Remote has no branch '$branch' for $path; using repository default branch." >&2 - fi - if ! GIT_TERMINAL_PROMPT=0 git "${git_github_pat_c[@]}" -c core.sparseCheckout=false -c index.sparse=false submodule add "${submod_args[@]}" -- "$url" "$path"; then - echo "submodulize: failed to add submodule $path ← $url" >&2 - echo " If the repo is private: configure HTTPS credentials, or re-run with --ssh (needs GitHub SSH access)." >&2 - echo " After a failed add you may need: git submodule deinit -f -- $path 2>/dev/null; rm -rf .git/modules/$path $path" >&2 - exit 1 - fi - fi - done < "$MANIFEST" - - if [[ "$manifest_entries" -eq 0 ]]; then - echo "No entries in manifest." >&2 - exit 1 - fi - - if $DRY_RUN; then - echo "Dry run complete." - submodulize_stage_root_manifest - $BOOTSTRAP && echo "Bootstrap: would commit on submodulized, then run unsubmodulize replay (omit --dry-run)." >&2 - exit 0 - fi - - submodulize_stage_root_manifest - - if ! $NO_COMMIT; then - if git diff --cached --quiet 2>/dev/null; then - echo "Nothing staged; skipping commit." - else - git commit -m "chore: add plugin submodules per plugin-submodules.manifest" - fi - fi - - if $BOOTSTRAP; then - : - else - echo "Done. Submodule layout is ready (clean repo)." - submodulize_hint_switch_from_submodule_branch - fi -} - -submodulize_bootstrap_pipeline() { - local vend_tip unsub_args - vend_tip="$(git rev-parse HEAD)" - echo "Bootstrap: vendored fork-point for unsub replay: $vend_tip" >&2 - - if git show-ref --verify --quiet refs/heads/submodulized && ! $FORCE_REPLAY; then - echo "Branch submodulized already exists. Use --force to replace it, delete the branch, or run without bootstrap." >&2 - exit 1 - fi - $FORCE_REPLAY && git branch -D submodulized 2>/dev/null || true - - git checkout -B submodulized - - # Unsub replay reads commit trees on --source; submodule adds must be committed so - # submodulized records 160000 gitlinks, not only the pre-submodule vendored tree. - local _saved_no_commit=false - if $NO_COMMIT; then - _saved_no_commit=true - echo "submodulize: bootstrap commits submodule layout on submodulized (required for unsub replay); ignoring --no-commit for this step." >&2 - NO_COMMIT=false - fi - submodulize_one_shot_apply_manifest - if $_saved_no_commit; then - NO_COMMIT=true - fi - - if $DRY_RUN; then - exit 0 - fi - - unsub_args=(--repo "$REPO_ROOT" --fork-point "$vend_tip" --source submodulized --target unsubmodulized) - $FORCE_REPLAY && unsub_args+=(--force) - - bash "$SCRIPT_DIR/unsubmodulize.sh" "${unsub_args[@]}" - echo "Bootstrap complete: submodulized (submodules) and unsubmodulized (vendored replay) are ready." >&2 - submodulize_hint_switch_from_submodule_branch -} - -submodulize_replay_mode() { - git rev-parse --verify "$FORK_POINT^{commit}" >/dev/null - git rev-parse --verify "$SOURCE_BRANCH^{commit}" >/dev/null - - if git show-ref --verify --quiet "refs/heads/$TARGET_BRANCH" && ! $FORCE_REPLAY; then - echo "Branch $TARGET_BRANCH exists; pass --force" >&2 - exit 1 - fi - - ls_path_mode_sha() { - git ls-tree "$1" -- "$2" 2>/dev/null | awk '{print $1 "\t" $3}' | head -n1 - } - - plugin_base_override_for() { - local want="$1" entry - for entry in "${PLUGIN_BASE_OVERRIDES[@]:-}"; do - [[ "${entry%%=*}" == "$want" ]] && { echo "${entry#*=}"; return 0; } - done - return 1 - } - - find_plugin_commit_for_tree() { - local pdir="$1" want_tree="$2" c t - while IFS= read -r c; do - t="$(git -C "$pdir" rev-parse "$c^{tree}" 2>/dev/null || true)" - [[ "$t" == "$want_tree" ]] && { echo "$c"; return 0; } - done < <(git -C "$pdir" rev-list --first-parent --reverse --all) - return 1 - } - - declare -a M_PATHS=() M_URLS=() M_BRANCHES=() - - while IFS='|' read -r raw_path raw_url raw_branch; do - path="${raw_path#"${raw_path%%[![:space:]]*}"}" - path="${path%"${path##*[![:space:]]}"}" - url="${raw_url#"${raw_url%%[![:space:]]*}"}" - url="${url%"${url##*[![:space:]]}"}" - branch="${raw_branch#"${raw_branch%%[![:space:]]*}"}" - branch="${branch%"${branch##*[![:space:]]}"}" - [[ -z "$path" || "$path" =~ ^# ]] && continue - [[ -z "$url" ]] && { echo "Manifest: missing URL for $path" >&2; exit 1; } - [[ -z "$branch" ]] && branch="main" - M_PATHS+=("$path") - M_URLS+=("$url") - M_BRANCHES+=("$branch") - done < "$MANIFEST" - - [[ ${#M_PATHS[@]} -gt 0 ]] || { echo "No manifest entries." >&2; exit 1; } - - SOURCE_TIP="$(git rev-parse "$SOURCE_BRANCH^{commit}")" - TMPD="$(mktemp -d "${TMPDIR:-/tmp}/sub-replay.XXXXXX")" - trap 'rm -rf "$TMPD"' EXIT - - declare -a EVENT_LINES=() - - for i in "${!M_PATHS[@]}"; do - P="${M_PATHS[$i]}" - URL="$(rewrite_github_url_to_ssh "${M_URLS[$i]}")" - PDIR="$TMPD/plugin_${i}_$(echo "$P" | tr '/' '_')" - - ms="$(ls_path_mode_sha "$FORK_POINT" "$P")" - mode="${ms%%$'\t'*}" - from_sha="${ms#*$'\t'}" - - ms2="$(ls_path_mode_sha "$SOURCE_TIP" "$P")" - to_sha="${ms2#*$'\t'}" - mode2="${ms2%%$'\t'*}" - - if [[ ! -d "$PDIR/.git" ]]; then - GIT_TERMINAL_PROMPT=0 git "${git_github_pat_c[@]}" clone --bare "$URL" "$PDIR" - fi - GIT_TERMINAL_PROMPT=0 git -C "$PDIR" fetch -q origin 2>/dev/null || true - GIT_TERMINAL_PROMPT=0 git -C "$PDIR" fetch -q "$URL" "+refs/*:refs/remotes/import/*" 2>/dev/null || true - - if [[ "$mode2" == "160000" && -n "$to_sha" ]]; then - : - elif [[ "$mode2" == "040000" ]]; then - tr_end="$(git rev-parse "$SOURCE_TIP:$P" 2>/dev/null || true)" - if [[ -z "$tr_end" ]]; then - echo "No tree at $SOURCE_BRANCH:$P (source tip)" >&2 - exit 1 - fi - to_sha="$(find_plugin_commit_for_tree "$PDIR" "$tr_end")" || { - echo "Could not match plugin commit to vendored tree at $SOURCE_BRANCH:$P" >&2 - exit 1 - } - else - echo "Path $P: source tip must be gitlink or vendored directory for end SHA" >&2 - exit 1 - fi - - if ! plugin_base_override_for "$P" >/dev/null; then - if [[ "$mode" == "160000" && -n "$from_sha" ]]; then - : - elif [[ "$mode" == "040000" ]]; then - tr_sha="$(git rev-parse "$FORK_POINT:$P" 2>/dev/null || true)" - if [[ -z "$tr_sha" ]]; then - echo "No tree at $FORK_POINT:$P" >&2 - exit 1 - fi - from_sha="$(find_plugin_commit_for_tree "$PDIR" "$tr_sha")" || { - echo "Could not find plugin commit matching tree at $FORK_POINT:$P" >&2 - exit 1 - } - else - echo "Path $P at fork-point must be gitlink or directory; use --plugin-base ${P}=SHA" >&2 - exit 1 - fi - else - from_sha="$(plugin_base_override_for "$P")" - fi - - GIT_TERMINAL_PROMPT=0 git -C "$PDIR" cat-file -e "${from_sha}^{commit}" 2>/dev/null || { - echo "Missing plugin object $from_sha for $P" >&2 - exit 1 - } - GIT_TERMINAL_PROMPT=0 git -C "$PDIR" cat-file -e "${to_sha}^{commit}" 2>/dev/null || { - echo "Missing plugin object $to_sha for $P" >&2 - exit 1 - } - - mapfile -t PCOMMITS < <(GIT_TERMINAL_PROMPT=0 git -C "$PDIR" rev-list --first-parent --reverse "${from_sha}..${to_sha}") - [[ ${#PCOMMITS[@]} -eq 0 ]] && continue - - for c in "${PCOMMITS[@]}"; do - ct="$(git -C "$PDIR" show -s --format=%ct "$c")" - EVENT_LINES+=("${ct}"$'\t'"${P}"$'\t'"${c}"$'\t'"${PDIR}"$'\t'"${M_URLS[$i]}") - done - done - - [[ ${#EVENT_LINES[@]} -gt 0 ]] || { echo "No plugin commits to replay." >&2; exit 0; } - - IFS=$'\n' - sorted="$(printf '%s\n' "${EVENT_LINES[@]}" | LC_ALL=C sort -t $'\t' -k1,1n -k2,2 -k3,3)" - unset IFS - - if $DRY_RUN; then - echo "Planned ${#EVENT_LINES[@]} commits on $TARGET_BRANCH (from $FORK_POINT)" - while IFS=$'\t' read -r ct p sha pdir url; do - [[ -z "${ct:-}" ]] && continue - echo " $ct $p $sha ($(git -C "$pdir" show -s --format=%s "$sha"))" - done <<< "$sorted" - exit 0 - fi - - $FORCE_REPLAY && git show-ref --verify --quiet "refs/heads/$TARGET_BRANCH" && git branch -D "$TARGET_BRANCH" 2>/dev/null || true - git worktree prune 2>/dev/null || true - - WT="$TMPD/wt" - rm -rf "$WT" - - manifest_url_for_path() { - local want="$1" _i - for _i in "${!M_PATHS[@]}"; do - if [[ "${M_PATHS[$_i]}" == "$want" ]]; then - echo "${M_URLS[$_i]}" - return 0 - fi - done - return 1 - } - - rebuild_gitmodules() { - rm -f "$WT/.gitmodules" - git -C "$WT" ls-files -s | while read -r mode sha _st path; do - [[ "$mode" == "160000" ]] || continue - u="$(manifest_url_for_path "$path")" || continue - printf '[submodule "%s"]\n\tpath = %s\n\turl = %s\n' "$path" "$path" "$u" >>"$WT/.gitmodules" - done - } - - GIT_TERMINAL_PROMPT=0 git "${git_github_pat_c[@]}" worktree add -B "$TARGET_BRANCH" "$WT" "$FORK_POINT" --force - - while IFS=$'\t' read -r ct P csha pdir url; do - [[ -z "${ct:-}" ]] && continue - git -C "$WT" reset --hard -q HEAD - git -C "$WT" rm -rf --cached --ignore-unmatch "$P" 2>/dev/null || true - rm -rf "${WT:?}/${P}" - git -C "$WT" update-index --add --cacheinfo "160000,$csha,$P" - rebuild_gitmodules - git -C "$WT" add -f .gitmodules 2>/dev/null || true - - an="$(git -C "$pdir" show -s --format=%an "$csha")" - ae="$(git -C "$pdir" show -s --format=%ae "$csha")" - adate="$(git -C "$pdir" show -s --format=%ai "$csha")" - body="$(git -C "$pdir" show -s --format=%B "$csha")" - { - printf '%s\n\n' "$body" - printf 'Replayed-from: %s\n' "$csha" - printf 'Plugin-path: %s\n' "$P" - printf 'gitlink: submodulize.sh (replay)\n' - } >"$TMPD/commitmsg.txt" - - GIT_AUTHOR_NAME="$an" GIT_AUTHOR_EMAIL="$ae" GIT_AUTHOR_DATE="$adate" \ - GIT_COMMITTER_NAME="$an" GIT_COMMITTER_EMAIL="$ae" GIT_COMMITTER_DATE="$adate" \ - git -C "$WT" commit -F "$TMPD/commitmsg.txt" - done <<< "$sorted" - - git -C "$REPO_ROOT" worktree remove -f "$WT" 2>/dev/null || true - echo "Done. Branch $TARGET_BRANCH -> $(git rev-parse "$TARGET_BRANCH")" -} - -if $BOOTSTRAP; then - submodulize_bootstrap_pipeline - exit 0 -fi - -# Default "submodulize ./" on an existing --target branch: extend layout from manifest (new submodule paths) -# instead of requiring --force. Full replay rebuild still needs explicit --fork-point/--source/--target intent + --force. -if $REPLAY && ! $FORCE_REPLAY && git show-ref --verify --quiet "refs/heads/$TARGET_BRANCH"; then - if ! $FORK_POINT_EXPLICIT && ! $REPLAY_SOURCE_EXPLICIT && ! $REPLAY_TARGET_EXPLICIT && ((${#PLUGIN_BASE_OVERRIDES[@]} == 0)); then - echo "submodulize: branch $TARGET_BRANCH exists — applying manifest updates (same as --no-replay). Use --force to replay-recreate $TARGET_BRANCH from --fork-point." >&2 - REPLAY=false - active_br="$(git symbolic-ref -q --short HEAD 2>/dev/null || true)" - if [[ "$active_br" != "$TARGET_BRANCH" ]]; then - echo "submodulize: checking out $TARGET_BRANCH" >&2 - git checkout "$TARGET_BRANCH" - fi - fi -fi - -if $REPLAY; then - submodulize_replay_mode - exit 0 -fi - -# After manifest one-shot: replay unsubmodulized → submodulized when unsub is ahead (or when --sync-unsub). -RUN_SYNC_FROM_UNSUB=false -if ! $NO_SYNC_FROM_UNSUB && ! $NO_COMMIT; then - if git show-ref --verify --quiet "refs/heads/$TARGET_BRANCH" && git show-ref --verify --quiet refs/heads/unsubmodulized; then - if $SYNC_UNSUB; then - RUN_SYNC_FROM_UNSUB=true - else - _mb="$(git merge-base unsubmodulized "$TARGET_BRANCH" 2>/dev/null || true)" - _ut="$(git rev-parse unsubmodulized^{commit} 2>/dev/null || true)" - if [[ -n "$_mb" && -n "$_ut" && "$_mb" != "$_ut" ]]; then - RUN_SYNC_FROM_UNSUB=true - fi - fi - fi -fi - -if $RUN_SYNC_FROM_UNSUB && ! $SYNC_UNSUB; then - echo "submodulize: unsubmodulized is ahead of merge-base with $TARGET_BRANCH — syncing plugin gitlinks." >&2 -fi - -if $RUN_SYNC_FROM_UNSUB; then - sb="$(git symbolic-ref -q --short HEAD 2>/dev/null || true)" - if [[ "$sb" != "$TARGET_BRANCH" ]]; then - echo "submodulize: checking out $TARGET_BRANCH (sync from unsubmodulized)" >&2 - git checkout "$TARGET_BRANCH" - fi - if [[ ! -f "$MANIFEST" ]]; then - if git cat-file -e "$TARGET_BRANCH:plugin-submodules.manifest" 2>/dev/null; then - git show "$TARGET_BRANCH:plugin-submodules.manifest" >"$MANIFEST" - echo "submodulize: wrote plugin-submodules.manifest from $TARGET_BRANCH (was missing at repo root)." >&2 - else - echo "Manifest not found: $MANIFEST (and not in $TARGET_BRANCH:plugin-submodules.manifest)" >&2 - exit 1 - fi - fi -fi - -submodulize_one_shot_apply_manifest - -if $RUN_SYNC_FROM_UNSUB; then - submodulize_sync_from_unsubmodulized -fi -exit 0 diff --git a/cleandev/tests/lib.sh b/cleandev/tests/lib.sh deleted file mode 100644 index b09f58c..0000000 --- a/cleandev/tests/lib.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env bash -# Shared helpers for cleandev/tests. Sourced by test-*.sh; do not run directly. -# shellcheck shell=bash - -fail() { - echo "FAIL: $*" >&2 - exit 1 -} - -ok() { - echo "ok: $*" -} - -assert_file() { - [[ -f "$1" ]] || fail "expected file missing: $1" -} - -assert_dir() { - [[ -d "$1" ]] || fail "expected directory missing: $1" -} - -assert_no_file() { - [[ ! -e "$1" ]] || fail "expected missing path exists: $1" -} - -# CLEANDEV must be set by run.sh -require_cleandev() { - [[ -n "${CLEANDEV:-}" && -d "$CLEANDEV" ]] || fail "CLEANDEV not set or not a directory" -} diff --git a/cleandev/tests/run.sh b/cleandev/tests/run.sh deleted file mode 100644 index cb7477e..0000000 --- a/cleandev/tests/run.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env bash -# Run all cleandev automated tests. Uses only temp dirs under /tmp; does not modify the workspace. -# -# Usage: bash cleandev/tests/run.sh -# or: cd cleandev/tests && ./run.sh -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -CLEANDEV="$(cd "$SCRIPT_DIR/.." && pwd)" -export CLEANDEV - -echo "CLEANDEV=$CLEANDEV" -echo "" - -failed=0 -for t in "$SCRIPT_DIR"/test-*.sh; do - [[ -f "$t" ]] || continue - echo "---------- $(basename "$t") ----------" - if bash "$t"; then - echo "" - else - echo "FAILED: $(basename "$t")" >&2 - failed=1 - echo "" - fi -done - -if [[ "$failed" -ne 0 ]]; then - echo "Some tests failed." >&2 - exit 1 -fi -echo "All tests passed." diff --git a/cleandev/tests/test-bootstrap.sh b/cleandev/tests/test-bootstrap.sh deleted file mode 100644 index aea5e90..0000000 --- a/cleandev/tests/test-bootstrap.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/usr/bin/env bash -# Bootstrap: submodulize.sh with no submodulized branch → one-shot + unsub replay (file:// plugin, no network). -set -euo pipefail -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -# shellcheck source=lib.sh -source "$SCRIPT_DIR/lib.sh" -require_cleandev - -TMP=$(mktemp -d) -export HOME="$TMP/_home" -mkdir -p "$HOME" -git config --global protocol.file.allow always -trap 'rm -rf "$TMP"' EXIT - -PLUGIN="$TMP/plugin-upstream" -MOODLE="$TMP/moodle" -mkdir -p "$PLUGIN" -git -C "$PLUGIN" init -b main -git -C "$PLUGIN" config core.autocrlf false -echo "plug=1" > "$PLUGIN/version.txt" -git -C "$PLUGIN" add version.txt -git -C "$PLUGIN" commit -q -m "init plugin" - -mkdir -p "$MOODLE/mod/testplugin" -git -C "$MOODLE" init -b master -git -C "$MOODLE" config core.autocrlf false -# Unpack from plugin so tree/blob IDs match plugin commits (avoids CRLF tree mismatch on Windows). -git -C "$PLUGIN" archive HEAD | tar -x -C "$MOODLE/mod/testplugin" -git -C "$MOODLE" add mod/testplugin -git -C "$MOODLE" commit -q -m "vendored plugin" - -printf '%s\n' 'mod/testplugin|../plugin-upstream|main' > "$MOODLE/plugin-submodules.manifest" - -# No local submodulized branch → auto-bootstrap (same as explicit --bootstrap) -"$CLEANDEV/submodulize.sh" --repo "$MOODLE" --no-commit - -git -C "$MOODLE" show-ref --verify --quiet refs/heads/submodulized || fail "expected submodulized branch" -git -C "$MOODLE" show-ref --verify --quiet refs/heads/unsubmodulized || fail "expected unsubmodulized branch" -git -C "$MOODLE" ls-tree submodulized mod/testplugin | grep -q '^160000' || fail "expected gitlink (submodule) at mod/testplugin on submodulized" -git -C "$MOODLE" ls-tree -r submodulized --name-only | grep -q '^plugin-submodules.manifest$' || fail "expected tracked plugin-submodules.manifest on submodulized" -git -C "$MOODLE" ls-tree -r unsubmodulized --name-only | grep -q '^mod/testplugin/version.txt$' || fail "expected vendored file on unsubmodulized" -if git -C "$MOODLE" ls-tree -r unsubmodulized --name-only | grep -q '^plugin-submodules.manifest$'; then - fail "did not expect plugin-submodules.manifest on unsubmodulized" -fi - -ok "submodulize bootstrap (auto when submodulized missing)" diff --git a/cleandev/tests/test-manifest-default-repo.sh b/cleandev/tests/test-manifest-default-repo.sh deleted file mode 100644 index ec930ee..0000000 --- a/cleandev/tests/test-manifest-default-repo.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env bash -# Default manifest path: ROOT/plugin-submodules.manifest when --manifest is omitted. -set -euo pipefail -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -# shellcheck source=lib.sh -source "$SCRIPT_DIR/lib.sh" -require_cleandev - -TMP=$(mktemp -d) -export HOME="$TMP/_home" -mkdir -p "$HOME" -trap 'rm -rf "$TMP"' EXIT - -MOODLE="$TMP/moodle" -mkdir -p "$MOODLE" -git -C "$MOODLE" init -b develop -q -echo "core" >"$MOODLE/README.txt" -git -C "$MOODLE" add README.txt -git -C "$MOODLE" commit -q -m "init" - -# No plugin-submodules.manifest at repo root → scripts fail -if "$CLEANDEV/submodulize.sh" --repo "$MOODLE" --dry-run 2>/dev/null; then - fail "expected submodulize to fail when default manifest is missing" -fi -if "$CLEANDEV/unsubmodulize.sh" --repo "$MOODLE" --dry-run 2>/dev/null; then - fail "expected unsubmodulize to fail when default manifest is missing" -fi - -set +e -sub_out=$("$CLEANDEV/submodulize.sh" --repo "$MOODLE" --dry-run 2>&1) -sub_ec=$? -red_out=$(bash "$CLEANDEV/manifest-submodulize-redundant.sh" --repo "$MOODLE" 2>&1) -red_ec=$? -set -e -[[ "$sub_ec" -ne 0 ]] || fail "expected submodulize nonzero exit" -[[ "$red_ec" -ne 0 ]] || fail "expected manifest-submodulize-redundant nonzero exit" -echo "$sub_out" | grep -Fq "Manifest not found" || fail "expected submodulize stderr to mention Manifest not found" -echo "$red_out" | grep -Fq "Manifest not found" || fail "expected redundant stderr to mention Manifest not found" - -ok "default manifest path requires ROOT/plugin-submodules.manifest when --manifest omitted" diff --git a/cleandev/tests/test-manifest-lint.sh b/cleandev/tests/test-manifest-lint.sh deleted file mode 100644 index 5a94a78..0000000 --- a/cleandev/tests/test-manifest-lint.sh +++ /dev/null @@ -1,86 +0,0 @@ -#!/usr/bin/env bash -# Validate plugin-submodules.manifest: format, duplicate paths, monorepo-style duplicate URLs. -# Limitation: two active lines with the same URL but different branches still count as duplicate URL (intentional). -# Does not validate that URLs are reachable or that paths exist on disk. -# Portable: no associative arrays (works on macOS /bin/bash 3.2 if needed for local runs). -set -euo pipefail -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -# shellcheck source=lib.sh -source "$SCRIPT_DIR/lib.sh" -require_cleandev - -MANIFEST="${1:-$CLEANDEV/plugin-submodules.manifest}" -[[ -f "$MANIFEST" ]] || fail "manifest not found: $MANIFEST" - -line_num=0 -active_count=0 -seen_paths="" - -url_exists() { - local u="$1" - local entry - while IFS= read -r entry; do - [[ "$entry" == "$u" ]] && return 0 - done <<< "${url_list:-}" - return 1 -} - -url_record() { - local u="$1" - url_list="${url_list:-}${url_list:+$'\n'}$u" -} - -path_exists_in_list() { - local p="$1" - local x - while IFS= read -r x; do - [[ "$x" == "$p" ]] && return 0 - done <<< "${seen_paths:-}" - return 1 -} - -url_list="" - -while IFS= read -r raw || [[ -n "$raw" ]]; do - line_num=$((line_num + 1)) - line="${raw//$'\r'/}" - trimmed="${line#"${line%%[![:space:]]*}"}" - trimmed="${trimmed%"${trimmed##*[![:space:]]}"}" - [[ -z "$trimmed" ]] && continue - [[ "$trimmed" =~ ^# ]] && continue - - if [[ "$trimmed" != *'|'* ]]; then - fail "line $line_num: expected path|url|branch (no pipe): $trimmed" - fi - - path="${trimmed%%|*}" - rest="${trimmed#*|}" - url="${rest%%|*}" - # Optional third field (branch) is ignored here; scripts default it. - - path="${path#"${path%%[![:space:]]*}"}" - path="${path%"${path##*[![:space:]]}"}" - url="${url#"${url%%[![:space:]]*}"}" - url="${url%"${url##*[![:space:]]}"}" - - [[ -n "$path" ]] || fail "line $line_num: empty path" - [[ -n "$url" ]] || fail "line $line_num: empty url for path $path" - - pipe_count=$(printf '%s' "$trimmed" | tr -cd '|' | wc -c | tr -d ' ') - [[ "$pipe_count" -le 2 ]] || fail "line $line_num: too many '|' fields (max 2): $trimmed" - - if path_exists_in_list "$path"; then - fail "duplicate manifest path '$path' (line $line_num)" - fi - seen_paths="${seen_paths:-}${seen_paths:+$'\n'}$path" - active_count=$((active_count + 1)) - - if url_exists "$url"; then - fail "monorepo-style duplicate clone URL for two active paths (submodulize cannot map 1:1): url='$url' (line $line_num)" - fi - url_record "$url" -done < "$MANIFEST" - -[[ "$active_count" -gt 0 ]] || fail "no active manifest lines (only comments/empty)" - -ok "manifest lint: $active_count active lines, $MANIFEST" diff --git a/cleandev/tests/test-manifest-submod-redundant.sh b/cleandev/tests/test-manifest-submod-redundant.sh deleted file mode 100644 index aca33ba..0000000 --- a/cleandev/tests/test-manifest-submod-redundant.sh +++ /dev/null @@ -1,119 +0,0 @@ -#!/usr/bin/env bash -# Unit tests for cleandev/manifest-submodulize-redundant.sh (new.sh --submodulize skip logic). -set -euo pipefail -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -# shellcheck source=lib.sh -source "$SCRIPT_DIR/lib.sh" -require_cleandev - -REDUNDANT_SH="$CLEANDEV/manifest-submodulize-redundant.sh" -assert_file "$REDUNDANT_SH" - -run_redundant() { - local repo="$1" - local man="${2:-}" - if [[ -n "$man" ]]; then - bash "$REDUNDANT_SH" --repo "$repo" --manifest "$man" - else - bash "$REDUNDANT_SH" --repo "$repo" - fi -} - -write_gitmodules() { - local repo="$1" - shift - local pair - : >"$repo/.gitmodules" - for pair in "$@"; do - local spath="${pair%%=*}" - local url="${pair#*=}" - { - printf '[submodule "%s"]\n' "$spath" - printf '\tpath = %s\n' "$spath" - printf '\turl = %s\n' "$url" - } >>"$repo/.gitmodules" - done -} - -expect_redundant() { - local repo="$1" - local man="${2:-}" - if run_redundant "$repo" "$man"; then - return 0 - fi - fail "expected exit 0 (submodulize redundant), got 1 — repo=$repo manifest=${man:-}" -} - -expect_not_redundant() { - local repo="$1" - local man="${2:-}" - if ! run_redundant "$repo" "$man"; then - return 0 - fi - fail "expected exit 1 (submodulize needed), got 0 — repo=$repo manifest=${man:-}" -} - -TMP_BASE="$(mktemp -d "${TMPDIR:-/tmp}/manifest-submod-redundant.XXXXXX")" -trap 'rm -rf "$TMP_BASE"' EXIT - -# --- all manifest paths in .gitmodules → redundant (exit 0) --- -R1="$TMP_BASE/r1" -mkdir -p "$R1" -git -C "$R1" init -q -write_gitmodules "$R1" \ - "mod/foo=https://example.com/foo.git" \ - "blocks/bar=https://example.com/bar.git" -M1="$TMP_BASE/m1.manifest" -{ - printf '# comment\n' - printf 'mod/foo|https://example.com/foo.git|main\n' - printf ' blocks/bar |https://example.com/bar.git|main\n' -} >"$M1" -expect_redundant "$R1" "$M1" - -# --- one path missing from .gitmodules → not redundant --- -R2="$TMP_BASE/r2" -mkdir -p "$R2" -git -C "$R2" init -q -write_gitmodules "$R2" "mod/foo=https://example.com/foo.git" -M2="$TMP_BASE/m2.manifest" -printf 'mod/foo|u|main\nmod/missing|u|main\n' >"$M2" -expect_not_redundant "$R2" "$M2" - -# --- no .gitmodules → not redundant --- -R3="$TMP_BASE/r3" -mkdir -p "$R3" -git -C "$R3" init -q -M3="$TMP_BASE/m3.manifest" -printf 'mod/foo|u|main\n' >"$M3" -expect_not_redundant "$R3" "$M3" - -# --- empty .gitmodules → not redundant ([ -s .gitmodules ] false) --- -R4="$TMP_BASE/r4" -mkdir -p "$R4" -git -C "$R4" init -q -: >"$R4/.gitmodules" -M4="$TMP_BASE/m4.manifest" -printf 'mod/foo|u|main\n' >"$M4" -expect_not_redundant "$R4" "$M4" - -# --- manifest only comments/blank → not redundant (no -s .gitmodules match with vacuous need_submod=0) --- -R5="$TMP_BASE/r5" -mkdir -p "$R5" -git -C "$R5" init -q -write_gitmodules "$R5" "mod/foo=https://example.com/foo.git" -M5="$TMP_BASE/m5.manifest" -printf '# only comments\n\n# foo\n' >"$M5" -expect_not_redundant "$R5" "$M5" - -# --- default manifest at ROOT/plugin-submodules.manifest (no --manifest flag) --- -R6="$TMP_BASE/r6" -mkdir -p "$R6" -git -C "$R6" init -q -write_gitmodules "$R6" \ - "mod/foo=https://example.com/foo.git" \ - "blocks/bar=https://example.com/bar.git" -printf 'mod/foo|https://example.com/foo.git|main\nblocks/bar|https://example.com/bar.git|main\n' >"$R6/plugin-submodules.manifest" -expect_redundant "$R6" - -ok "manifest-submodulize-redundant.sh (skip vs run submodulize)" diff --git a/cleandev/tests/test-pat-parity.sh b/cleandev/tests/test-pat-parity.sh deleted file mode 100644 index 5d4ca74..0000000 --- a/cleandev/tests/test-pat-parity.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env bash -# Regression guard: source still contains the PAT git -c wiring (substring match). -# This does not prove HTTPS auth works at runtime; integration with GitHub needs manual/CI secrets tests. -# shellcheck disable=SC2016 # grep -F patterns intentionally contain literal ${git_github_pat_c[@]} -set -euo pipefail -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -# shellcheck source=lib.sh -source "$SCRIPT_DIR/lib.sh" -require_cleandev - -SUB="$CLEANDEV/submodulize.sh" -UNS="$CLEANDEV/unsubmodulize.sh" - -grep -Fq 'git_github_pat_c=()' "$SUB" || fail "submodulize: missing git_github_pat_c init" -grep -Fq 'git "${git_github_pat_c[@]}" ls-remote' "$SUB" || fail "submodulize: ls-remote missing PAT -c array" -grep -Fq 'git "${git_github_pat_c[@]}" -c core.sparseCheckout=false' "$SUB" || fail "submodulize: submodule add missing PAT -c array" - -grep -Fq 'git_github_pat_c=()' "$UNS" || fail "unsubmodulize: missing git_github_pat_c init" -grep -Fq 'git "${git_github_pat_c[@]}" ls-remote' "$UNS" || fail "unsubmodulize: ls-remote missing PAT -c array" -grep -Fq 'git "${git_github_pat_c[@]}" clone' "$UNS" || fail "unsubmodulize: clone missing PAT -c array" - -ok "PAT -c parity present in submodulize.sh and unsubmodulize.sh" diff --git a/cleandev/tests/test-replay-both.sh b/cleandev/tests/test-replay-both.sh deleted file mode 100644 index de177de..0000000 --- a/cleandev/tests/test-replay-both.sh +++ /dev/null @@ -1,70 +0,0 @@ -#!/usr/bin/env bash -# Replay mode (default): unsubmodulize + submodulize on temp repos (file:// plugin, no network). -set -euo pipefail -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -# shellcheck source=lib.sh -source "$SCRIPT_DIR/lib.sh" -require_cleandev - -TMP=$(mktemp -d) -export HOME="$TMP/_home" -mkdir -p "$HOME" -git config --global protocol.file.allow always -trap 'rm -rf "$TMP"' EXIT - -PLUGIN="$TMP/plugin" -MOODLE="$TMP/moodle" -mkdir -p "$PLUGIN" "$MOODLE" - -git -C "$PLUGIN" init -b main -echo "v1" > "$PLUGIN/f.txt" -git -C "$PLUGIN" add f.txt -git -C "$PLUGIN" commit -q -m "p1" -echo "v2" >> "$PLUGIN/f.txt" -git -C "$PLUGIN" commit -q -am "p2" - -C0="$(git -C "$PLUGIN" rev-parse HEAD~1)" -C1="$(git -C "$PLUGIN" rev-parse HEAD)" - -git -C "$MOODLE" init -b main -printf 'core\n' > "$MOODLE/README.md" -mkdir -p "$MOODLE/mod/foo" -git -C "$PLUGIN" archive "$C0" | tar -x -C "$MOODLE/mod/foo" -git -C "$MOODLE" add README.md mod/foo -git -C "$MOODLE" commit -q -m "vendored base" - -BASE="$(git -C "$MOODLE" rev-parse HEAD)" - -git -C "$MOODLE" config -f .gitmodules submodule.mod/foo.path mod/foo -git -C "$MOODLE" config -f .gitmodules submodule.mod/foo.url "$PLUGIN" -git -C "$MOODLE" add .gitmodules -git -C "$MOODLE" rm -rf --cached mod/foo -git -C "$MOODLE" update-index --add --cacheinfo "160000,$C0,mod/foo" -git -C "$MOODLE" commit -q -m "sub at c0" - -git -C "$MOODLE" checkout -q -b submodulized -git -C "$MOODLE" update-index --add --cacheinfo "160000,$C1,mod/foo" -git -C "$MOODLE" commit -q -m "sub at c1" - -MANIFEST="$MOODLE/plugin-submodules.manifest" -printf 'mod/foo|%s|main\n' "$PLUGIN" > "$MANIFEST" - -# --- unsub replay: one commit (c1) in plugin range c0..c1 (explicit plugin base avoids CRLF tree mismatch) --- -"$CLEANDEV/unsubmodulize.sh" --repo "$MOODLE" --fork-point "$BASE" --source submodulized \ - --target unsubmodulized --force --plugin-base "mod/foo=$C0" - -N="$(git -C "$MOODLE" rev-list --count "${BASE}..unsubmodulized" 2>/dev/null || echo 0)" -[[ "$N" -eq 1 ]] || fail "expected 1 commit on unsubmodulized past BASE, got $N" -git -C "$MOODLE" ls-tree -r unsubmodulized --name-only | grep -q '^mod/foo/f.txt$' || fail "expected vendored file path" -git -C "$MOODLE" show unsubmodulized:mod/foo/f.txt | grep -q v2 || fail "expected v2 in tree" - -# --- sub replay from vendored BASE: match tree at BASE to plugin c0 --- -git -C "$MOODLE" checkout -q main -"$CLEANDEV/submodulize.sh" --repo "$MOODLE" --fork-point "$BASE" --source submodulized \ - --target sub2 --force --plugin-base "mod/foo=$C0" - -N2="$(git -C "$MOODLE" rev-list --count "${BASE}..sub2" 2>/dev/null || echo 0)" -[[ "$N2" -eq 1 ]] || fail "expected 1 commit on sub2 past BASE, got $N2" -git -C "$MOODLE" ls-tree sub2 mod/foo | grep -q 160000 || fail "expected gitlink at mod/foo" - -ok "unsubmodulize + submodulize replay (fixture)" diff --git a/cleandev/tests/test-shellcheck.sh b/cleandev/tests/test-shellcheck.sh deleted file mode 100644 index 9154d55..0000000 --- a/cleandev/tests/test-shellcheck.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env bash -# Optional: shellcheck cleandev scripts when shellcheck is installed. -set -euo pipefail -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -# shellcheck source=lib.sh -source "$SCRIPT_DIR/lib.sh" -CLEANDEV="${CLEANDEV:-$(cd "$SCRIPT_DIR/.." && pwd)}" -export CLEANDEV -require_cleandev - -if ! command -v shellcheck >/dev/null 2>&1; then - echo "SKIP shellcheck (not installed)" - exit 0 -fi - -# Run from this directory so `source=lib.sh` directives resolve (shellcheck -x). -cd "$SCRIPT_DIR" -shellcheck -x lib.sh run.sh test-*.sh ../submodulize.sh ../unsubmodulize.sh ../manifest-submodulize-redundant.sh -ok "shellcheck" diff --git a/cleandev/tests/test-sparse-submod.sh b/cleandev/tests/test-sparse-submod.sh deleted file mode 100644 index 3463948..0000000 --- a/cleandev/tests/test-sparse-submod.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env bash -# submodulize still works when the superproject had sparse-checkout enabled. -set -euo pipefail -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -# shellcheck source=lib.sh -source "$SCRIPT_DIR/lib.sh" -require_cleandev - -TMP=$(mktemp -d) -export HOME="$TMP/_home" -mkdir -p "$HOME" -git config --global protocol.file.allow always -trap 'rm -rf "$TMP"' EXIT - -PLUGIN="$TMP/plugin-upstream" -MOODLE="$TMP/moodle" -mkdir -p "$PLUGIN" "$MOODLE" -git -C "$PLUGIN" init -b main -echo "p" > "$PLUGIN/a.txt" -git -C "$PLUGIN" add a.txt -git -C "$PLUGIN" commit -q -m "i" - -git -C "$MOODLE" init -b develop -echo "core" > "$MOODLE/README.txt" -mkdir -p "$MOODLE/mod/testplugin" -echo "v" > "$MOODLE/mod/testplugin/x.txt" -git -C "$MOODLE" add README.txt mod/testplugin -git -C "$MOODLE" commit -q -m "init" - -# Cone mode expects directory paths; include whole `mod/` so `mod/testplugin` is on disk. -git -C "$MOODLE" sparse-checkout init --cone -git -C "$MOODLE" sparse-checkout set mod - -printf '%s\n' 'mod/testplugin|../plugin-upstream|main' > "$MOODLE/plugin-submodules.manifest" - -"$CLEANDEV/submodulize.sh" --no-replay --repo "$MOODLE" --no-commit - -assert_file "$MOODLE/mod/testplugin/a.txt" -# submodulize disables the same flags it clears in disable_sparse_checkout_if_needed() -[[ "$(git -C "$MOODLE" config --bool core.sparseCheckout 2>/dev/null || echo false)" != "true" ]] \ - || fail "expected core.sparseCheckout not true after submodulize" -[[ "$(git -C "$MOODLE" config --bool index.sparse 2>/dev/null || echo false)" != "true" ]] \ - || fail "expected index.sparse not true after submodulize" - -ok "submodulize with prior sparse-checkout" diff --git a/cleandev/tests/test-submod-roundtrip.sh b/cleandev/tests/test-submod-roundtrip.sh deleted file mode 100644 index ffdee20..0000000 --- a/cleandev/tests/test-submod-roundtrip.sh +++ /dev/null @@ -1,60 +0,0 @@ -#!/usr/bin/env bash -# Vendored plugin -> submodulize -> unsubmodulize round-trip in isolated temp repos (no network). -set -euo pipefail -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -# shellcheck source=lib.sh -source "$SCRIPT_DIR/lib.sh" -require_cleandev - -TMP=$(mktemp -d) -export HOME="$TMP/_home" -mkdir -p "$HOME" -# Submodule clone resolves sibling paths via file://; allow only inside this test process. -git config --global protocol.file.allow always -trap 'rm -rf "$TMP"' EXIT - -PLUGIN="$TMP/plugin-upstream" -MOODLE="$TMP/moodle" -mkdir -p "$PLUGIN" -git -C "$PLUGIN" init -b main -echo "plug=1" > "$PLUGIN/version.txt" -git -C "$PLUGIN" add version.txt -git -C "$PLUGIN" commit -q -m "init plugin" - -mkdir -p "$MOODLE/mod/testplugin" -git -C "$MOODLE" init -b develop -echo "vendored-only" > "$MOODLE/mod/testplugin/local.txt" -git -C "$MOODLE" add mod/testplugin -git -C "$MOODLE" commit -q -m "vendored plugin" - -printf '%s\n' 'mod/testplugin|../plugin-upstream|main' > "$MOODLE/plugin-submodules.manifest" - -# --- dry-run must not convert --- -"$CLEANDEV/submodulize.sh" --no-replay --repo "$MOODLE" --dry-run -assert_file "$MOODLE/mod/testplugin/local.txt" -assert_no_file "$MOODLE/.gitmodules" -assert_no_file "$MOODLE/.git/modules/mod/testplugin" - -# --- submodulize --- -"$CLEANDEV/submodulize.sh" --no-replay --repo "$MOODLE" --no-commit -assert_file "$MOODLE/.gitmodules" -assert_file "$MOODLE/mod/testplugin/version.txt" -assert_no_file "$MOODLE/mod/testplugin/local.txt" -[[ -f "$MOODLE/mod/testplugin/.git" || -d "$MOODLE/mod/testplugin/.git" ]] || fail "expected submodule gitlink" -git -C "$MOODLE" submodule status --recursive | grep -q 'mod/testplugin' || fail "expected mod/testplugin in submodule status" - -# --- unsubmodulize --- -"$CLEANDEV/unsubmodulize.sh" --no-replay --repo "$MOODLE" --no-commit -assert_file "$MOODLE/mod/testplugin/version.txt" -assert_no_file "$MOODLE/mod/testplugin/.git" -# Git may leave an empty .gitmodules; ensure no submodule entries remain. -if [[ -f "$MOODLE/.gitmodules" ]] && git -C "$MOODLE" config -f .gitmodules --get-regexp path 2>/dev/null | grep -q .; then - fail "expected no [submodule] entries in .gitmodules after unsubmodulize" -fi -# unsubmodulize stages vendored files; ensure superproject tracks them (not just loose files on disk). -git -C "$MOODLE" ls-files --error-unmatch mod/testplugin/version.txt >/dev/null 2>&1 \ - || fail "expected mod/testplugin/version.txt in git index after unsubmodulize" -! git -C "$MOODLE" submodule status 2>/dev/null | grep -q 'mod/testplugin' \ - || fail "expected mod/testplugin not to remain a submodule after unsubmodulize" - -ok "submodulize <-> unsubmodulize round-trip (default ROOT/plugin-submodules.manifest)" diff --git a/cleandev/tests/test-submodulize-extend-manifest.sh b/cleandev/tests/test-submodulize-extend-manifest.sh deleted file mode 100644 index 7756b58..0000000 --- a/cleandev/tests/test-submodulize-extend-manifest.sh +++ /dev/null @@ -1,62 +0,0 @@ -#!/usr/bin/env bash -# With submodulized present, default submodulize (replay) applies manifest without --force; new paths become submodules. -set -euo pipefail -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -# shellcheck source=lib.sh -source "$SCRIPT_DIR/lib.sh" -require_cleandev - -TMP=$(mktemp -d) -export HOME="$TMP/_home" -mkdir -p "$HOME" -git config --global protocol.file.allow always -trap 'rm -rf "$TMP"' EXIT - -PLUGIN1="$TMP/plugin1" -PLUGIN2="$TMP/plugin2" -MOODLE="$TMP/moodle" - -mkdir -p "$PLUGIN1" -git -C "$PLUGIN1" init -b main -git -C "$PLUGIN1" config core.autocrlf false -echo "one" > "$PLUGIN1/a.txt" -git -C "$PLUGIN1" add a.txt -git -C "$PLUGIN1" commit -q -m "p1" - -mkdir -p "$PLUGIN2" -git -C "$PLUGIN2" init -b main -git -C "$PLUGIN2" config core.autocrlf false -echo "two" > "$PLUGIN2/b.txt" -git -C "$PLUGIN2" add b.txt -git -C "$PLUGIN2" commit -q -m "p2" - -mkdir -p "$MOODLE/mod/p1" -git -C "$MOODLE" init -b master -git -C "$MOODLE" config core.autocrlf false -git -C "$PLUGIN1" archive HEAD | tar -x -C "$MOODLE/mod/p1" -git -C "$MOODLE" add mod/p1 -git -C "$MOODLE" commit -q -m "vendored p1" - -printf '%s\n' "mod/p1|../plugin1|main" > "$MOODLE/plugin-submodules.manifest" - -"$CLEANDEV/submodulize.sh" --repo "$MOODLE" - -# Second run: no new manifest lines — should not require --force (one-shot no-op / skips). -"$CLEANDEV/submodulize.sh" --repo "$MOODLE" - -git -C "$MOODLE" checkout -q submodulized -mkdir -p "$MOODLE/mod/p2" -git -C "$PLUGIN2" archive HEAD | tar -x -C "$MOODLE/mod/p2" -git -C "$MOODLE" add mod/p2 -git -C "$MOODLE" commit -q -m "vendored p2 for submod" - -{ - echo "mod/p1|../plugin1|main" - echo "mod/p2|../plugin2|main" -} > "$MOODLE/plugin-submodules.manifest" - -"$CLEANDEV/submodulize.sh" --repo "$MOODLE" - -git -C "$MOODLE" ls-tree submodulized mod/p2 | grep -q '^160000' || fail "expected gitlink at mod/p2 after manifest extend" - -ok "submodulize extends submodulized from manifest without --force" diff --git a/cleandev/tests/test-sync-unsub.sh b/cleandev/tests/test-sync-unsub.sh deleted file mode 100644 index 82a2977..0000000 --- a/cleandev/tests/test-sync-unsub.sh +++ /dev/null @@ -1,119 +0,0 @@ -#!/usr/bin/env bash -# Auto sync: when unsubmodulized is ahead of merge-base with submodulized, submodulize replays onto gitlinks. -set -euo pipefail -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -# shellcheck source=lib.sh -source "$SCRIPT_DIR/lib.sh" -require_cleandev - -TMP=$(mktemp -d) -export HOME="$TMP/_home" -mkdir -p "$HOME" -git config --global protocol.file.allow always -trap 'rm -rf "$TMP"' EXIT - -PLUGIN="$TMP/plugin" -MOODLE="$TMP/moodle" -mkdir -p "$PLUGIN" "$MOODLE" - -git -C "$PLUGIN" init -b main -echo "v1" > "$PLUGIN/f.txt" -git -C "$PLUGIN" add f.txt -git -C "$PLUGIN" commit -q -m "p1" -echo "v2" >> "$PLUGIN/f.txt" -git -C "$PLUGIN" commit -q -am "p2" - -C0="$(git -C "$PLUGIN" rev-parse HEAD~1)" -C1="$(git -C "$PLUGIN" rev-parse HEAD)" - -git -C "$MOODLE" init -b main -printf 'core\n' > "$MOODLE/README.md" -mkdir -p "$MOODLE/mod/foo" -git -C "$PLUGIN" archive "$C0" | tar -x -C "$MOODLE/mod/foo" -git -C "$MOODLE" add README.md mod/foo -git -C "$MOODLE" commit -q -m "vendored base" - -M0="$(git -C "$MOODLE" rev-parse HEAD)" - -git -C "$MOODLE" checkout -q -b submodulized -printf 'mod/foo|%s|main\n' "$PLUGIN" > "$MOODLE/plugin-submodules.manifest" -git -C "$MOODLE" config -f .gitmodules submodule.mod/foo.path mod/foo -git -C "$MOODLE" config -f .gitmodules submodule.mod/foo.url "$PLUGIN" -git -C "$MOODLE" add .gitmodules plugin-submodules.manifest -git -C "$MOODLE" rm -rf --cached mod/foo -git -C "$MOODLE" update-index --add --cacheinfo "160000,$C0,mod/foo" -git -C "$MOODLE" commit -q -m "sub at c0" -# Manual gitlink commit can leave vendored files on disk (untracked); remove before branch hops. -git -C "$MOODLE" reset --hard -q HEAD -rm -rf "$MOODLE/mod/foo" - -git -C "$MOODLE" branch -q unsubmodulized "$M0" - -git -C "$MOODLE" checkout -q unsubmodulized -find "$MOODLE/mod/foo" -mindepth 1 -delete -git -C "$PLUGIN" archive "$C1" | tar -x -C "$MOODLE/mod/foo" -git -C "$MOODLE" add mod/foo -git -C "$MOODLE" commit -q -m "vendor bump to c1" - -# Stay on unsubmodulized; default one-shot path checks out submodulized and syncs when unsub is ahead. -"$CLEANDEV/submodulize.sh" --repo "$MOODLE" - -link="$(git -C "$MOODLE" ls-tree submodulized mod/foo | awk '{print $3}')" -[[ "$link" == "$C1" ]] || fail "expected gitlink $C1 on submodulized, got ${link:-empty}" - -git -C "$MOODLE" log -1 --format=%B submodulized | grep -q '^Synced-from-unsub:' || fail "expected Synced-from-unsub footer" - -ok "submodulize auto sync from unsubmodulized (fixture)" - -# --- opt-out: same vendor bump but --no-sync-from-unsub leaves gitlink at C0 --- -TMP2=$(mktemp -d) -export HOME="$TMP2/_home" -mkdir -p "$HOME" -git config --global protocol.file.allow always -trap 'rm -rf "$TMP" "$TMP2"' EXIT - -PLUGIN2="$TMP2/plugin" -MOODLE2="$TMP2/moodle" -mkdir -p "$PLUGIN2" "$MOODLE2" - -git -C "$PLUGIN2" init -b main -echo "v1" > "$PLUGIN2/f.txt" -git -C "$PLUGIN2" add f.txt -git -C "$PLUGIN2" commit -q -m "p1" -echo "v2" >> "$PLUGIN2/f.txt" -git -C "$PLUGIN2" commit -q -am "p2" -C0b="$(git -C "$PLUGIN2" rev-parse HEAD~1)" -C1b="$(git -C "$PLUGIN2" rev-parse HEAD)" - -git -C "$MOODLE2" init -b main -printf 'core\n' > "$MOODLE2/README.md" -mkdir -p "$MOODLE2/mod/foo" -git -C "$PLUGIN2" archive "$C0b" | tar -x -C "$MOODLE2/mod/foo" -git -C "$MOODLE2" add README.md mod/foo -git -C "$MOODLE2" commit -q -m "vendored base" -M0b="$(git -C "$MOODLE2" rev-parse HEAD)" - -git -C "$MOODLE2" checkout -q -b submodulized -printf 'mod/foo|%s|main\n' "$PLUGIN2" > "$MOODLE2/plugin-submodules.manifest" -git -C "$MOODLE2" config -f .gitmodules submodule.mod/foo.path mod/foo -git -C "$MOODLE2" config -f .gitmodules submodule.mod/foo.url "$PLUGIN2" -git -C "$MOODLE2" add .gitmodules plugin-submodules.manifest -git -C "$MOODLE2" rm -rf --cached mod/foo -git -C "$MOODLE2" update-index --add --cacheinfo "160000,$C0b,mod/foo" -git -C "$MOODLE2" commit -q -m "sub at c0" -git -C "$MOODLE2" reset --hard -q HEAD -rm -rf "$MOODLE2/mod/foo" - -git -C "$MOODLE2" branch -q unsubmodulized "$M0b" -git -C "$MOODLE2" checkout -q unsubmodulized -find "$MOODLE2/mod/foo" -mindepth 1 -delete -git -C "$PLUGIN2" archive "$C1b" | tar -x -C "$MOODLE2/mod/foo" -git -C "$MOODLE2" add mod/foo -git -C "$MOODLE2" commit -q -m "vendor bump to c1" - -"$CLEANDEV/submodulize.sh" --repo "$MOODLE2" --no-sync-from-unsub - -linkb="$(git -C "$MOODLE2" ls-tree submodulized mod/foo | awk '{print $3}')" -[[ "$linkb" == "$C0b" ]] || fail "expected gitlink unchanged at $C0b with --no-sync-from-unsub, got ${linkb:-empty}" - -ok "submodulize --no-sync-from-unsub skips sync (fixture)" diff --git a/cleandev/tests/test-unsub-incremental.sh b/cleandev/tests/test-unsub-incremental.sh deleted file mode 100644 index ad1718e..0000000 --- a/cleandev/tests/test-unsub-incremental.sh +++ /dev/null @@ -1,86 +0,0 @@ -#!/usr/bin/env bash -# Incremental unsub replay: default --source submodulized; no --force when unsubmodulized exists. -set -euo pipefail -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -# shellcheck source=lib.sh -source "$SCRIPT_DIR/lib.sh" -require_cleandev - -TMP=$(mktemp -d) -export HOME="$TMP/_home" -mkdir -p "$HOME" -git config --global protocol.file.allow always -trap 'rm -rf "$TMP"' EXIT - -PLUGIN="$TMP/plugin" -MOODLE="$TMP/moodle" -mkdir -p "$PLUGIN" "$MOODLE" - -git -C "$PLUGIN" init -b main -git -C "$PLUGIN" config core.autocrlf false -echo "v1" > "$PLUGIN/f.txt" -git -C "$PLUGIN" add f.txt -git -C "$PLUGIN" commit -q -m "p1" -echo "v2" >> "$PLUGIN/f.txt" -git -C "$PLUGIN" commit -q -am "p2" -echo "v3" >> "$PLUGIN/f.txt" -git -C "$PLUGIN" commit -q -am "p3" - -C0="$(git -C "$PLUGIN" rev-parse HEAD~2)" -C1="$(git -C "$PLUGIN" rev-parse HEAD~1)" -C2="$(git -C "$PLUGIN" rev-parse HEAD)" - -git -C "$MOODLE" init -b main -printf 'core\n' > "$MOODLE/README.md" -mkdir -p "$MOODLE/mod/foo" -git -C "$PLUGIN" archive "$C0" | tar -x -C "$MOODLE/mod/foo" -git -C "$MOODLE" add README.md mod/foo -git -C "$MOODLE" commit -q -m "vendored base" - -BASE="$(git -C "$MOODLE" rev-parse HEAD)" - -git -C "$MOODLE" config -f .gitmodules submodule.mod/foo.path mod/foo -git -C "$MOODLE" config -f .gitmodules submodule.mod/foo.url "$PLUGIN" -git -C "$MOODLE" add .gitmodules -git -C "$MOODLE" rm -rf --cached mod/foo -git -C "$MOODLE" update-index --add --cacheinfo "160000,$C0,mod/foo" -git -C "$MOODLE" commit -q -m "sub at c0" - -git -C "$MOODLE" checkout -q -b submodulized -git -C "$MOODLE" update-index --add --cacheinfo "160000,$C1,mod/foo" -git -C "$MOODLE" commit -q -m "sub at c1" - -MANIFEST="$MOODLE/plugin-submodules.manifest" -printf 'mod/foo|%s|main\n' "$PLUGIN" > "$MANIFEST" - -# Initial build (force ok); submodulized tip is still C1 so replay is c0..c1 only. -"$CLEANDEV/unsubmodulize.sh" --repo "$MOODLE" --fork-point "$BASE" --source submodulized \ - --target unsubmodulized --force --plugin-base "mod/foo=$C0" - -N1="$(git -C "$MOODLE" rev-list --count "${BASE}..unsubmodulized" 2>/dev/null || echo 0)" -[[ "$N1" -eq 1 ]] || fail "expected 1 replay commit past BASE after first unsub, got $N1" -git -C "$MOODLE" show "unsubmodulized:mod/foo/f.txt" | grep -q v2 || fail "expected v2 after first unsub" -if git -C "$MOODLE" show "unsubmodulized:mod/foo/f.txt" | grep -q v3; then - fail "did not expect v3 after first unsub (only c0..c1 replayed)" -fi - -# Advance submodulized to C2, then incremental unsub (no --force). -git -C "$MOODLE" checkout -q submodulized -git -C "$MOODLE" update-index --add --cacheinfo "160000,$C2,mod/foo" -git -C "$MOODLE" commit -q -m "sub at c2" - -"$CLEANDEV/unsubmodulize.sh" --repo "$MOODLE" --fork-point "$BASE" --source submodulized \ - --target unsubmodulized - -N2="$(git -C "$MOODLE" rev-list --count "${BASE}..unsubmodulized" 2>/dev/null || echo 0)" -[[ "$N2" -eq 2 ]] || fail "expected 2 replay commits past BASE after incremental unsub, got $N2" -git -C "$MOODLE" show "unsubmodulized:mod/foo/f.txt" | grep -q v3 || fail "expected v3 after incremental unsub" - -# Idempotent -"$CLEANDEV/unsubmodulize.sh" --repo "$MOODLE" --fork-point "$BASE" --source submodulized \ - --target unsubmodulized - -N3="$(git -C "$MOODLE" rev-list --count "${BASE}..unsubmodulized" 2>/dev/null || echo 0)" -[[ "$N3" -eq 2 ]] || fail "expected still 2 commits past BASE when already up to date, got $N3" - -ok "unsubmodulize incremental replay without --force" diff --git a/cleandev/unsubmodulize.sh b/cleandev/unsubmodulize.sh deleted file mode 100644 index eaea80c..0000000 --- a/cleandev/unsubmodulize.sh +++ /dev/null @@ -1,649 +0,0 @@ -#!/usr/bin/env bash -# Replace git submodules with plain tracked directories (vendored plugins), matching lsuce-moodle -# develop style. For the submodule-layout Moodle superproject: manifest and .gitmodules live in that -# repo; this script reads plugin-submodules.manifest at the superproject root unless --manifest is set. -# Clones each plugin repo at the given branch, drops nested .git, and adds files to the parent. -# Run from inside the clone, or pass the clone path as ROOT / --repo. -# Default mode is replay: builds branch unsubmodulized with one superproject commit per plugin-repo commit. -# Use --no-replay for one-shot conversion (manifest loop only). -# -# Usage: -# ./cleandev/unsubmodulize.sh [ROOT] [--dry-run] [--no-commit] [--ssh] [--manifest PATH] [--repo ROOT] -# ./cleandev/unsubmodulize.sh --fork-point BASE ... # replay (default) -# ./cleandev/unsubmodulize.sh --no-replay ... # one-shot -# Bare ROOT is the same as --repo (optional; may appear before or after flags). -# -# Private GitHub repos over HTTPS: set GITHUB_TOKEN (PAT) so ls-remote / clone use -# -c url.https://TOKEN@github.com/.insteadOf=https://github.com/ -# (same as submodulize.sh). Or use --ssh. -# -# Requires bash (arrays, pipefail). Do not run as `sh this-script.sh`; use `bash` or execute directly. - -if [ -z "${BASH_VERSION:-}" ]; then - printf '%s: requires bash, not sh. Example: bash "%s" ./ ...your args...\n' "${0##*/}" "$0" >&2 - exit 1 -fi - -set -euo pipefail - -MANIFEST="" -MANIFEST_EXPLICIT=false -DRY_RUN=false -NO_COMMIT=false -USE_SSH=false -REPO_ROOT="" -REPLAY=true -FORK_POINT="" -SOURCE_BRANCH="" -TARGET_BRANCH="unsubmodulized" -REPLAY_ORDER="chronological" -FORCE_REPLAY=false -REPLAY_SOURCE_EXPLICIT=false -REPLAY_TARGET_EXPLICIT=false -REPLAY_ORDER_EXPLICIT=false -declare -a PLUGIN_BASE_OVERRIDES=() - -usage() { - cat <<'EOF' -Replace git submodules listed in the manifest with plain tracked directories (vendored plugins). -For each entry: clones the plugin at the manifest branch, removes nested .git, deinits the submodule, -and stages the tree in the parent Moodle superproject (cleandev-style). The manifest normally lives -at the superproject root as plugin-submodules.manifest. - -Moodle root defaults to the current directory’s git superproject (git rev-parse --show-toplevel), unless you set it explicitly. - -Usage: - unsubmodulize.sh [ROOT] [OPTIONS...] - unsubmodulize.sh [OPTIONS...] [ROOT] - - ROOT — optional path to the Moodle git checkout. Give it as a single bare argument (no leading -), - anywhere among the flags; same meaning as --repo. Only one repo path: do not pass a second - bare path, and do not put a bare path after --repo (that is rejected). - -Examples: - unsubmodulize.sh --no-replay ~/workspace/moodle - unsubmodulize.sh --dry-run --no-replay ~/workspace/moodle - unsubmodulize.sh --fork-point abc123 --target unsubmodulized - -Options: - --dry-run Print what would happen without changing the repo - --no-commit Stage vendored trees but do not commit - --ssh Use git@github.com URLs for github.com HTTPS entries - --manifest PATH Plugin manifest (default: ROOT/plugin-submodules.manifest) - --repo ROOT Moodle git root (explicit form of a bare ROOT; overrides an earlier bare ROOT; a bare path after --repo is an error) - -Replay (default — one superproject commit per plugin-repo commit on --target): - --no-replay One-shot mode: convert submodules per manifest (no branch replay) - --replay Replay mode (default; explicit if you toggled --no-replay earlier on the command line) - --fork-point BASE Superproject where replay starts (default: local master, else main, when omitted — see docs) - --source BR Submodule/vendored tip to replay from (default: submodulized, else unsubmodulized, else master, else main) - --target BR Branch to create/update (default: unsubmodulized) - --order NAME Only chronological (committer date, then path, then SHA) - --force Replay only: delete and rebuild --target from --fork-point (omit for incremental update when --target exists) - --plugin-base P=S Optional start SHA for manifest path P (e.g. when BASE has no gitlink) - -Requires: git, and clean submodule working trees for paths being converted (no uncommitted changes inside submodules). - -Private GitHub over HTTPS: set GITHUB_TOKEN (PAT) so ls-remote / clone can authenticate (same pattern as submodulize.sh), -or use --ssh. -EOF -} - -while [[ $# -gt 0 ]]; do - case "$1" in - --dry-run) DRY_RUN=true; shift ;; - --no-commit) NO_COMMIT=true; shift ;; - --ssh) USE_SSH=true; shift ;; - --no-replay) REPLAY=false; shift ;; - --replay) REPLAY=true; shift ;; - --fork-point) - FORK_POINT="${2:?}" - shift 2 - ;; - --source) - SOURCE_BRANCH="${2:?}" - REPLAY_SOURCE_EXPLICIT=true - shift 2 - ;; - --target) - TARGET_BRANCH="${2:?}" - REPLAY_TARGET_EXPLICIT=true - shift 2 - ;; - --order) - REPLAY_ORDER="${2:?}" - REPLAY_ORDER_EXPLICIT=true - shift 2 - ;; - --force) - FORCE_REPLAY=true - shift - ;; - --plugin-base) - PLUGIN_BASE_OVERRIDES+=("${2:?}") - shift 2 - ;; - --manifest) - MANIFEST="${2:?}" - MANIFEST_EXPLICIT=true - shift 2 - ;; - --repo) - REPO_ROOT="${2:?}" - shift 2 - ;; - -h|--help) - usage - exit 0 - ;; - -*) - echo "Unknown option: $1" >&2 - exit 1 - ;; - *) - if [[ -n "$REPO_ROOT" ]]; then - echo "Unexpected argument: $1 (repo already set via --repo or positional path)" >&2 - exit 1 - fi - REPO_ROOT="$1" - shift - ;; - esac -done - -if [[ -z "$REPO_ROOT" ]]; then - REPO_ROOT="$(git rev-parse --show-toplevel 2>/dev/null)" || { - echo "Not inside a git repository. Pass the Moodle root as a bare path or use --repo /path/to/moodle" >&2 - exit 1 - } -fi - -if ! $MANIFEST_EXPLICIT; then - MANIFEST="${REPO_ROOT%/}/plugin-submodules.manifest" -fi - -if [[ ! -f "$MANIFEST" ]]; then - echo "Manifest not found: $MANIFEST" >&2 - exit 1 -fi - -if $REPLAY; then - [[ "$REPLAY_ORDER" == "chronological" ]] || { - echo "Unsupported --order (only chronological): $REPLAY_ORDER" >&2 - exit 1 - } -else - [[ -z "$FORK_POINT" ]] || { - echo "--fork-point is only used in replay mode (omit --no-replay)" >&2 - exit 1 - } - ((${#PLUGIN_BASE_OVERRIDES[@]} == 0)) || { - echo "--plugin-base is only valid in replay mode (omit --no-replay)" >&2 - exit 1 - } - $FORCE_REPLAY && { - echo "--force is only valid in replay mode (omit --no-replay)" >&2 - exit 1 - } - $REPLAY_SOURCE_EXPLICIT && { - echo "--source is only valid in replay mode (omit --no-replay)" >&2 - exit 1 - } - $REPLAY_TARGET_EXPLICIT && { - echo "--target is only valid in replay mode (omit --no-replay)" >&2 - exit 1 - } - $REPLAY_ORDER_EXPLICIT && { - echo "--order is only valid in replay mode (omit --no-replay)" >&2 - exit 1 - } -fi - -cd "$REPO_ROOT" - -# When --fork-point is omitted in replay mode, default to local master (else main) if safe; see resolve below. -if $REPLAY && [[ -z "$FORK_POINT" ]]; then - base_ref="" - if git show-ref --verify --quiet refs/heads/master; then - base_ref=master - elif git show-ref --verify --quiet refs/heads/main; then - base_ref=main - else - echo "Replay needs --fork-point (no local master or main branch to use as default)." >&2 - exit 1 - fi - head_sha="$(git rev-parse HEAD)" - master_sha="$(git rev-parse "${base_ref}^{commit}")" - target_exists=false - git show-ref --verify --quiet "refs/heads/$TARGET_BRANCH" && target_exists=true - if ! $target_exists; then - FORK_POINT="$base_ref" - echo "Default --fork-point $FORK_POINT ($TARGET_BRANCH does not exist yet)." >&2 - else - target_sha="$(git rev-parse "$TARGET_BRANCH^{commit}")" - if [[ "$head_sha" == "$master_sha" ]] || [[ "$head_sha" == "$target_sha" ]]; then - FORK_POINT="$base_ref" - echo "Default --fork-point $FORK_POINT (HEAD is ${base_ref} or $TARGET_BRANCH)." >&2 - else - FORK_POINT="$base_ref" - echo "Default --fork-point $FORK_POINT ($TARGET_BRANCH exists: incremental replay without --force, or --force to rebuild from ${base_ref})." >&2 - fi - fi -fi - -# When --source is omitted in replay mode: prefer submodulized (gitlink tip) for replaying into unsubmodulized. -if $REPLAY && ! $REPLAY_SOURCE_EXPLICIT; then - if git show-ref --verify --quiet refs/heads/submodulized; then - SOURCE_BRANCH=submodulized - elif git show-ref --verify --quiet "refs/heads/unsubmodulized"; then - SOURCE_BRANCH=unsubmodulized - elif git show-ref --verify --quiet refs/heads/master; then - SOURCE_BRANCH=master - elif git show-ref --verify --quiet refs/heads/main; then - SOURCE_BRANCH=main - else - echo "Replay: pass --source BR (no local submodulized, unsubmodulized, master, or main branch found)." >&2 - exit 1 - fi - echo "Default --source $SOURCE_BRANCH" >&2 -fi - -disable_sparse_checkout_if_needed() { - $DRY_RUN && return 0 - local active= - [[ -f .git/info/sparse-checkout ]] && active=1 - [[ "$(git config --bool core.sparseCheckout 2>/dev/null)" == "true" ]] && active=1 - [[ "$(git config --bool index.sparse 2>/dev/null)" == "true" ]] && active=1 - if [[ -z "$active" ]] && command -v git >/dev/null; then - local listed - listed="$(git sparse-checkout list 2>/dev/null | head -n 1 || true)" - [[ -n "${listed// }" ]] && active=1 - fi - [[ -z "$active" ]] && return 0 - echo "Disabling sparse-checkout so plugin paths can be vendored." >&2 - git sparse-checkout disable 2>/dev/null || true - git config core.sparseCheckout false 2>/dev/null || true - git config --unset-all core.sparseCheckoutCone 2>/dev/null || true - git config index.sparse false 2>/dev/null || true - rm -f .git/info/sparse-checkout -} - -disable_sparse_checkout_if_needed - -# Extra -c flags so clone honors GitHub PAT (superproject local url.insteadOf is not always used here). -git_github_pat_c=() -if [[ -n "${GITHUB_TOKEN:-}" ]]; then - git_github_pat_c+=(-c "url.https://${GITHUB_TOKEN}@github.com/.insteadOf=https://github.com/") -fi - -rewrite_github_url_to_ssh() { - local u="$1" - if $USE_SSH && [[ "$u" == https://github.com/* ]]; then - printf '%s\n' "git@github.com:${u#https://github.com/}" - else - printf '%s\n' "$u" - fi -} - -unsubmodulize_replay_mode() { - git rev-parse --verify "$FORK_POINT^{commit}" >/dev/null - git rev-parse --verify "$SOURCE_BRANCH^{commit}" >/dev/null - - ls_path_mode_sha() { - local commit="$1" path="$2" - git ls-tree "$commit" -- "$path" 2>/dev/null | awk '{print $1 "\t" $3}' | head -n1 - } - - plugin_base_override_for() { - local want="$1" - local entry - for entry in "${PLUGIN_BASE_OVERRIDES[@]:-}"; do - if [[ "${entry%%=*}" == "$want" ]]; then - printf '%s\n' "${entry#*=}" - return 0 - fi - done - return 1 - } - - find_plugin_commit_for_tree() { - local pdir="$1" want_tree="$2" c t - while IFS= read -r c; do - t="$(git -C "$pdir" rev-parse "$c^{tree}" 2>/dev/null || true)" - [[ "$t" == "$want_tree" ]] && { echo "$c"; return 0; } - done < <(git -C "$pdir" rev-list --first-parent --reverse --all) - return 1 - } - - declare -a M_PATHS=() - declare -a M_URLS=() - declare -a M_BRANCHES=() - - while IFS='|' read -r raw_path raw_url raw_branch; do - path="${raw_path#"${raw_path%%[![:space:]]*}"}" - path="${path%"${path##*[![:space:]]}"}" - url="${raw_url#"${raw_url%%[![:space:]]*}"}" - url="${url%"${url##*[![:space:]]}"}" - branch="${raw_branch#"${raw_branch%%[![:space:]]*}"}" - branch="${branch%"${branch##*[![:space:]]}"}" - [[ -z "$path" || "$path" =~ ^# ]] && continue - [[ -z "$url" ]] && { echo "Manifest: missing URL for path $path" >&2; exit 1; } - [[ -z "$branch" ]] && branch="main" - M_PATHS+=("$path") - M_URLS+=("$url") - M_BRANCHES+=("$branch") - done < "$MANIFEST" - - if [[ ${#M_PATHS[@]} -eq 0 ]]; then - echo "No manifest entries." >&2 - exit 1 - fi - - SOURCE_TIP="$(git rev-parse "$SOURCE_BRANCH^{commit}")" - - INCREMENTAL=false - FROM_REF="$FORK_POINT" - if git show-ref --verify --quiet "refs/heads/$TARGET_BRANCH" && ! $FORCE_REPLAY; then - INCREMENTAL=true - FROM_REF="$(git rev-parse "$TARGET_BRANCH^{commit}")" - for _xi in "${!M_PATHS[@]}"; do - _xp="${M_PATHS[_xi]}" - _ms="$(ls_path_mode_sha "$FROM_REF" "$_xp")" - _mode="${_ms%%$'\t'*}" - if [[ "$_mode" != "040000" ]]; then - echo "unsubmodulize: $TARGET_BRANCH at $_xp is not a vendored directory (tree mode $_mode)." >&2 - echo " Use --force to delete $TARGET_BRANCH and rebuild from --fork-point $FORK_POINT." >&2 - exit 1 - fi - done - echo "unsubmodulize: incremental update of $TARGET_BRANCH from $(git rev-parse --short "$FROM_REF") toward $SOURCE_BRANCH." >&2 - fi - - TMPD="$(mktemp -d "${TMPDIR:-/tmp}/unsub-replay.XXXXXX")" - trap 'rm -rf "$TMPD"' EXIT - - declare -a EVENT_LINES=() - - for i in "${!M_PATHS[@]}"; do - P="${M_PATHS[$i]}" - URL="${M_URLS[$i]}" - URL="$(rewrite_github_url_to_ssh "$URL")" - - ms="$(ls_path_mode_sha "$FROM_REF" "$P")" - mode="${ms%%$'\t'*}" - from_sha="${ms#*$'\t'}" - - ms2="$(ls_path_mode_sha "$SOURCE_TIP" "$P")" - mode2="${ms2%%$'\t'*}" - to_sha="${ms2#*$'\t'}" - - PDIR="$TMPD/plugin_${i}_$(echo "$P" | tr '/' '_')" - if [[ ! -d "$PDIR/.git" ]]; then - GIT_TERMINAL_PROMPT=0 git "${git_github_pat_c[@]}" clone --bare "$URL" "$PDIR" - fi - - GIT_TERMINAL_PROMPT=0 git -C "$PDIR" fetch -q origin 2>/dev/null || true - GIT_TERMINAL_PROMPT=0 git -C "$PDIR" fetch -q "$URL" "+refs/*:refs/remotes/import/*" 2>/dev/null || true - - if [[ "$mode2" == "160000" && -n "$to_sha" ]]; then - : - elif [[ "$mode2" == "040000" ]]; then - tr_end="$(git rev-parse "$SOURCE_TIP:$P" 2>/dev/null || true)" - if [[ -z "$tr_end" ]]; then - echo "No tree at $SOURCE_BRANCH:$P (source tip)" >&2 - exit 1 - fi - to_sha="$(find_plugin_commit_for_tree "$PDIR" "$tr_end")" || { - echo "Could not match plugin commit to vendored tree at $SOURCE_BRANCH:$P" >&2 - exit 1 - } - else - echo "Path $P at $SOURCE_BRANCH tip must be gitlink or vendored directory — cannot determine end SHA" >&2 - exit 1 - fi - - if plugin_base_override_for "$P" >/dev/null; then - from_sha="$(plugin_base_override_for "$P")" - elif [[ "$mode" == "160000" && -n "$from_sha" ]]; then - : - elif [[ "$mode" == "040000" ]]; then - tr_sha="$(git rev-parse "$FROM_REF:$P" 2>/dev/null || true)" - if [[ -z "$tr_sha" ]]; then - echo "No tree at $FROM_REF:$P" >&2 - exit 1 - fi - from_sha="$(find_plugin_commit_for_tree "$PDIR" "$tr_sha")" || { - echo "Could not find plugin commit matching tree at $FROM_REF:$P" >&2 - exit 1 - } - else - echo "Path $P at replay base $FROM_REF must be gitlink or vendored directory; use --plugin-base ${P}=SHA" >&2 - exit 1 - fi - - if ! GIT_TERMINAL_PROMPT=0 git -C "$PDIR" cat-file -e "${from_sha}^{commit}" 2>/dev/null; then - echo "Plugin object $from_sha not found for path $P (fetch the remote?)" >&2 - exit 1 - fi - if ! GIT_TERMINAL_PROMPT=0 git -C "$PDIR" cat-file -e "${to_sha}^{commit}" 2>/dev/null; then - echo "Plugin object $to_sha not found for path $P" >&2 - exit 1 - fi - - mapfile -t PCOMMITS < <(GIT_TERMINAL_PROMPT=0 git -C "$PDIR" rev-list --first-parent --reverse "${from_sha}..${to_sha}") - - if [[ ${#PCOMMITS[@]} -eq 0 ]]; then - echo "No plugin commits in range ${from_sha}..${to_sha} for $P (already at tip?)" >&2 - continue - fi - - for c in "${PCOMMITS[@]}"; do - ct="$(git -C "$PDIR" show -s --format=%ct "$c")" - EVENT_LINES+=("${ct}"$'\t'"${P}"$'\t'"${c}"$'\t'"${PDIR}") - done - done - - if [[ ${#EVENT_LINES[@]} -eq 0 ]]; then - echo "No plugin commits to replay." >&2 - if $DRY_RUN; then - if $INCREMENTAL; then - echo "Planned: no new commits ($TARGET_BRANCH already matches $SOURCE_BRANCH submodule tips)." >&2 - else - echo "Planned: branch $TARGET_BRANCH at $FORK_POINT (fork already matches submodule plugin SHAs)" >&2 - fi - exit 0 - fi - if $INCREMENTAL; then - echo "Branch $TARGET_BRANCH is already up to date with $SOURCE_BRANCH." >&2 - exit 0 - fi - if $FORCE_REPLAY && git show-ref --verify --quiet "refs/heads/$TARGET_BRANCH"; then - git branch -D "$TARGET_BRANCH" 2>/dev/null || true - fi - git branch "$TARGET_BRANCH" "$FORK_POINT" - echo "Done. Branch $TARGET_BRANCH -> $(git rev-parse "$TARGET_BRANCH")" - exit 0 - fi - - IFS=$'\n' - sorted="$(printf '%s\n' "${EVENT_LINES[@]}" | LC_ALL=C sort -t $'\t' -k1,1n -k2,2 -k3,3)" - unset IFS - - if $DRY_RUN; then - if $INCREMENTAL; then - echo "Planned ${#EVENT_LINES[@]} incremental commit(s) on $TARGET_BRANCH ($(git rev-parse --short "$FROM_REF") → $SOURCE_BRANCH)" >&2 - else - echo "Planned ${#EVENT_LINES[@]} commits on $TARGET_BRANCH (from $FORK_POINT)" >&2 - fi - while IFS=$'\t' read -r ct p sha pdir; do - [[ -z "${ct:-}" ]] && continue - echo " $ct $p $sha ($(git -C "$pdir" show -s --format=%s "$sha"))" - done <<< "$sorted" - exit 0 - fi - - if $FORCE_REPLAY && git show-ref --verify --quiet "refs/heads/$TARGET_BRANCH"; then - git branch -D "$TARGET_BRANCH" 2>/dev/null || true - fi - git worktree prune 2>/dev/null || true - - WT="$TMPD/wt" - rm -rf "$WT" - REPLAY_ROOT="$FORK_POINT" - REPLAY_ROOT_REV="$(git rev-parse "$FORK_POINT^{commit}")" - if $INCREMENTAL; then - REPLAY_ROOT="$FROM_REF" - REPLAY_ROOT_REV="$(git rev-parse "$FROM_REF^{commit}")" - fi - - materialize_commit_at() { - local commit="$1" dest="$2" - rm -rf "${dest:?}/"* - mkdir -p "$dest" - git archive --format=tar "$commit" | tar -x -C "$dest" - local _i _P _ms _mode _sha _PDIR - for _i in "${!M_PATHS[@]}"; do - _P="${M_PATHS[$_i]}" - _ms="$(ls_path_mode_sha "$commit" "$_P")" - _mode="${_ms%%$'\t'*}" - _sha="${_ms#*$'\t'}" - if [[ "$_mode" == "160000" && -n "$_sha" ]]; then - _PDIR="$TMPD/plugin_${_i}_$(echo "$_P" | tr '/' '_')" - rm -rf "$dest/$_P" - mkdir -p "$dest/$_P" - git -C "$_PDIR" archive --format=tar "$_sha" | tar -x -C "$dest/$_P" - fi - done - rm -f "$dest/.gitmodules" - } - - fill_worktree_from_parent() { - local parent="$1" dest="$2" - if [[ "$(git rev-parse "$parent^{commit}")" == "$REPLAY_ROOT_REV" ]]; then - materialize_commit_at "$REPLAY_ROOT" "$dest" - else - rm -rf "${dest:?}/"* - mkdir -p "$dest" - git archive --format=tar "$parent" | tar -x -C "$dest" - fi - } - - if $INCREMENTAL; then - GIT_TERMINAL_PROMPT=0 git "${git_github_pat_c[@]}" worktree add -f "$WT" "$TARGET_BRANCH" - else - GIT_TERMINAL_PROMPT=0 git "${git_github_pat_c[@]}" worktree add -B "$TARGET_BRANCH" "$WT" "$FORK_POINT" --force - fi - - while IFS=$'\t' read -r ct P csha pdir; do - [[ -z "${ct:-}" ]] && continue - cur="$(git -C "$WT" rev-parse HEAD)" - fill_worktree_from_parent "$cur" "$WT" - rm -rf "${WT:?}/${P}" - mkdir -p "${WT}/${P}" - git -C "$pdir" archive --format=tar "$csha" | tar -x -C "${WT}/${P}" - rm -f "$WT/.gitmodules" - - an="$(git -C "$pdir" show -s --format=%an "$csha")" - ae="$(git -C "$pdir" show -s --format=%ae "$csha")" - adate="$(git -C "$pdir" show -s --format=%ai "$csha")" - body="$(git -C "$pdir" show -s --format=%B "$csha")" - { - printf '%s\n\n' "$body" - printf 'Replayed-from: %s\n' "$csha" - printf 'Plugin-path: %s\n' "$P" - } >"$TMPD/commitmsg.txt" - - git -C "$WT" add -A - GIT_AUTHOR_NAME="$an" GIT_AUTHOR_EMAIL="$ae" GIT_AUTHOR_DATE="$adate" \ - GIT_COMMITTER_NAME="$an" GIT_COMMITTER_EMAIL="$ae" GIT_COMMITTER_DATE="$adate" \ - git -C "$WT" commit -F "$TMPD/commitmsg.txt" - done <<< "$sorted" - - git -C "$REPO_ROOT" worktree remove -f "$WT" 2>/dev/null || true - echo "Done. Branch $TARGET_BRANCH -> $(git rev-parse "$TARGET_BRANCH")" -} - -if $REPLAY; then - unsubmodulize_replay_mode - exit 0 -fi - -manifest_entries=0 -while IFS='|' read -r raw_path raw_url raw_branch; do - path="${raw_path#"${raw_path%%[![:space:]]*}"}" - path="${path%"${path##*[![:space:]]}"}" - url="${raw_url#"${raw_url%%[![:space:]]*}"}" - url="${url%"${url##*[![:space:]]}"}" - branch="${raw_branch#"${raw_branch%%[![:space:]]*}"}" - branch="${branch%"${branch##*[![:space:]]}"}" - - [[ -z "$path" || "$path" =~ ^# ]] && continue - [[ -z "$url" ]] && { echo "Manifest: missing URL for path $path" >&2; exit 1; } - [[ -z "$branch" ]] && branch="main" - url="$(rewrite_github_url_to_ssh "$url")" - - ((++manifest_entries)) || true - - if [[ ! -f .gitmodules ]] || ! git config -f .gitmodules --get-regexp path 2>/dev/null | awk '{print $2}' | grep -Fxq "$path"; then - echo "Not listed as submodule in .gitmodules: $path — skipping (already vendored or unknown)" - continue - fi - - if ! $DRY_RUN; then - if [[ -d "$path" ]] && (cd "$path" && git status --porcelain 2>/dev/null | grep -q .); then - echo "Submodule $path has local changes — commit/push inside submodule or stash first." >&2 - exit 1 - fi - fi - - echo "Unsubmodulizing: $path (from $url @ $branch)" - - if $DRY_RUN; then - printf '[dry-run] would: clone %s @ %s → %s, deinit submodule, rm .git, git add\n' "$url" "$branch" "$path" - continue - fi - - tmp="$(mktemp -d "${TMPDIR:-/tmp}/unsubmodulize.XXXXXX")" - trap 'rm -rf "$tmp"' EXIT - if [[ -n "$branch" ]] && GIT_TERMINAL_PROMPT=0 git "${git_github_pat_c[@]}" ls-remote --heads "$url" "refs/heads/$branch" 2>/dev/null | grep -q .; then - GIT_TERMINAL_PROMPT=0 git "${git_github_pat_c[@]}" clone --depth 1 -b "$branch" -- "$url" "$tmp/clone" - else - [[ -n "$branch" ]] && echo "Remote has no branch '$branch' for $path; cloning default branch." >&2 - GIT_TERMINAL_PROMPT=0 git "${git_github_pat_c[@]}" clone --depth 1 -- "$url" "$tmp/clone" - fi - git submodule deinit -f -- "$path" - git rm -f --sparse -- "$path" 2>/dev/null || git rm -f -- "$path" - mod_gitdir="$(git rev-parse --git-path "modules/$path")" - if [[ -n "$mod_gitdir" && -e "$mod_gitdir" ]]; then - rm -rf -- "$mod_gitdir" - fi - parent="$(dirname "$path")" - [[ "$parent" != "." ]] && mkdir -p "$parent" - rm -rf -- "$path" - cp -a "$tmp/clone/." "$path/" - rm -rf -- "$path/.git" - trap - EXIT - rm -rf "$tmp" - - git -c core.sparseCheckout=false -c index.sparse=false add -- "$path" -done < "$MANIFEST" - -if [[ "$manifest_entries" -eq 0 ]]; then - echo "No entries in manifest." >&2 - exit 1 -fi - -if $DRY_RUN; then - echo "Dry run complete." - exit 0 -fi - -if ! $NO_COMMIT; then - if git diff --cached --quiet 2>/dev/null; then - echo "Nothing staged; skipping commit." - else - git commit -m "chore: vendor plugin trees (remove submodules per manifest)" - fi -fi - -echo "Done. Plugin directories are plain files (dirty / upstream Moodle style)." diff --git a/docker-compose.yml b/docker-compose.yml index 9484f5a..7023f5d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -18,8 +18,13 @@ services: # Build from ./Dockerfile (php:8.3-apache). lsuonline/moodle-dev:latest has shipped PHP 8.4, which # breaks Moodle 4.5 CLI install: "Cannot compile closures which import variables using the use keyword". # To use a pre-built image instead, remove `build:` below and set `image: your-registry/moodle-dev:tag`. - build: . - image: lsuce-moodle-web:local + # `secrets:` forwards GITHUB_TOKEN to the build so the Dockerfile can clone the private + # lsuonline/moodleus repo. new.sh resolves+exports GITHUB_TOKEN before `docker compose up`. + build: + context: . + secrets: + - github_token + image: lsuonline/moodle-dev:latest container_name: ${BRANCH_NAME}-moodle depends_on: db: @@ -88,4 +93,8 @@ services: networks: traefik: - external: true \ No newline at end of file + external: true + +secrets: + github_token: + environment: GITHUB_TOKEN \ No newline at end of file diff --git a/new.sh b/new.sh index b430821..12821ff 100755 --- a/new.sh +++ b/new.sh @@ -88,7 +88,7 @@ ensure_traefik_running() { # Usage: set_config [component] name value # - If component is empty or "-", sets a core config resolve_submod_github_token() { - local github_token_file="${PROJECT_ROOT}/cleandev/.github-token" + local github_token_file="${PROJECT_ROOT}/submodulizer-local/.github-token" SUBMOD_GIT_TOKEN="" if [ "${SUBMODULIZE_SSH:-}" = "1" ]; then return 0 @@ -109,20 +109,20 @@ resolve_submod_github_token() { fi if [ -t 0 ] && [ -t 1 ]; then echo "GitHub personal access token needed for private submodule repos (read access to org repos)." >&2 - echo "Saved to cleandev/.github-token (gitignored). Use SUBMODULIZE_SSH=1 instead if you use SSH in the container." >&2 + echo "Saved to submodulizer-local/.github-token (gitignored). Use SUBMODULIZE_SSH=1 instead if you use SSH in the container." >&2 read -r -s -p "GitHub token: " SUBMOD_GIT_TOKEN echo >&2 if [ -n "$SUBMOD_GIT_TOKEN" ]; then mkdir -p "$(dirname "$github_token_file")" ( umask 077 && printf '%s\n' "$SUBMOD_GIT_TOKEN" >"$github_token_file" ) chmod 600 "$github_token_file" 2>/dev/null || true - echo "Token stored in cleandev/.github-token" >&2 + echo "Token stored in submodulizer-local/.github-token" >&2 fi fi if [ -n "$SUBMOD_GIT_TOKEN" ]; then return 0 fi - echo "new.sh: No GitHub token for --submodulize. Options: export GITHUB_TOKEN or GH_TOKEN, create cleandev/.github-token," >&2 + echo "new.sh: No GitHub token for --submodulize. Options: export GITHUB_TOKEN or GH_TOKEN, create submodulizer-local/.github-token," >&2 echo " run this script in a terminal (interactive prompt), or set SUBMODULIZE_SSH=1 for SSH URLs." >&2 exit 1 } @@ -167,9 +167,9 @@ while [[ $# -gt 0 ]]; do Options: -h --help Shows this text. -s --skip Skip automatic Moodle installation. - --submodulize After merge, run cleandev/submodulize.sh --no-replay for manifest plugins (submodules). + --submodulize After merge, run submodulizer/submodulize.sh --no-replay for manifest plugins (submodules). Safe on cleandev-style trees: paths already in .gitmodules are skipped (no bulk no-op run). - GitHub auth (private lsuonline/*): GITHUB_TOKEN or GH_TOKEN, file cleandev/.github-token, + GitHub auth (private lsuonline/*): GITHUB_TOKEN or GH_TOKEN, file submodulizer-local/.github-token, or an interactive prompt (first run) saves the token there (gitignored). Alternatively SUBMODULIZE_SSH=1 with SSH usable inside the container."; exit; @@ -196,9 +196,17 @@ if [ -z "$NAME" ]; then NAME=$(dd if=/dev/urandom bs=2 count=1 2>/dev/null | od -An -t x1 | tr -d ' \n') fi -# Populated when --submodulize runs (env, cleandev/.github-token, or prompt). +# Populated when --submodulize runs (env, submodulizer-local/.github-token, or prompt). SUBMOD_GIT_TOKEN="" +# Resolve a GitHub token now: the Docker build clones the private lsuonline/moodleus +# repo, so GITHUB_TOKEN must be exported before `docker compose up` so the +# `github_token` build secret declared in docker-compose.yml can pick it up. +# The same token is reused for --submodulize further down (no second prompt). +# shellcheck source=submodulizer-local/lib-github-token.sh +. "${PROJECT_ROOT}/submodulizer-local/lib-github-token.sh" +resolve_github_token || exit 1 + # Ensure Traefik infrastructure is ready ensure_traefik_network ensure_traefik_running @@ -209,14 +217,14 @@ BRANCH_NAME=$NAME docker compose -p "${NAME}" up -d # with block_lsu_people). skip-worktree on ues_people and config.php for a clean git status after startup. # enrol/workdaystudent and blocks/wdsprefs come from the lsuce-moodle tree (or --submodulize manifest). if [ "$SUBMODULIZE" = true ]; then - if [ ! -f "${PROJECT_ROOT}/cleandev/submodulize.sh" ] || [ ! -f "${PROJECT_ROOT}/cleandev/manifest-submodulize-redundant.sh" ] || [ ! -f "${PROJECT_ROOT}/cleandev/plugin-submodules.manifest" ]; then - echo "new.sh --submodulize requires cleandev/submodulize.sh, cleandev/manifest-submodulize-redundant.sh, and cleandev/plugin-submodules.manifest next to new.sh." >&2 + if [ ! -f "${PROJECT_ROOT}/submodulizer/submodulize.sh" ] || [ ! -f "${PROJECT_ROOT}/submodulizer-local/manifest-submodulize-redundant.sh" ] || [ ! -f "${PROJECT_ROOT}/submodulizer-local/plugin-submodules.manifest" ]; then + echo "new.sh --submodulize requires submodulizer/submodulize.sh (submodule; run 'git submodule update --init --recursive'), submodulizer-local/manifest-submodulize-redundant.sh, and submodulizer-local/plugin-submodules.manifest." >&2 exit 1 fi resolve_submod_github_token - MSYS_NO_PATHCONV=1 docker cp "$(host_path_for_docker_cp "${PROJECT_ROOT}/cleandev/submodulize.sh")" "${NAME}-moodle:/tmp/submodulize.sh" - MSYS_NO_PATHCONV=1 docker cp "$(host_path_for_docker_cp "${PROJECT_ROOT}/cleandev/manifest-submodulize-redundant.sh")" "${NAME}-moodle:/tmp/manifest-submodulize-redundant.sh" - MSYS_NO_PATHCONV=1 docker cp "$(host_path_for_docker_cp "${PROJECT_ROOT}/cleandev/plugin-submodules.manifest")" "${NAME}-moodle:/tmp/plugin-submodules.manifest" + MSYS_NO_PATHCONV=1 docker cp "$(host_path_for_docker_cp "${PROJECT_ROOT}/submodulizer/submodulize.sh")" "${NAME}-moodle:/tmp/submodulize.sh" + MSYS_NO_PATHCONV=1 docker cp "$(host_path_for_docker_cp "${PROJECT_ROOT}/submodulizer-local/manifest-submodulize-redundant.sh")" "${NAME}-moodle:/tmp/manifest-submodulize-redundant.sh" + MSYS_NO_PATHCONV=1 docker cp "$(host_path_for_docker_cp "${PROJECT_ROOT}/submodulizer-local/plugin-submodules.manifest")" "${NAME}-moodle:/tmp/plugin-submodules.manifest" # docker cp leaves root-owned files; sed -i as www-data fails with "cannot rename: Operation not permitted" MSYS_NO_PATHCONV=1 docker exec "${NAME}-moodle" sh -c 'sed -i '"'"'s/\r$//'"'"' /tmp/submodulize.sh /tmp/manifest-submodulize-redundant.sh /tmp/plugin-submodules.manifest && chmod +x /tmp/submodulize.sh /tmp/manifest-submodulize-redundant.sh && chown www-data:www-data /tmp/submodulize.sh /tmp/manifest-submodulize-redundant.sh /tmp/plugin-submodules.manifest' submod_docker_env=(-u www-data) @@ -231,11 +239,11 @@ if [ "$SUBMODULIZE" = true ]; then cd /var/www/html git config --add safe.directory /var/www/html if [ -n "${GITHUB_TOKEN:-}" ]; then - git config --local url."https://${GITHUB_TOKEN}@github.com/".insteadOf "https://github.com/" + git config --local url."https://smatts3%40lsu.edu:${GITHUB_TOKEN}@github.com/".insteadOf "https://github.com/" fi rm -rf blocks/ues_people - git fetch origin develop - git merge origin/develop + git fetch origin MOODLE_405_MAIN + git merge origin/MOODLE_405_MAIN git ls-files blocks/ues_people | xargs -r git update-index --skip-worktree rm -rf blocks/ues_people cp /tmp/plugin-submodules.manifest /var/www/html/plugin-submodules.manifest @@ -249,14 +257,23 @@ if [ "$SUBMODULIZE" = true ]; then git update-index --skip-worktree config.php ' else - MSYS_NO_PATHCONV=1 docker exec -u www-data "${NAME}-moodle" sh -c ' - cd /var/www/html && \ - git config --add safe.directory /var/www/html && \ - rm -rf blocks/ues_people && \ - git fetch origin develop && \ - git merge origin/develop && \ - git ls-files blocks/ues_people | xargs -r git update-index --skip-worktree && \ - rm -rf blocks/ues_people && \ + # Pass GITHUB_TOKEN into the container so the fetch against the private + # lsuonline/moodleus can authenticate. Once set, `git config --local + # url...insteadOf` writes the rewrite into /var/www/html/.git/config so + # subsequent in-container git operations (e.g. moodle-pull) work without + # re-supplying the token. + MSYS_NO_PATHCONV=1 docker exec -u www-data -e "GITHUB_TOKEN=${GITHUB_TOKEN:-}" "${NAME}-moodle" sh -c ' + set -e + cd /var/www/html + git config --add safe.directory /var/www/html + if [ -n "${GITHUB_TOKEN:-}" ]; then + git config --local url."https://smatts3%40lsu.edu:${GITHUB_TOKEN}@github.com/".insteadOf "https://github.com/" + fi + rm -rf blocks/ues_people + git fetch origin MOODLE_405_MAIN + git merge origin/MOODLE_405_MAIN + git ls-files blocks/ues_people | xargs -r git update-index --skip-worktree + rm -rf blocks/ues_people git update-index --skip-worktree config.php ' fi diff --git a/submodulizer b/submodulizer new file mode 160000 index 0000000..feaac0e --- /dev/null +++ b/submodulizer @@ -0,0 +1 @@ +Subproject commit feaac0ed5042a9dfa37e72090468921da5aea460 diff --git a/cleandev/TEAM-PROCESS.md b/submodulizer-local/TEAM-PROCESS.md similarity index 94% rename from cleandev/TEAM-PROCESS.md rename to submodulizer-local/TEAM-PROCESS.md index b3f79b2..c4459b4 100644 --- a/cleandev/TEAM-PROCESS.md +++ b/submodulizer-local/TEAM-PROCESS.md @@ -2,7 +2,7 @@ This document covers **branch policy** for [lsuce-moodle](https://github.com/lsuonline/lsuce-moodle), the **Docker image decision** for this repo, and **how to refresh** `plugin-submodules.manifest` from the plugin inventory CSV. -**Where the manifest lives:** On **cleandev**, treat `plugin-submodules.manifest` as part of the Moodle superproject (repo root), alongside `.gitmodules`. The copy in **this** repo under [`cleandev/plugin-submodules.manifest`](plugin-submodules.manifest) is the maintained source for CSV refresh and CI lint; `submodulize.sh` / `unsubmodulize.sh` default to `ROOT/plugin-submodules.manifest` unless you pass `--manifest`. +**Where the manifest lives:** On **cleandev**, treat `plugin-submodules.manifest` as part of the Moodle superproject (repo root), alongside `.gitmodules`. The copy in **this** repo under [`submodulizer-local/plugin-submodules.manifest`](plugin-submodules.manifest) is the maintained source for CSV refresh and CI lint; `submodulize.sh` / `unsubmodulize.sh` default to `ROOT/plugin-submodules.manifest` unless you pass `--manifest`. **Branch replay (default):** `unsubmodulize.sh` and `submodulize.sh` run replay unless you pass **`--no-replay`**. Replay builds histories with **one Moodle superproject commit per plugin-repo commit** (chronological order, carry-forward). **`--fork-point`** defaults to local **`master`** (else **`main`**) when omitted, when the script can infer it; otherwise pass it explicitly. **`--source`** defaults when omitted: **unsub** prefers **`unsubmodulized`**, then **`master`**, **`main`**, **`submodulized`**; **sub** prefers **`submodulized`**, **`master`**, **`main`**. Source tips may be **gitlinks or vendored trees** (tree matched to plugin commits). Defaults **`--target`** branch names (`submodulized`, `unsubmodulized`). If the fork-point checkout is **vendored** at a manifest path, the script matches the embedded tree to a plugin commit (or use `--plugin-base path=SHA`). `new.sh --submodulize` runs **`submodulize.sh --no-replay`** (one-shot only). @@ -32,8 +32,8 @@ This document covers **branch policy** for [lsuce-moodle](https://github.com/lsu ### Local Docker stack (`new.sh`) - **Without `--submodulize`:** Container tracks **vendored** layout after merge from `origin/develop` (matches **develop**). -- **With `--submodulize`:** After that merge, runs `submodulize.sh --no-replay` so manifest-listed paths become **submodules** (closer to **cleandev**). Requires `GITHUB_TOKEN` / `cleandev/.github-token` or `SUBMODULIZE_SSH=1` for private GitHub repos. -- Never commit **`cleandev/.github-token`** (gitignored). +- **With `--submodulize`:** After that merge, runs `submodulize.sh --no-replay` so manifest-listed paths become **submodules** (closer to **cleandev**). Requires `GITHUB_TOKEN` / `submodulizer-local/.github-token` or `SUBMODULIZE_SSH=1` for private GitHub repos. +- Never commit **`submodulizer-local/.github-token`** (gitignored). ### Submodule hygiene diff --git a/cleandev/docker-compose.github-runner.yml b/submodulizer-local/docker-compose.github-runner.yml similarity index 100% rename from cleandev/docker-compose.github-runner.yml rename to submodulizer-local/docker-compose.github-runner.yml diff --git a/cleandev/github-actions.workflow.example.yml b/submodulizer-local/github-actions.workflow.example.yml similarity index 100% rename from cleandev/github-actions.workflow.example.yml rename to submodulizer-local/github-actions.workflow.example.yml diff --git a/cleandev/github-runner.env.example b/submodulizer-local/github-runner.env.example similarity index 100% rename from cleandev/github-runner.env.example rename to submodulizer-local/github-runner.env.example diff --git a/submodulizer-local/lib-github-token.sh b/submodulizer-local/lib-github-token.sh new file mode 100644 index 0000000..42773d6 --- /dev/null +++ b/submodulizer-local/lib-github-token.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +# lib-github-token.sh -- sourced by new.sh and build.sh. +# +# Defines resolve_github_token(): on success, GITHUB_TOKEN is set and exported, +# so docker compose / docker build can forward it to the BuildKit secret +# `github_token` consumed by Dockerfile. Returns non-zero (and prints guidance) +# when no source provides a token. +# +# Sources, in order: +# 1. GITHUB_TOKEN env var (already set) +# 2. GH_TOKEN env var +# 3. submodulizer-local/.github-token file (next to this lib) +# 4. Interactive prompt (TTY only); persists to the file above (chmod 600) +# +# SUBMODULIZE_SSH=1 is intentionally NOT honored here: the Docker build always +# clones lsuonline/moodleus over HTTPS, regardless of how the in-container +# submodulize step in new.sh chooses to authenticate later. + +resolve_github_token() { + local lib_dir token_file value + lib_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + token_file="${lib_dir}/.github-token" + + if [ -n "${GITHUB_TOKEN:-}" ]; then + export GITHUB_TOKEN + return 0 + fi + if [ -n "${GH_TOKEN:-}" ]; then + GITHUB_TOKEN="$GH_TOKEN" + export GITHUB_TOKEN + return 0 + fi + if [ -f "$token_file" ]; then + value="$( + grep -v '^[[:space:]]*#' "$token_file" 2>/dev/null \ + | grep -v '^[[:space:]]*$' | head -n1 | tr -d '\r' \ + | sed 's/^[[:space:]]*//;s/[[:space:]]*$//' + )" + if [ -n "$value" ]; then + GITHUB_TOKEN="$value" + export GITHUB_TOKEN + return 0 + fi + fi + if [ -t 0 ] && [ -t 1 ]; then + echo "GitHub personal access token needed: the Docker build clones private lsuonline/moodleus." >&2 + echo "Read scope on lsuonline/* repos is enough. Token will be saved to" >&2 + echo " ${token_file} (gitignored, chmod 600)." >&2 + read -r -s -p "GitHub token: " value + echo >&2 + if [ -n "$value" ]; then + mkdir -p "$(dirname "$token_file")" + ( umask 077 && printf '%s\n' "$value" >"$token_file" ) + chmod 600 "$token_file" 2>/dev/null || true + echo "Token stored in submodulizer-local/.github-token" >&2 + GITHUB_TOKEN="$value" + export GITHUB_TOKEN + return 0 + fi + fi + echo "lib-github-token.sh: No GitHub token resolved. lsuonline/moodleus is private," >&2 + echo " so the Docker build cannot clone it. Options:" >&2 + echo " - export GITHUB_TOKEN (or GH_TOKEN) before re-running" >&2 + echo " - create ${token_file}" >&2 + echo " - run the script in a terminal to be prompted" >&2 + return 1 +} diff --git a/cleandev/manifest-submodulize-redundant.sh b/submodulizer-local/manifest-submodulize-redundant.sh similarity index 91% rename from cleandev/manifest-submodulize-redundant.sh rename to submodulizer-local/manifest-submodulize-redundant.sh index 2c3f33a..a49e8a1 100644 --- a/cleandev/manifest-submodulize-redundant.sh +++ b/submodulizer-local/manifest-submodulize-redundant.sh @@ -1,10 +1,10 @@ #!/usr/bin/env bash # Exit 0 if running submodulize.sh would be a no-op: every active manifest path is already listed # in .gitmodules and .gitmodules is non-empty. Exit 1 otherwise (submodulize should run). -# Used by new.sh --submodulize and cleandev/tests. +# Used by new.sh --submodulize and submodulizer/tests. # Default manifest: ROOT/plugin-submodules.manifest (same as submodulize.sh). # -# Usage: ./cleandev/manifest-submodulize-redundant.sh --repo ROOT [--manifest PATH] +# Usage: ./submodulizer-local/manifest-submodulize-redundant.sh --repo ROOT [--manifest PATH] set -euo pipefail REPO_ROOT="" diff --git a/cleandev/plugin-submodules.manifest b/submodulizer-local/plugin-submodules.manifest similarity index 99% rename from cleandev/plugin-submodules.manifest rename to submodulizer-local/plugin-submodules.manifest index c418b9c..628f399 100644 --- a/cleandev/plugin-submodules.manifest +++ b/submodulizer-local/plugin-submodules.manifest @@ -5,7 +5,7 @@ # # Synced from repo CSV: Evaluation of LSU moodle messy plugins(List) (3).csv # Custom URL preferred over third-party when both are set. -# How to refresh from CSV and team policy: cleandev/TEAM-PROCESS.md +# How to refresh from CSV and team policy: submodulizer-local/TEAM-PROCESS.md # # --- One repo per path (compatible with submodulize.sh / unsubmodulize.sh) --- From 9f880adc4ec7267fd1770c0f90f0d1602643d69a Mon Sep 17 00:00:00 2001 From: Steve Mattsen Date: Mon, 1 Jun 2026 17:04:29 -0500 Subject: [PATCH 17/18] Remove stray submodule gitlinks under `demo/` that broke `actions/checkout` in CI with `fatal: No url found for submodule path 'demo/plugin1' in .gitmodules`. The five paths (`demo/plugin1`, `demo/plugin2`, `demo/plugin3`, `demo/raw`, `demo/sub`) were committed in 7a3669c as mode-160000 gitlinks but never given matching entries in `.gitmodules` -- they were local sandbox repos created while iterating on `submodulize.sh` and have no remote URL to clone from. Drop the gitlinks from the index with `git rm --cached` and add `/demo/` to `.gitignore` so the local scratch directories on disk are preserved and no longer trip `git status` or future stagings. The real `submodulizer` submodule entry is unchanged. Co-authored-by: Cursor --- .gitignore | 7 ++++++- demo/plugin1 | 1 - demo/plugin2 | 1 - demo/plugin3 | 1 - demo/raw | 1 - demo/sub | 1 - 6 files changed, 6 insertions(+), 6 deletions(-) delete mode 160000 demo/plugin1 delete mode 160000 demo/plugin2 delete mode 160000 demo/plugin3 delete mode 160000 demo/raw delete mode 160000 demo/sub diff --git a/.gitignore b/.gitignore index 7687fab..6c78f21 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,8 @@ confidential submodulizer-local/github-runner.env -submodulizer-local/.github-token \ No newline at end of file +submodulizer-local/.github-token + +# Local sandbox repos used when iterating on submodulizer.sh. These are not +# real submodules (no entries in .gitmodules, no remotes) -- keep them out of +# git status / future stages. +/demo/ \ No newline at end of file diff --git a/demo/plugin1 b/demo/plugin1 deleted file mode 160000 index 6e9895b..0000000 --- a/demo/plugin1 +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 6e9895b10c577bd0277bb41be86c2a1a5759c071 diff --git a/demo/plugin2 b/demo/plugin2 deleted file mode 160000 index fab9bed..0000000 --- a/demo/plugin2 +++ /dev/null @@ -1 +0,0 @@ -Subproject commit fab9bede9d1d76afb4032e7788280f2f500b80de diff --git a/demo/plugin3 b/demo/plugin3 deleted file mode 160000 index c23687a..0000000 --- a/demo/plugin3 +++ /dev/null @@ -1 +0,0 @@ -Subproject commit c23687ab630cde28a770a4bd1165b7d81d6603a4 diff --git a/demo/raw b/demo/raw deleted file mode 160000 index 2103bac..0000000 --- a/demo/raw +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 2103bac06d1d55ddfa5ae3cd4509d56f30c81d8f diff --git a/demo/sub b/demo/sub deleted file mode 160000 index 2103bac..0000000 --- a/demo/sub +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 2103bac06d1d55ddfa5ae3cd4509d56f30c81d8f From fffcd7a493336e71a50fe2f6896cf21f87ffddc2 Mon Sep 17 00:00:00 2001 From: Steve Mattsen Date: Mon, 1 Jun 2026 17:11:13 -0500 Subject: [PATCH 18/18] Fix CI by giving the GitHub Actions runner what the upstream submodulizer tests assume: a configured git identity and a copy of `manifest-submodulize-redundant.sh` next to `submodulize.sh`. Without `git config --global user.email/user.name`, every test that creates a temp repo and commits into it dies with `fatal: empty ident name (for )` -- 15 of the 17 tests in `bash submodulizer/tests/run.sh` were failing for this reason alone. The upstream tests `test-manifest-submod-redundant.sh` and `test-shellcheck.sh` additionally `assert_file "$CLEANDEV/manifest-submodulize-redundant.sh"` / `shellcheck ../manifest-submodulize-redundant.sh` against the submodulizer repo root, but this project keeps that script as project-owned glue under `submodulizer-local/`, so the file isn't where the tests look. Add a CI-only `ln -sf` step that exposes it inside `submodulizer/` (symlink lives only on the runner; nothing is committed and the submodule working tree is discarded with the job). `test-shellcheck.sh` will still fail until the SC2115/SC2015/SC1083 findings in `submodulizer/submodulize.sh` and `submodulizer/unsubmodulize.sh` are addressed upstream in smatts3/submodulizer; this commit deliberately does not touch the submodule. Co-authored-by: Cursor --- .github/workflows/submodulizer-tests.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/.github/workflows/submodulizer-tests.yml b/.github/workflows/submodulizer-tests.yml index 42c725d..a45a364 100644 --- a/.github/workflows/submodulizer-tests.yml +++ b/.github/workflows/submodulizer-tests.yml @@ -25,5 +25,23 @@ jobs: - name: Install shellcheck run: sudo apt-get update && sudo apt-get install -y shellcheck + # GitHub Actions runners ship without a git identity. The upstream + # submodulizer tests build temp repos and `git commit` into them, which + # fails with `fatal: empty ident name` unless we set one here. + - name: Configure git identity for upstream tests + run: | + git config --global user.email "ci@github.actions" + git config --global user.name "GitHub Actions (submodulizer-tests)" + + # Upstream submodulizer tests (test-manifest-submod-redundant.sh, + # test-shellcheck.sh) expect manifest-submodulize-redundant.sh to live at + # the repo root next to submodulize.sh. In this project it ships as + # project-owned glue under submodulizer-local/. Symlink it into the + # submodule checkout for the duration of CI so the tests find it. + # CI-only: nothing here is committed and the submodule working tree is + # discarded with the runner. + - name: Expose manifest-submodulize-redundant.sh to upstream tests + run: ln -sf ../submodulizer-local/manifest-submodulize-redundant.sh submodulizer/manifest-submodulize-redundant.sh + - name: Run submodulizer test suite (upstream) run: bash submodulizer/tests/run.sh