diff --git a/.gitattributes b/.gitattributes index 691422b..1e6d3fd 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,2 +1,3 @@ # Shell scripts copied into Linux containers must use LF only (CRLF breaks bash). -cleandev/*.sh text eol=lf +# (Scripts in the submodulizer/ submodule are governed by that repo's own .gitattributes.) +submodulizer-local/*.sh text eol=lf diff --git a/.github/workflows/submodulizer-tests.yml b/.github/workflows/submodulizer-tests.yml new file mode 100644 index 0000000..a45a364 --- /dev/null +++ b/.github/workflows/submodulizer-tests.yml @@ -0,0 +1,47 @@ +name: submodulizer tests + +on: + push: + paths: + - ".gitmodules" + - "submodulizer" + - "submodulizer-local/**" + - ".github/workflows/submodulizer-tests.yml" + pull_request: + paths: + - ".gitmodules" + - "submodulizer" + - "submodulizer-local/**" + - ".github/workflows/submodulizer-tests.yml" + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Install shellcheck + run: sudo apt-get update && sudo apt-get install -y shellcheck + + # GitHub Actions runners ship without a git identity. The upstream + # submodulizer tests build temp repos and `git commit` into them, which + # fails with `fatal: empty ident name` unless we set one here. + - name: Configure git identity for upstream tests + run: | + git config --global user.email "ci@github.actions" + git config --global user.name "GitHub Actions (submodulizer-tests)" + + # Upstream submodulizer tests (test-manifest-submod-redundant.sh, + # test-shellcheck.sh) expect manifest-submodulize-redundant.sh to live at + # the repo root next to submodulize.sh. In this project it ships as + # project-owned glue under submodulizer-local/. Symlink it into the + # submodule checkout for the duration of CI so the tests find it. + # CI-only: nothing here is committed and the submodule working tree is + # discarded with the runner. + - name: Expose manifest-submodulize-redundant.sh to upstream tests + run: ln -sf ../submodulizer-local/manifest-submodulize-redundant.sh submodulizer/manifest-submodulize-redundant.sh + + - name: Run submodulizer test suite (upstream) + run: bash submodulizer/tests/run.sh diff --git a/.gitignore b/.gitignore index e1f8b9b..6c78f21 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,8 @@ confidential -cleandev/github-runner.env -cleandev/.github-token \ No newline at end of file +submodulizer-local/github-runner.env +submodulizer-local/.github-token + +# Local sandbox repos used when iterating on submodulizer.sh. These are not +# real submodules (no entries in .gitmodules, no remotes) -- keep them out of +# git status / future stages. +/demo/ \ No newline at end of file diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..8b53c38 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,4 @@ +[submodule "submodulizer"] + path = submodulizer + url = https://github.com/smatts3/submodulizer.git + branch = master diff --git a/Dockerfile b/Dockerfile index 0c1b010..63cce40 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,11 +3,28 @@ FROM php:8.3-apache # Install basic tools RUN apt update && apt install git -y -# Import dev source code -RUN su -g www-data -c "git clone --branch develop --single-branch https://github.com/lsuonline/lsuce-moodle.git /var/www/html/" +# Import dev source code. lsuonline/moodleus is private; the token comes in via +# a BuildKit secret (id=github_token). docker-compose.yml maps it from the +# GITHUB_TOKEN env var; build.sh passes --secret id=github_token,env=GITHUB_TOKEN. +# Both entry points use submodulizer-local/lib-github-token.sh to populate +# GITHUB_TOKEN. The token is consumed via `git -c url...insteadOf`, so neither +# the token nor the username persist in image layers or /var/www/html/.git/config. +# Falls back to an anonymous clone when no secret is provided (useful for testing +# against any future public mirror; fails for the real private repo). +RUN --mount=type=secret,id=github_token \ + su -g www-data -c 'URL=https://github.com/lsuonline/moodleus.git; \ + if [ -s /run/secrets/github_token ]; then \ + TOKEN=$(cat /run/secrets/github_token); \ + git -c "url.https://smatts3%40lsu.edu:${TOKEN}@github.com/.insteadOf=https://github.com/" \ + clone --branch MOODLE_405_MAIN --single-branch "$URL" /var/www/html/; \ + else \ + git clone --branch MOODLE_405_MAIN --single-branch "$URL" /var/www/html/; \ + fi' -# Install dependencies -RUN apt-get install -y --fix-missing \ +# Install dependencies. `apt-get update` must run in the same layer as install: +# the base image ships a cached apt index that pins -security package versions +# Debian removes over time (e.g. libpng1.6 .deb 404s on stale indexes). +RUN apt-get update && apt-get install -y --fix-missing \ libpng-dev \ libonig-dev \ libjpeg-dev \ diff --git a/Dockerfile.txt b/Dockerfile.txt index 6e30cb7..7fcf790 100644 --- a/Dockerfile.txt +++ b/Dockerfile.txt @@ -3,11 +3,23 @@ FROM php:8.3-apache # Install basic tools RUN apt update && apt install git -y -# Import dev source code -RUN su -g www-data -c "git clone --branch develop --single-branch https://github.com/lsuonline/lsuce-moodle.git /var/www/html/" +# Import dev source code. See Dockerfile for full notes on auth: the token +# comes in via a BuildKit secret (id=github_token) supplied by build.sh / +# docker-compose.yml, consumed via `git -c url...insteadOf` so it never +# persists in image layers or .git/config. +RUN --mount=type=secret,id=github_token \ + su -g www-data -c 'URL=https://github.com/lsuonline/moodleus.git; \ + if [ -s /run/secrets/github_token ]; then \ + TOKEN=$(cat /run/secrets/github_token); \ + git -c "url.https://smatts3%40lsu.edu:${TOKEN}@github.com/.insteadOf=https://github.com/" \ + clone --branch MOODLE_405_MAIN --single-branch "$URL" /var/www/html/; \ + else \ + git clone --branch MOODLE_405_MAIN --single-branch "$URL" /var/www/html/; \ + fi' -# Install dependencies -RUN apt-get install -y \ +# Install dependencies. apt-get update must be in the same layer as install +# (stale base-image apt index causes -security 404s on removed packages). +RUN apt-get update && apt-get install -y \ libpng-dev \ libonig-dev \ libjpeg-dev \ diff --git a/README.md b/README.md index f1c4b95..617b937 100644 --- a/README.md +++ b/README.md @@ -62,7 +62,7 @@ To convert vendored plugin directories in the container to Git submodules (see * ./new.sh NAME --submodulize ``` -GitHub auth for private `lsuonline/*` repos: `GITHUB_TOKEN` or `GH_TOKEN`, file `cleandev/.github-token` (gitignored), interactive prompt when run in a TTY, or `SUBMODULIZE_SSH=1` if SSH works inside the container. +GitHub auth for private `lsuonline/*` repos: `GITHUB_TOKEN` or `GH_TOKEN`, file `submodulizer-local/.github-token` (gitignored), interactive prompt when run in a TTY, or `SUBMODULIZE_SSH=1` if SSH works inside the container. # Build @@ -88,13 +88,23 @@ These are **branches on the Moodle repo** (`lsuce-moodle`), not branch names in - Porting changes **both ways** (submodule layout ↔ vendored layout). - **Redoing or replaying commits** with a **provenance note** (e.g. original commit SHA / branch)—so history stays traceable across layouts. -**Current repo state:** This project ships **one-shot layout converters** (`submodulize.sh` / `unsubmodulize.sh`). It does **not** yet implement commit-by-commit replay, automatic provenance headers, or a `new.sh` mode that detects vendored vs submodulized state without manual choice. +**Current repo state:** This project ships **layout converters** (`submodulize.sh` / `unsubmodulize.sh`). **Replay is the default:** they build `submodulized` / `unsubmodulized` branches with **one superproject commit per plugin-repo commit** (chronological ordering, carry-forward). **`--fork-point`** defaults to local **`master`** (else **`main`**) when omitted, if a default can be chosen safely; **`--source`** defaults for **unsub** to **`unsubmodulized`**, else **`master`**, else **`main`**, else **`submodulized`** (for **sub**: **`submodulized`**, **`master`**, **`main`**). Use **`--no-replay`** for one-shot conversion over the manifest only. A `new.sh` mode that detects vendored vs submodulized state without manual choice is still not implemented. -**Branch policy, Docker image decision, and updating the manifest from the CSV:** [cleandev/TEAM-PROCESS.md](cleandev/TEAM-PROCESS.md). +**Branch policy, Docker image decision, and updating the manifest from the CSV:** [submodulizer-local/TEAM-PROCESS.md](submodulizer-local/TEAM-PROCESS.md). -# Submodule tooling (`cleandev/`) +# Submodule tooling (`submodulizer/` + `submodulizer-local/`) -## Manifest (`cleandev/plugin-submodules.manifest`) +The conversion scripts (`submodulize.sh`, `unsubmodulize.sh`, their tests) live in **[smatts3/submodulizer](https://github.com/smatts3/submodulizer)**, vendored here as a Git submodule at `submodulizer/`. Project-owned glue (manifest, team process, runner config) lives next to it in `submodulizer-local/`. + +After cloning this repo: + +```bash +git submodule update --init --recursive +``` + +## Manifest (`submodulizer-local/plugin-submodules.manifest`) + +This repo keeps a **canonical copy** under `submodulizer-local/` for linting and for `new.sh --submodulize` (copied into the Moodle tree at runtime). When you run `submodulize.sh` / `unsubmodulize.sh` against a checkout, the default manifest path is **`plugin-submodules.manifest` at the Moodle superproject root** (`--repo` / current directory), not next to the scripts; use `--manifest PATH` to override. - Format: `relative_path|clone_url|branch` (e.g. `mod/hvp|https://github.com/...|main`). Lines starting with `#` are ignored. - If the third field is empty, scripts default the branch to `main`; if `refs/heads/` is missing on the remote, they omit `-b` and use the remote’s default branch. @@ -107,14 +117,20 @@ These are **branches on the Moodle repo** (`lsuce-moodle`), not branch names in | Script | Role | |--------|------| -| `cleandev/submodulize.sh` | Vendored trees → submodules (sparse-checkout disabled first; skips paths already in `.gitmodules`; `GITHUB_TOKEN` passed via `-c url...insteadOf` for `ls-remote` / `submodule add`). | -| `cleandev/unsubmodulize.sh` | Submodules → vendored trees (clone depth 1, drop nested `.git`, `git add`). Uses the same `GITHUB_TOKEN` `-c url...insteadOf` for `ls-remote` / `clone` as `submodulize.sh`, or `--ssh`. | +| `submodulizer/submodulize.sh` | Default **replay**; **`--fork-point`** defaults to `master`/`main` when omitted (see script help). **`--no-replay`**: one-shot vendored → submodules (sparse-checkout disabled first; skips paths already in `.gitmodules`; `GITHUB_TOKEN` via `-c url...insteadOf` for `ls-remote` / `submodule add`). | +| `submodulizer/unsubmodulize.sh` | Default **replay**; same **`--fork-point`** defaulting. **`--no-replay`**: one-shot submodules → vendored (clone depth 1, drop nested `.git`, `git add`). Same `GITHUB_TOKEN` / `--ssh` as `submodulize.sh`. | Run manually from a Moodle clone: ```bash -./cleandev/submodulize.sh [--dry-run] [--no-commit] [--ssh] [--manifest PATH] [--repo ROOT] -./cleandev/unsubmodulize.sh [--dry-run] [--no-commit] [--ssh] [--manifest PATH] [--repo ROOT] +./submodulizer/submodulize.sh [--no-replay] [--dry-run] [--no-commit] [--ssh] [--manifest PATH] [--repo ROOT] +./submodulizer/unsubmodulize.sh [--no-replay] [--dry-run] [--no-commit] [--ssh] [--manifest PATH] [--repo ROOT] +``` + +Automated tests (manifest lint, PAT wiring checks, `submodulize`/`unsubmodulize` round-trip in temp repos—no changes to your working tree): + +```bash +bash submodulizer/tests/run.sh ``` ## Container startup (`new.sh`) @@ -122,9 +138,9 @@ Run manually from a Moodle clone: - Compose project name = first argument (containers `{NAME}-moodle`, etc.). - Web service builds from `.` per `docker-compose.yml`. - As `www-data`: `git fetch` / `git merge origin/develop`; removes `blocks/ues_people` and uses `skip-worktree` so it does not clash with `block_lsu_people` (see `config/moodle-pull` for the same idea on `git pull`). -- With `--submodulize`: copies `submodulize.sh` + manifest into the container, resolves GitHub token, sets local `url...insteadOf` when using HTTPS token, runs `submodulize.sh --repo /var/www/html ... --no-commit` (optional SSH via `SUBMODULIZE_SSH=1`). +- With `--submodulize`: copies scripts into the container, stages `submodulizer-local/plugin-submodules.manifest` into `/var/www/html/plugin-submodules.manifest` after the merge, resolves GitHub token, sets local `url...insteadOf` when using HTTPS token, runs `manifest-submodulize-redundant.sh` / `submodulize.sh --no-replay` with default manifest paths (`--repo /var/www/html`, optional SSH via `SUBMODULIZE_SSH=1`). -Secrets: `cleandev/.github-token` is listed in `.gitignore`. +Secrets: `submodulizer-local/.github-token` is listed in `.gitignore`. # Roadmap diff --git a/ROADMAP.md b/ROADMAP.md index fbba8e5..68010c7 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -6,24 +6,25 @@ Progress toward **seamless `develop` ↔ `cleandev`** (on [lsuce-moodle](https:/ - [x] Docker Compose + Traefik-oriented local stack via `new.sh` - [x] Web image built from repo `Dockerfile` (PHP 8.3) to avoid Moodle 4.5 + PHP 8.4 CLI `SerializableClosure` install failure (`docker-compose.yml` documents reverting to a pre-built image if desired) -- [x] `cleandev/plugin-submodules.manifest` + `submodulize.sh` + `unsubmodulize.sh` +- [x] `submodulizer-local/plugin-submodules.manifest` + `submodulizer/submodulize.sh` + `submodulizer/unsubmodulize.sh` (the scripts vendored as a Git submodule from [smatts3/submodulizer](https://github.com/smatts3/submodulizer)) - [x] `new.sh --submodulize` with token file / env / prompt and optional SSH - [x] Sparse-checkout disabled in both scripts before mutating plugin paths - [x] `blocks/ues_people` handling in `new.sh` and `config/moodle-pull` (merge / pull alias) -- [x] Git ignore for `cleandev/.github-token` -- [x] **Team process, image strategy, manifest-from-CSV docs:** [cleandev/TEAM-PROCESS.md](cleandev/TEAM-PROCESS.md) +- [x] Git ignore for `submodulizer-local/.github-token` +- [x] **Team process, image strategy, manifest-from-CSV docs:** [submodulizer-local/TEAM-PROCESS.md](submodulizer-local/TEAM-PROCESS.md) ## TODO - [ ] **Commit portability:** Script or documented workflow to **replay or port commits** across layouts with a **header** noting the original commit (and branch/repo). *Not in repo today—only bulk layout conversion exists.* -- [ ] **`new.sh` / tooling:** Detect or select **vendored vs submodulized** Moodle tree so one flow works without guessing flags. +- [x] **`new.sh` / tooling:** With `--submodulize`, **detect** when every manifest path is already a submodule (cleandev-style) and **skip** `submodulize.sh` instead of a no-op pass. *Not done: auto-run conversion without passing `--submodulize`.* - [x] **`unsubmodulize.sh`:** Same `GITHUB_TOKEN` `-c url.insteadOf` pattern as `submodulize.sh` for private HTTPS (parity with submodule add). -- [ ] **Monorepo manifest + scripts:** Support “clone once, map subpaths”, or publish **one-shot manual procedures** for commented monorepo lines. +- [x] **Monorepo manifest + scripts:** **One-shot manual procedures** for commented monorepo lines: [submodulizer-local/TEAM-PROCESS.md](submodulizer-local/TEAM-PROCESS.md) (manifest regeneration → Monorepos). *Not done: automated “clone once, map subpaths” in `submodulize.sh`.* - [ ] **`local/ml`:** Point at the correct Git remote when known; uncomment or add manifest line. +- [x] **CI (submodulizer):** GitHub Actions runs `submodulizer/tests/run.sh` (manifest lint, round-trip integration tests, shellcheck on Ubuntu). - [ ] **Optional CI:** Check that manifest paths still match **lsuce-moodle** `develop` plugin layout when plugins move. --- -For branch policy, when to use `new.sh --submodulize`, image strategy, and refreshing `plugin-submodules.manifest` from the CSV, see [cleandev/TEAM-PROCESS.md](cleandev/TEAM-PROCESS.md). +For branch policy, when to use `new.sh --submodulize`, image strategy, and refreshing `plugin-submodules.manifest` from the CSV, see [submodulizer-local/TEAM-PROCESS.md](submodulizer-local/TEAM-PROCESS.md). -For a concise summary of flags and auth behavior, see comments at the top of `cleandev/submodulize.sh` and `cleandev/unsubmodulize.sh`. +For a concise summary of flags and auth behavior, see comments at the top of `submodulizer/submodulize.sh` and `submodulizer/unsubmodulize.sh`. diff --git a/build.sh b/build.sh index bc25b93..caaabd9 100644 --- a/build.sh +++ b/build.sh @@ -1,5 +1,16 @@ -#!/bin/sh +#!/usr/bin/env bash +set -euo pipefail -SCRIPT_DIR="$(dirname "$(readlink -f "$0")")"; -cd $SCRIPT_DIR; -docker build -t lsuonline/moodle-dev:latest "$@" . \ No newline at end of file +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +cd "$SCRIPT_DIR" + +# lsuonline/moodleus is a private repo; the Dockerfile clones it via a BuildKit +# secret. Resolve+export GITHUB_TOKEN, then pass --secret so docker build can +# mount it. See submodulizer-local/lib-github-token.sh for sources/prompts. +# shellcheck source=submodulizer-local/lib-github-token.sh +. "${SCRIPT_DIR}/submodulizer-local/lib-github-token.sh" +resolve_github_token || exit 1 + +DOCKER_BUILDKIT=1 docker build \ + --secret id=github_token,env=GITHUB_TOKEN \ + -t lsuonline/moodle-dev:latest "$@" . diff --git a/cleandev/submodulize.sh b/cleandev/submodulize.sh deleted file mode 100644 index ade80b6..0000000 --- a/cleandev/submodulize.sh +++ /dev/null @@ -1,202 +0,0 @@ -#!/usr/bin/env bash -# Convert vendored plugin directories (plain files in the Moodle repo, e.g. lsuce-moodle develop) -# into git submodules. Run from anywhere inside the Moodle clone; uses repo root. -# -# Requires: git, a clean enough working tree (commit or stash first if paths are dirty). -# -# Usage: -# ./cleandev/submodulize.sh [--dry-run] [--no-commit] [--ssh] [--manifest PATH] [--repo ROOT] -# -# Private GitHub repos over HTTPS need credentials. Set GITHUB_TOKEN (PAT) so HTTPS URLs are rewritten -# for ls-remote / submodule add (parent repo url.insteadOf is not always applied to submodule clone). -# Or use --ssh. In Docker with no TTY you see: "could not read Username for 'https://github.com'". - -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -MANIFEST="${SCRIPT_DIR}/plugin-submodules.manifest" -DRY_RUN=false -NO_COMMIT=false -USE_SSH=false -REPO_ROOT="" - -while [[ $# -gt 0 ]]; do - case "$1" in - --dry-run) DRY_RUN=true; shift ;; - --no-commit) NO_COMMIT=true; shift ;; - --ssh) USE_SSH=true; shift ;; - --manifest) - MANIFEST="${2:?}" - shift 2 - ;; - --repo) - REPO_ROOT="${2:?}" - shift 2 - ;; - -h|--help) - sed -n '1,25p' "$0" - exit 0 - ;; - *) - echo "Unknown option: $1" >&2 - exit 1 - ;; - esac -done - -if [[ ! -f "$MANIFEST" ]]; then - echo "Manifest not found: $MANIFEST" >&2 - exit 1 -fi - -if [[ -z "$REPO_ROOT" ]]; then - REPO_ROOT="$(git rev-parse --show-toplevel 2>/dev/null)" || { - echo "Not inside a git repository. Use --repo /path/to/moodle" >&2 - exit 1 - } -fi - -cd "$REPO_ROOT" - -# Moodle dev images often use sparse-checkout (cone, index.sparse, or only .git/info/sparse-checkout). -# If any of that is active, git rm / submodule add can refuse paths "outside" the cone. -disable_sparse_checkout_if_needed() { - $DRY_RUN && return 0 - local active= - [[ -f .git/info/sparse-checkout ]] && active=1 - [[ "$(git config --bool core.sparseCheckout 2>/dev/null)" == "true" ]] && active=1 - [[ "$(git config --bool index.sparse 2>/dev/null)" == "true" ]] && active=1 - if [[ -z "$active" ]] && command -v git >/dev/null; then - local listed - listed="$(git sparse-checkout list 2>/dev/null | head -n 1 || true)" - [[ -n "${listed// }" ]] && active=1 - fi - [[ -z "$active" ]] && return 0 - echo "Disabling sparse-checkout so plugin paths can be converted to submodules." >&2 - git sparse-checkout disable 2>/dev/null || true - git config core.sparseCheckout false 2>/dev/null || true - git config --unset-all core.sparseCheckoutCone 2>/dev/null || true - git config index.sparse false 2>/dev/null || true - rm -f .git/info/sparse-checkout -} - -disable_sparse_checkout_if_needed - -# Extra -c flags so submodule clone honors GitHub PAT (superproject local config is skipped by some git versions). -git_github_pat_c=() -if [[ -n "${GITHUB_TOKEN:-}" ]]; then - git_github_pat_c+=(-c "url.https://${GITHUB_TOKEN}@github.com/.insteadOf=https://github.com/") -fi - -# github.com HTTPS → SSH (git@github.com:org/repo.git) when --ssh is set. -rewrite_github_url_to_ssh() { - local u="$1" - if $USE_SSH && [[ "$u" == https://github.com/* ]]; then - printf '%s\n' "git@github.com:${u#https://github.com/}" - else - printf '%s\n' "$u" - fi -} - -run() { - if $DRY_RUN; then - printf '[dry-run] %q\n' "$@" - else - "$@" - fi -} - -manifest_entries=0 -while IFS='|' read -r raw_path raw_url raw_branch; do - path="${raw_path#"${raw_path%%[![:space:]]*}"}" - path="${path%"${path##*[![:space:]]}"}" - url="${raw_url#"${raw_url%%[![:space:]]*}"}" - url="${url%"${url##*[![:space:]]}"}" - branch="${raw_branch#"${raw_branch%%[![:space:]]*}"}" - branch="${branch%"${branch##*[![:space:]]}"}" - - [[ -z "$path" || "$path" =~ ^# ]] && continue - [[ -z "$url" ]] && { echo "Manifest: missing URL for path $path" >&2; exit 1; } - [[ -z "$branch" ]] && branch="main" - url="$(rewrite_github_url_to_ssh "$url")" - - ((++manifest_entries)) || true - - if [[ -f .gitmodules ]] && git config -f .gitmodules --get-regexp path 2>/dev/null | awk '{print $2}' | grep -Fxq "$path"; then - echo "Already a submodule (per .gitmodules): $path — skipping" - continue - fi - - if [[ -d "$path/.git" ]] || [[ -f "$path/.git" ]]; then - echo "Path already looks like a nested git repo: $path" >&2 - echo " Remove or convert it manually, or run unsubmodulize first." >&2 - exit 1 - fi - - if [[ -e "$path" ]] && ! $DRY_RUN; then - if ! git diff --quiet -- "$path" 2>/dev/null || ! git diff --cached --quiet -- "$path" 2>/dev/null; then - echo "Uncommitted changes under $path — commit or stash first." >&2 - exit 1 - fi - fi - - echo "Submodulizing: $path ← $url (branch $branch)" - - parent="$(dirname "$path")" - if [[ "$parent" != "." ]]; then - run mkdir -p "$parent" - fi - - # Use tracked-file listing, not --error-unmatch: for a directory, Git often has no - # single index entry named exactly $path, only files underneath — then plain rm - # would leave the path in the index and submodule add fails. - if [[ -n "$(git ls-files -- "$path" 2>/dev/null)" ]]; then - if $DRY_RUN; then - printf '[dry-run] git rm -rf [--sparse] -- %q\n' "$path" - else - git rm -rf --sparse -- "$path" 2>/dev/null || git rm -rf -- "$path" - fi - elif [[ -e "$path" ]]; then - run rm -rf -- "$path" - fi - - if $DRY_RUN; then - printf '[dry-run] git submodule add -f (-b %q if exists on remote, else default branch) -- %q %q\n' "$branch" "$url" "$path" - else - # -f: Moodle .gitignore often ignores plugin dirs; without it submodule add fails. - # -c overrides: sparse-checkout can still block the add otherwise. - # Many upstream Moodle plugins use master or MOODLE_*_STABLE, not main — omit -b to use remote HEAD. - submod_args=(-f) - if [[ -n "$branch" ]] && GIT_TERMINAL_PROMPT=0 git "${git_github_pat_c[@]}" ls-remote --heads "$url" "refs/heads/$branch" 2>/dev/null | grep -q .; then - submod_args+=(-b "$branch") - else - [[ -n "$branch" ]] && echo "Remote has no branch '$branch' for $path; using repository default branch." >&2 - fi - if ! GIT_TERMINAL_PROMPT=0 git "${git_github_pat_c[@]}" -c core.sparseCheckout=false -c index.sparse=false submodule add "${submod_args[@]}" -- "$url" "$path"; then - echo "submodulize: failed to add submodule $path ← $url" >&2 - echo " If the repo is private: configure HTTPS credentials, or re-run with --ssh (needs GitHub SSH access)." >&2 - echo " After a failed add you may need: git submodule deinit -f -- $path 2>/dev/null; rm -rf .git/modules/$path $path" >&2 - exit 1 - fi - fi -done < "$MANIFEST" - -if [[ "$manifest_entries" -eq 0 ]]; then - echo "No entries in manifest." >&2 - exit 1 -fi - -if $DRY_RUN; then - echo "Dry run complete." - exit 0 -fi - -if ! $NO_COMMIT; then - if git diff --cached --quiet 2>/dev/null; then - echo "Nothing staged; skipping commit." - else - git commit -m "chore: add plugin submodules per plugin-submodules.manifest" - fi -fi - -echo "Done. Submodule layout is ready (clean repo)." diff --git a/cleandev/unsubmodulize.sh b/cleandev/unsubmodulize.sh deleted file mode 100644 index 2e00cff..0000000 --- a/cleandev/unsubmodulize.sh +++ /dev/null @@ -1,175 +0,0 @@ -#!/usr/bin/env bash -# Replace git submodules with plain tracked directories (vendored plugins), matching lsuce-moodle -# develop style. Clones each plugin repo at the given branch, drops nested .git, and adds files -# to the parent Moodle repository. -# -# Usage: -# ./cleandev/unsubmodulize.sh [--dry-run] [--no-commit] [--ssh] [--manifest PATH] [--repo ROOT] -# -# Private GitHub repos over HTTPS: set GITHUB_TOKEN (PAT) so ls-remote / clone use -# -c url.https://TOKEN@github.com/.insteadOf=https://github.com/ -# (same as submodulize.sh). Or use --ssh. - -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -MANIFEST="${SCRIPT_DIR}/plugin-submodules.manifest" -DRY_RUN=false -NO_COMMIT=false -USE_SSH=false -REPO_ROOT="" - -while [[ $# -gt 0 ]]; do - case "$1" in - --dry-run) DRY_RUN=true; shift ;; - --no-commit) NO_COMMIT=true; shift ;; - --ssh) USE_SSH=true; shift ;; - --manifest) - MANIFEST="${2:?}" - shift 2 - ;; - --repo) - REPO_ROOT="${2:?}" - shift 2 - ;; - -h|--help) - sed -n '1,25p' "$0" - exit 0 - ;; - *) - echo "Unknown option: $1" >&2 - exit 1 - ;; - esac -done - -if [[ ! -f "$MANIFEST" ]]; then - echo "Manifest not found: $MANIFEST" >&2 - exit 1 -fi - -if [[ -z "$REPO_ROOT" ]]; then - REPO_ROOT="$(git rev-parse --show-toplevel 2>/dev/null)" || { - echo "Not inside a git repository. Use --repo /path/to/moodle" >&2 - exit 1 - } -fi - -cd "$REPO_ROOT" - -disable_sparse_checkout_if_needed() { - $DRY_RUN && return 0 - local active= - [[ -f .git/info/sparse-checkout ]] && active=1 - [[ "$(git config --bool core.sparseCheckout 2>/dev/null)" == "true" ]] && active=1 - [[ "$(git config --bool index.sparse 2>/dev/null)" == "true" ]] && active=1 - if [[ -z "$active" ]] && command -v git >/dev/null; then - local listed - listed="$(git sparse-checkout list 2>/dev/null | head -n 1 || true)" - [[ -n "${listed// }" ]] && active=1 - fi - [[ -z "$active" ]] && return 0 - echo "Disabling sparse-checkout so plugin paths can be vendored." >&2 - git sparse-checkout disable 2>/dev/null || true - git config core.sparseCheckout false 2>/dev/null || true - git config --unset-all core.sparseCheckoutCone 2>/dev/null || true - git config index.sparse false 2>/dev/null || true - rm -f .git/info/sparse-checkout -} - -disable_sparse_checkout_if_needed - -# Extra -c flags so clone honors GitHub PAT (superproject local url.insteadOf is not always used here). -git_github_pat_c=() -if [[ -n "${GITHUB_TOKEN:-}" ]]; then - git_github_pat_c+=(-c "url.https://${GITHUB_TOKEN}@github.com/.insteadOf=https://github.com/") -fi - -rewrite_github_url_to_ssh() { - local u="$1" - if $USE_SSH && [[ "$u" == https://github.com/* ]]; then - printf '%s\n' "git@github.com:${u#https://github.com/}" - else - printf '%s\n' "$u" - fi -} - -manifest_entries=0 -while IFS='|' read -r raw_path raw_url raw_branch; do - path="${raw_path#"${raw_path%%[![:space:]]*}"}" - path="${path%"${path##*[![:space:]]}"}" - url="${raw_url#"${raw_url%%[![:space:]]*}"}" - url="${url%"${url##*[![:space:]]}"}" - branch="${raw_branch#"${raw_branch%%[![:space:]]*}"}" - branch="${branch%"${branch##*[![:space:]]}"}" - - [[ -z "$path" || "$path" =~ ^# ]] && continue - [[ -z "$url" ]] && { echo "Manifest: missing URL for path $path" >&2; exit 1; } - [[ -z "$branch" ]] && branch="main" - url="$(rewrite_github_url_to_ssh "$url")" - - ((++manifest_entries)) || true - - if [[ ! -f .gitmodules ]] || ! git config -f .gitmodules --get-regexp path 2>/dev/null | awk '{print $2}' | grep -Fxq "$path"; then - echo "Not listed as submodule in .gitmodules: $path — skipping (already vendored or unknown)" - continue - fi - - if ! $DRY_RUN; then - if [[ -d "$path" ]] && (cd "$path" && git status --porcelain 2>/dev/null | grep -q .); then - echo "Submodule $path has local changes — commit/push inside submodule or stash first." >&2 - exit 1 - fi - fi - - echo "Unsubmodulizing: $path (from $url @ $branch)" - - if $DRY_RUN; then - printf '[dry-run] would: clone %s @ %s → %s, deinit submodule, rm .git, git add\n' "$url" "$branch" "$path" - continue - fi - - tmp="$(mktemp -d "${TMPDIR:-/tmp}/unsubmodulize.XXXXXX")" - trap 'rm -rf "$tmp"' EXIT - if [[ -n "$branch" ]] && GIT_TERMINAL_PROMPT=0 git "${git_github_pat_c[@]}" ls-remote --heads "$url" "refs/heads/$branch" 2>/dev/null | grep -q .; then - GIT_TERMINAL_PROMPT=0 git "${git_github_pat_c[@]}" clone --depth 1 -b "$branch" -- "$url" "$tmp/clone" - else - [[ -n "$branch" ]] && echo "Remote has no branch '$branch' for $path; cloning default branch." >&2 - GIT_TERMINAL_PROMPT=0 git "${git_github_pat_c[@]}" clone --depth 1 -- "$url" "$tmp/clone" - fi - git submodule deinit -f -- "$path" - git rm -f --sparse -- "$path" 2>/dev/null || git rm -f -- "$path" - mod_gitdir="$(git rev-parse --git-path "modules/$path")" - if [[ -n "$mod_gitdir" && -e "$mod_gitdir" ]]; then - rm -rf -- "$mod_gitdir" - fi - parent="$(dirname "$path")" - [[ "$parent" != "." ]] && mkdir -p "$parent" - rm -rf -- "$path" - cp -a "$tmp/clone/." "$path/" - rm -rf -- "$path/.git" - trap - EXIT - rm -rf "$tmp" - - git -c core.sparseCheckout=false -c index.sparse=false add -- "$path" -done < "$MANIFEST" - -if [[ "$manifest_entries" -eq 0 ]]; then - echo "No entries in manifest." >&2 - exit 1 -fi - -if $DRY_RUN; then - echo "Dry run complete." - exit 0 -fi - -if ! $NO_COMMIT; then - if git diff --cached --quiet 2>/dev/null; then - echo "Nothing staged; skipping commit." - else - git commit -m "chore: vendor plugin trees (remove submodules per manifest)" - fi -fi - -echo "Done. Plugin directories are plain files (dirty / upstream Moodle style)." diff --git a/docker-compose.yml b/docker-compose.yml index b514d36..7023f5d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,11 +15,16 @@ services: retries: 10 web: - # Moodle 4.5.x is supported through PHP 8.3. PHP 8.4+ can break CLI install with: - # Exception - Cannot compile closures which import variables using the `use` keyword - # (Laravel SerializableClosure). Override if your registry publishes a PHP 8.3 image, e.g.: - # MOODLE_DEV_IMAGE=lsuonline/moodle-dev:php83 docker compose -p NAME up -d - image: ${MOODLE_DEV_IMAGE:-lsuonline/moodle-dev:latest} + # Build from ./Dockerfile (php:8.3-apache). lsuonline/moodle-dev:latest has shipped PHP 8.4, which + # breaks Moodle 4.5 CLI install: "Cannot compile closures which import variables using the use keyword". + # To use a pre-built image instead, remove `build:` below and set `image: your-registry/moodle-dev:tag`. + # `secrets:` forwards GITHUB_TOKEN to the build so the Dockerfile can clone the private + # lsuonline/moodleus repo. new.sh resolves+exports GITHUB_TOKEN before `docker compose up`. + build: + context: . + secrets: + - github_token + image: lsuonline/moodle-dev:latest container_name: ${BRANCH_NAME}-moodle depends_on: db: @@ -88,4 +93,8 @@ services: networks: traefik: - external: true \ No newline at end of file + external: true + +secrets: + github_token: + environment: GITHUB_TOKEN \ No newline at end of file diff --git a/new.sh b/new.sh index 2252ed1..12821ff 100755 --- a/new.sh +++ b/new.sh @@ -88,7 +88,7 @@ ensure_traefik_running() { # Usage: set_config [component] name value # - If component is empty or "-", sets a core config resolve_submod_github_token() { - local github_token_file="${PROJECT_ROOT}/cleandev/.github-token" + local github_token_file="${PROJECT_ROOT}/submodulizer-local/.github-token" SUBMOD_GIT_TOKEN="" if [ "${SUBMODULIZE_SSH:-}" = "1" ]; then return 0 @@ -109,20 +109,20 @@ resolve_submod_github_token() { fi if [ -t 0 ] && [ -t 1 ]; then echo "GitHub personal access token needed for private submodule repos (read access to org repos)." >&2 - echo "Saved to cleandev/.github-token (gitignored). Use SUBMODULIZE_SSH=1 instead if you use SSH in the container." >&2 + echo "Saved to submodulizer-local/.github-token (gitignored). Use SUBMODULIZE_SSH=1 instead if you use SSH in the container." >&2 read -r -s -p "GitHub token: " SUBMOD_GIT_TOKEN echo >&2 if [ -n "$SUBMOD_GIT_TOKEN" ]; then mkdir -p "$(dirname "$github_token_file")" ( umask 077 && printf '%s\n' "$SUBMOD_GIT_TOKEN" >"$github_token_file" ) chmod 600 "$github_token_file" 2>/dev/null || true - echo "Token stored in cleandev/.github-token" >&2 + echo "Token stored in submodulizer-local/.github-token" >&2 fi fi if [ -n "$SUBMOD_GIT_TOKEN" ]; then return 0 fi - echo "new.sh: No GitHub token for --submodulize. Options: export GITHUB_TOKEN or GH_TOKEN, create cleandev/.github-token," >&2 + echo "new.sh: No GitHub token for --submodulize. Options: export GITHUB_TOKEN or GH_TOKEN, create submodulizer-local/.github-token," >&2 echo " run this script in a terminal (interactive prompt), or set SUBMODULIZE_SSH=1 for SSH URLs." >&2 exit 1 } @@ -167,8 +167,9 @@ while [[ $# -gt 0 ]]; do Options: -h --help Shows this text. -s --skip Skip automatic Moodle installation. - --submodulize After merge, use cleandev/submodulize.sh for manifest plugins (submodules). - GitHub auth (private lsuonline/*): GITHUB_TOKEN or GH_TOKEN, file cleandev/.github-token, + --submodulize After merge, run submodulizer/submodulize.sh --no-replay for manifest plugins (submodules). + Safe on cleandev-style trees: paths already in .gitmodules are skipped (no bulk no-op run). + GitHub auth (private lsuonline/*): GITHUB_TOKEN or GH_TOKEN, file submodulizer-local/.github-token, or an interactive prompt (first run) saves the token there (gitignored). Alternatively SUBMODULIZE_SSH=1 with SSH usable inside the container."; exit; @@ -195,9 +196,17 @@ if [ -z "$NAME" ]; then NAME=$(dd if=/dev/urandom bs=2 count=1 2>/dev/null | od -An -t x1 | tr -d ' \n') fi -# Populated when --submodulize runs (env, cleandev/.github-token, or prompt). +# Populated when --submodulize runs (env, submodulizer-local/.github-token, or prompt). SUBMOD_GIT_TOKEN="" +# Resolve a GitHub token now: the Docker build clones the private lsuonline/moodleus +# repo, so GITHUB_TOKEN must be exported before `docker compose up` so the +# `github_token` build secret declared in docker-compose.yml can pick it up. +# The same token is reused for --submodulize further down (no second prompt). +# shellcheck source=submodulizer-local/lib-github-token.sh +. "${PROJECT_ROOT}/submodulizer-local/lib-github-token.sh" +resolve_github_token || exit 1 + # Ensure Traefik infrastructure is ready ensure_traefik_network ensure_traefik_running @@ -208,15 +217,16 @@ BRANCH_NAME=$NAME docker compose -p "${NAME}" up -d # with block_lsu_people). skip-worktree on ues_people and config.php for a clean git status after startup. # enrol/workdaystudent and blocks/wdsprefs come from the lsuce-moodle tree (or --submodulize manifest). if [ "$SUBMODULIZE" = true ]; then - if [ ! -f "${PROJECT_ROOT}/cleandev/submodulize.sh" ] || [ ! -f "${PROJECT_ROOT}/cleandev/plugin-submodules.manifest" ]; then - echo "new.sh --submodulize requires cleandev/submodulize.sh and cleandev/plugin-submodules.manifest next to new.sh." >&2 + if [ ! -f "${PROJECT_ROOT}/submodulizer/submodulize.sh" ] || [ ! -f "${PROJECT_ROOT}/submodulizer-local/manifest-submodulize-redundant.sh" ] || [ ! -f "${PROJECT_ROOT}/submodulizer-local/plugin-submodules.manifest" ]; then + echo "new.sh --submodulize requires submodulizer/submodulize.sh (submodule; run 'git submodule update --init --recursive'), submodulizer-local/manifest-submodulize-redundant.sh, and submodulizer-local/plugin-submodules.manifest." >&2 exit 1 fi resolve_submod_github_token - MSYS_NO_PATHCONV=1 docker cp "$(host_path_for_docker_cp "${PROJECT_ROOT}/cleandev/submodulize.sh")" "${NAME}-moodle:/tmp/submodulize.sh" - MSYS_NO_PATHCONV=1 docker cp "$(host_path_for_docker_cp "${PROJECT_ROOT}/cleandev/plugin-submodules.manifest")" "${NAME}-moodle:/tmp/plugin-submodules.manifest" + MSYS_NO_PATHCONV=1 docker cp "$(host_path_for_docker_cp "${PROJECT_ROOT}/submodulizer/submodulize.sh")" "${NAME}-moodle:/tmp/submodulize.sh" + MSYS_NO_PATHCONV=1 docker cp "$(host_path_for_docker_cp "${PROJECT_ROOT}/submodulizer-local/manifest-submodulize-redundant.sh")" "${NAME}-moodle:/tmp/manifest-submodulize-redundant.sh" + MSYS_NO_PATHCONV=1 docker cp "$(host_path_for_docker_cp "${PROJECT_ROOT}/submodulizer-local/plugin-submodules.manifest")" "${NAME}-moodle:/tmp/plugin-submodules.manifest" # docker cp leaves root-owned files; sed -i as www-data fails with "cannot rename: Operation not permitted" - MSYS_NO_PATHCONV=1 docker exec "${NAME}-moodle" sh -c 'sed -i '"'"'s/\r$//'"'"' /tmp/submodulize.sh /tmp/plugin-submodules.manifest && chmod +x /tmp/submodulize.sh && chown www-data:www-data /tmp/submodulize.sh /tmp/plugin-submodules.manifest' + MSYS_NO_PATHCONV=1 docker exec "${NAME}-moodle" sh -c 'sed -i '"'"'s/\r$//'"'"' /tmp/submodulize.sh /tmp/manifest-submodulize-redundant.sh /tmp/plugin-submodules.manifest && chmod +x /tmp/submodulize.sh /tmp/manifest-submodulize-redundant.sh && chown www-data:www-data /tmp/submodulize.sh /tmp/manifest-submodulize-redundant.sh /tmp/plugin-submodules.manifest' submod_docker_env=(-u www-data) if [ -n "$SUBMOD_GIT_TOKEN" ]; then submod_docker_env+=(-e "GITHUB_TOKEN=${SUBMOD_GIT_TOKEN}") @@ -224,31 +234,46 @@ if [ "$SUBMODULIZE" = true ]; then if [ "${SUBMODULIZE_SSH:-}" = "1" ]; then submod_docker_env+=(-e "SUBMODULIZE_USE_SSH=1") fi - MSYS_NO_PATHCONV=1 docker exec "${submod_docker_env[@]}" "${NAME}-moodle" sh -c ' - cd /var/www/html && \ - git config --add safe.directory /var/www/html && \ - if [ -n "${GITHUB_TOKEN:-}" ]; then \ - git config --local url."https://${GITHUB_TOKEN}@github.com/".insteadOf "https://github.com/"; \ - fi && \ - rm -rf blocks/ues_people && \ - git fetch origin develop && \ - git merge origin/develop && \ - git ls-files blocks/ues_people | xargs -r git update-index --skip-worktree && \ - rm -rf blocks/ues_people && \ - SMF="--no-commit" && \ - if [ "${SUBMODULIZE_USE_SSH:-}" = "1" ]; then SMF="$SMF --ssh"; fi && \ - bash /tmp/submodulize.sh --repo /var/www/html --manifest /tmp/plugin-submodules.manifest $SMF && \ + MSYS_NO_PATHCONV=1 docker exec "${submod_docker_env[@]}" "${NAME}-moodle" bash -c ' + set -euo pipefail + cd /var/www/html + git config --add safe.directory /var/www/html + if [ -n "${GITHUB_TOKEN:-}" ]; then + git config --local url."https://smatts3%40lsu.edu:${GITHUB_TOKEN}@github.com/".insteadOf "https://github.com/" + fi + rm -rf blocks/ues_people + git fetch origin MOODLE_405_MAIN + git merge origin/MOODLE_405_MAIN + git ls-files blocks/ues_people | xargs -r git update-index --skip-worktree + rm -rf blocks/ues_people + cp /tmp/plugin-submodules.manifest /var/www/html/plugin-submodules.manifest + SMF="--no-commit" + if [ "${SUBMODULIZE_USE_SSH:-}" = "1" ]; then SMF="$SMF --ssh"; fi + if bash /tmp/manifest-submodulize-redundant.sh --repo /var/www/html; then + echo "new.sh: All manifest plugin paths are already submodules; skipping submodulize.sh." + else + bash /tmp/submodulize.sh --no-replay --repo /var/www/html $SMF + fi git update-index --skip-worktree config.php ' else - MSYS_NO_PATHCONV=1 docker exec -u www-data "${NAME}-moodle" sh -c ' - cd /var/www/html && \ - git config --add safe.directory /var/www/html && \ - rm -rf blocks/ues_people && \ - git fetch origin develop && \ - git merge origin/develop && \ - git ls-files blocks/ues_people | xargs -r git update-index --skip-worktree && \ - rm -rf blocks/ues_people && \ + # Pass GITHUB_TOKEN into the container so the fetch against the private + # lsuonline/moodleus can authenticate. Once set, `git config --local + # url...insteadOf` writes the rewrite into /var/www/html/.git/config so + # subsequent in-container git operations (e.g. moodle-pull) work without + # re-supplying the token. + MSYS_NO_PATHCONV=1 docker exec -u www-data -e "GITHUB_TOKEN=${GITHUB_TOKEN:-}" "${NAME}-moodle" sh -c ' + set -e + cd /var/www/html + git config --add safe.directory /var/www/html + if [ -n "${GITHUB_TOKEN:-}" ]; then + git config --local url."https://smatts3%40lsu.edu:${GITHUB_TOKEN}@github.com/".insteadOf "https://github.com/" + fi + rm -rf blocks/ues_people + git fetch origin MOODLE_405_MAIN + git merge origin/MOODLE_405_MAIN + git ls-files blocks/ues_people | xargs -r git update-index --skip-worktree + rm -rf blocks/ues_people git update-index --skip-worktree config.php ' fi diff --git a/submodulizer b/submodulizer new file mode 160000 index 0000000..feaac0e --- /dev/null +++ b/submodulizer @@ -0,0 +1 @@ +Subproject commit feaac0ed5042a9dfa37e72090468921da5aea460 diff --git a/cleandev/TEAM-PROCESS.md b/submodulizer-local/TEAM-PROCESS.md similarity index 58% rename from cleandev/TEAM-PROCESS.md rename to submodulizer-local/TEAM-PROCESS.md index 62279f1..c4459b4 100644 --- a/cleandev/TEAM-PROCESS.md +++ b/submodulizer-local/TEAM-PROCESS.md @@ -2,6 +2,10 @@ This document covers **branch policy** for [lsuce-moodle](https://github.com/lsuonline/lsuce-moodle), the **Docker image decision** for this repo, and **how to refresh** `plugin-submodules.manifest` from the plugin inventory CSV. +**Where the manifest lives:** On **cleandev**, treat `plugin-submodules.manifest` as part of the Moodle superproject (repo root), alongside `.gitmodules`. The copy in **this** repo under [`submodulizer-local/plugin-submodules.manifest`](plugin-submodules.manifest) is the maintained source for CSV refresh and CI lint; `submodulize.sh` / `unsubmodulize.sh` default to `ROOT/plugin-submodules.manifest` unless you pass `--manifest`. + +**Branch replay (default):** `unsubmodulize.sh` and `submodulize.sh` run replay unless you pass **`--no-replay`**. Replay builds histories with **one Moodle superproject commit per plugin-repo commit** (chronological order, carry-forward). **`--fork-point`** defaults to local **`master`** (else **`main`**) when omitted, when the script can infer it; otherwise pass it explicitly. **`--source`** defaults when omitted: **unsub** prefers **`unsubmodulized`**, then **`master`**, **`main`**, **`submodulized`**; **sub** prefers **`submodulized`**, **`master`**, **`main`**. Source tips may be **gitlinks or vendored trees** (tree matched to plugin commits). Defaults **`--target`** branch names (`submodulized`, `unsubmodulized`). If the fork-point checkout is **vendored** at a manifest path, the script matches the embedded tree to a plugin commit (or use `--plugin-base path=SHA`). `new.sh --submodulize` runs **`submodulize.sh --no-replay`** (one-shot only). + --- ## 1. Team process (develop ↔ cleandev) @@ -28,8 +32,8 @@ This document covers **branch policy** for [lsuce-moodle](https://github.com/lsu ### Local Docker stack (`new.sh`) - **Without `--submodulize`:** Container tracks **vendored** layout after merge from `origin/develop` (matches **develop**). -- **With `--submodulize`:** After that merge, runs `submodulize.sh` so manifest-listed paths become **submodules** (closer to **cleandev**). Requires `GITHUB_TOKEN` / `cleandev/.github-token` or `SUBMODULIZE_SSH=1` for private GitHub repos. -- Never commit **`cleandev/.github-token`** (gitignored). +- **With `--submodulize`:** After that merge, runs `submodulize.sh --no-replay` so manifest-listed paths become **submodules** (closer to **cleandev**). Requires `GITHUB_TOKEN` / `submodulizer-local/.github-token` or `SUBMODULIZE_SSH=1` for private GitHub repos. +- Never commit **`submodulizer-local/.github-token`** (gitignored). ### Submodule hygiene @@ -90,7 +94,33 @@ Do **not** add separate submodule lines per element subdirectory. ### Monorepos -If **one Git repository** contains **multiple** top-level Moodle plugin paths (e.g. Kaltura, Microsoft o365-moodle, `lsu-enrol_ues`), **do not** add one manifest line per path with the same URL: `submodulize.sh` cannot “clone once, map subpaths.” Keep a **commented block** in `plugin-submodules.manifest` listing the URL and paths (see existing `# --- Monorepos ---` section), and leave those plugins **vendored** or handle them with a **manual** procedure until tooling supports it. +If **one Git repository** contains **multiple** top-level Moodle plugin paths (e.g. Kaltura, Microsoft o365-moodle, `lsu-enrol_ues`), **do not** add one manifest line per path with the same URL: `submodulize.sh` cannot “clone once, map subpaths.” Keep a **commented block** in `plugin-submodules.manifest` listing the URL and paths (see `# --- Monorepos ---` in the manifest). Treat those directories as **vendored** in the superproject unless you adopt an advanced layout (see below). + +#### One-shot: refresh vendored monorepo plugins (develop or cleandev) + +Use this when upstream shipped changes and you need the same Moodle paths updated without submodule tooling. + +1. **Note the source** from the manifest comment block (clone URL and the list of Moodle-relative paths, e.g. `mod/kalvidassign`). +2. **Clone upstream** somewhere outside the Moodle tree (temp is fine), on the branch or tag you intend to ship: + ```bash + git clone --depth 1 -b BRANCH_OR_TAG https://github.com/org/monorepo.git /tmp/monorepo-src + ``` + Use a full clone if you need history or a non-default branch tip. +3. **Map paths** — upstream layout varies by project. Under `/tmp/monorepo-src`, locate the directory that corresponds to each Moodle path (often the path matches the repo tree; if the project nests plugins under `moodle/` or similar, copy from there). +4. **Copy into lsuce-moodle** from the Moodle repo root, one path at a time (adjust source side to match step 3): + ```bash + rsync -a --delete /tmp/monorepo-src/mod/kalvidassign/ mod/kalvidassign/ + ``` + Prefer `--delete` only when you intend to mirror upstream exactly; otherwise omit it. On Windows without `rsync`, use a graphical diff tool or `cp -r` with care. +5. **Review** (`git status`, smoke test in Moodle), then **commit** in **lsuce-moodle** (single commit per upstream bump or per path—follow team convention). + +Repeat for each path listed under that URL in the manifest comment. Bundles such as **Kaltura** (`moodle_plugin`), **Kaltura gallery** (`moodle-local_kalturamediagallery`), **o365-moodle**, and **lsu-enrol_ues** are independent clones; refresh each comment group from its own remote. + +#### One-shot: cleandev when monorepo paths stay vendored + +On **cleandev**, submodule-backed plugins use `.gitmodules`; monorepo-backed paths stay **normal tracked directories** in the superproject (no submodule entry). Do not add duplicate manifest active lines for the same URL. After `git submodule update --init`, those paths behave like core tree: edit, commit, and push on **lsuce-moodle** unless your team splits them out later. + +*Automated “clone once, map subpaths” for the manifest is still not implemented; the steps above are the supported manual approach.* ### No usable HTTPS URL @@ -98,7 +128,7 @@ If the CSV has no working public URL (internal, defunct, empty), add or keep a * ### After editing the manifest -1. Run **`submodulize.sh --dry-run`** from a **test clone** of lsuce-moodle to sanity-check paths and remotes. +1. Run **`submodulize.sh --no-replay --dry-run`** from a **test clone** of lsuce-moodle to sanity-check paths and remotes. 2. Commit manifest changes in **this** repo (`lsuce_moodle_project`) when the tooling repo owns the file; if the manifest is versioned only in lsuce-moodle, commit there per your layout. For script flags and `GITHUB_TOKEN` behavior, see the headers in `submodulize.sh` and `unsubmodulize.sh`. diff --git a/cleandev/docker-compose.github-runner.yml b/submodulizer-local/docker-compose.github-runner.yml similarity index 100% rename from cleandev/docker-compose.github-runner.yml rename to submodulizer-local/docker-compose.github-runner.yml diff --git a/cleandev/github-actions.workflow.example.yml b/submodulizer-local/github-actions.workflow.example.yml similarity index 100% rename from cleandev/github-actions.workflow.example.yml rename to submodulizer-local/github-actions.workflow.example.yml diff --git a/cleandev/github-runner.env.example b/submodulizer-local/github-runner.env.example similarity index 100% rename from cleandev/github-runner.env.example rename to submodulizer-local/github-runner.env.example diff --git a/submodulizer-local/lib-github-token.sh b/submodulizer-local/lib-github-token.sh new file mode 100644 index 0000000..42773d6 --- /dev/null +++ b/submodulizer-local/lib-github-token.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +# lib-github-token.sh -- sourced by new.sh and build.sh. +# +# Defines resolve_github_token(): on success, GITHUB_TOKEN is set and exported, +# so docker compose / docker build can forward it to the BuildKit secret +# `github_token` consumed by Dockerfile. Returns non-zero (and prints guidance) +# when no source provides a token. +# +# Sources, in order: +# 1. GITHUB_TOKEN env var (already set) +# 2. GH_TOKEN env var +# 3. submodulizer-local/.github-token file (next to this lib) +# 4. Interactive prompt (TTY only); persists to the file above (chmod 600) +# +# SUBMODULIZE_SSH=1 is intentionally NOT honored here: the Docker build always +# clones lsuonline/moodleus over HTTPS, regardless of how the in-container +# submodulize step in new.sh chooses to authenticate later. + +resolve_github_token() { + local lib_dir token_file value + lib_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + token_file="${lib_dir}/.github-token" + + if [ -n "${GITHUB_TOKEN:-}" ]; then + export GITHUB_TOKEN + return 0 + fi + if [ -n "${GH_TOKEN:-}" ]; then + GITHUB_TOKEN="$GH_TOKEN" + export GITHUB_TOKEN + return 0 + fi + if [ -f "$token_file" ]; then + value="$( + grep -v '^[[:space:]]*#' "$token_file" 2>/dev/null \ + | grep -v '^[[:space:]]*$' | head -n1 | tr -d '\r' \ + | sed 's/^[[:space:]]*//;s/[[:space:]]*$//' + )" + if [ -n "$value" ]; then + GITHUB_TOKEN="$value" + export GITHUB_TOKEN + return 0 + fi + fi + if [ -t 0 ] && [ -t 1 ]; then + echo "GitHub personal access token needed: the Docker build clones private lsuonline/moodleus." >&2 + echo "Read scope on lsuonline/* repos is enough. Token will be saved to" >&2 + echo " ${token_file} (gitignored, chmod 600)." >&2 + read -r -s -p "GitHub token: " value + echo >&2 + if [ -n "$value" ]; then + mkdir -p "$(dirname "$token_file")" + ( umask 077 && printf '%s\n' "$value" >"$token_file" ) + chmod 600 "$token_file" 2>/dev/null || true + echo "Token stored in submodulizer-local/.github-token" >&2 + GITHUB_TOKEN="$value" + export GITHUB_TOKEN + return 0 + fi + fi + echo "lib-github-token.sh: No GitHub token resolved. lsuonline/moodleus is private," >&2 + echo " so the Docker build cannot clone it. Options:" >&2 + echo " - export GITHUB_TOKEN (or GH_TOKEN) before re-running" >&2 + echo " - create ${token_file}" >&2 + echo " - run the script in a terminal to be prompted" >&2 + return 1 +} diff --git a/submodulizer-local/manifest-submodulize-redundant.sh b/submodulizer-local/manifest-submodulize-redundant.sh new file mode 100644 index 0000000..a49e8a1 --- /dev/null +++ b/submodulizer-local/manifest-submodulize-redundant.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash +# Exit 0 if running submodulize.sh would be a no-op: every active manifest path is already listed +# in .gitmodules and .gitmodules is non-empty. Exit 1 otherwise (submodulize should run). +# Used by new.sh --submodulize and submodulizer/tests. +# Default manifest: ROOT/plugin-submodules.manifest (same as submodulize.sh). +# +# Usage: ./submodulizer-local/manifest-submodulize-redundant.sh --repo ROOT [--manifest PATH] +set -euo pipefail + +REPO_ROOT="" +MANIFEST="" +MANIFEST_EXPLICIT=false + +while [[ $# -gt 0 ]]; do + case "$1" in + --repo) + REPO_ROOT="${2:?}" + shift 2 + ;; + --manifest) + MANIFEST="${2:?}" + MANIFEST_EXPLICIT=true + shift 2 + ;; + -h|--help) + sed -n '1,13p' "$0" + exit 0 + ;; + *) + echo "Unknown option: $1" >&2 + exit 2 + ;; + esac +done + +[[ -n "$REPO_ROOT" ]] || { + echo "Usage: $0 --repo ROOT [--manifest PATH]" >&2 + echo " Default manifest: ROOT/plugin-submodules.manifest" >&2 + exit 2 +} + +if ! $MANIFEST_EXPLICIT; then + MANIFEST="${REPO_ROOT%/}/plugin-submodules.manifest" +fi + +[[ -f "$MANIFEST" ]] || { + echo "Manifest not found: $MANIFEST" >&2 + exit 2 +} + +cd "$REPO_ROOT" + +need_submod=0 +active=0 +while IFS="|" read -r p _rest || [[ -n "${p:-}" ]]; do + case "${p#[[:space:]]}" in + ""|\#*) continue ;; + esac + p="${p#"${p%%[![:space:]]*}"}" + p="${p%"${p##*[![:space:]]}"}" + case "$p" in + ""|\#*) continue ;; + esac + active=$((active + 1)) + if ! git config -f .gitmodules --get-regexp path 2>/dev/null | awk '{print $2}' | grep -Fxq "$p"; then + need_submod=1 + break + fi +done < "$MANIFEST" + +if [[ "$active" -eq 0 ]]; then + exit 1 +fi +if [[ "$need_submod" -eq 0 && -s .gitmodules ]]; then + exit 0 +fi +exit 1 diff --git a/cleandev/plugin-submodules.manifest b/submodulizer-local/plugin-submodules.manifest similarity index 99% rename from cleandev/plugin-submodules.manifest rename to submodulizer-local/plugin-submodules.manifest index c418b9c..628f399 100644 --- a/cleandev/plugin-submodules.manifest +++ b/submodulizer-local/plugin-submodules.manifest @@ -5,7 +5,7 @@ # # Synced from repo CSV: Evaluation of LSU moodle messy plugins(List) (3).csv # Custom URL preferred over third-party when both are set. -# How to refresh from CSV and team policy: cleandev/TEAM-PROCESS.md +# How to refresh from CSV and team policy: submodulizer-local/TEAM-PROCESS.md # # --- One repo per path (compatible with submodulize.sh / unsubmodulize.sh) ---