diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b8bcd6c..ac1cc36 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,10 +20,10 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v6 - name: Setup Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: "3.10" @@ -43,10 +43,10 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v6 - name: Setup Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: "3.10" cache: pip @@ -67,10 +67,10 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v6 - name: Setup Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: "3.10" cache: pip @@ -95,10 +95,10 @@ jobs: python-version: ["3.10", "3.11"] steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v6 - name: Setup Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} cache: pip @@ -119,10 +119,10 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v6 - name: Setup Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: "3.10" cache: pip diff --git a/.github/workflows/wiki-sync.yml b/.github/workflows/wiki-sync.yml index 1b7c641..865c669 100644 --- a/.github/workflows/wiki-sync.yml +++ b/.github/workflows/wiki-sync.yml @@ -25,7 +25,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v6 - name: Publish wiki pages env: diff --git a/AGENTS.md b/AGENTS.md index 16733d6..19f3bc8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -18,7 +18,7 @@ Use the `Makefile` as the source of truth: - `make setup-infer`: install inference/deploy extras - `make install-server-system`: install Ubuntu/Debian system packages - `make install-systemd-services`: install/reload systemd service units for long-running workers -- `make install-user-systemd-services`: install/reload user-level systemd service units (no sudo path) +- `make install-user-systemd-services`: install/reload user-level systemd service units (no sudo path; includes shard/checkpoint timers) - `make doctor`: environment/tooling diagnostics - `make test`: run `unittest` test suite - `make lint`: run Ruff lint checks @@ -40,13 +40,16 @@ Use the `Makefile` as the source of truth: - `make fineweb-parquet-to-shards`: usage helper for direct FineWeb parquet -> tokenizer -> shard conversion - `make fineweb-manifest-dedupe`: usage helper for overlap-manifest dedupe audit/fix - `make stage-fineweb-from-warm`: usage helper for staging FineWeb parquet chunks from warm to hot -- `make fineweb-prefetch-hot-queue`: usage helper for warm->hot queue prefetch worker - `make fineweb-revalidate-bad-parquet`: usage helper for revalidating/restaging bad parquet entries +- `make reconcile-offloaded-manifests`: usage helper for restoring risky offloaded manifests and optional bin rehydrate +- `make shard-offload-cycle`: usage helper for safe reconcile -> offload -> reconcile timer cycle - `make offload-shard-bins-warm`: usage helper for replacing older local shard `.bin` files with warm-storage symlinks, disabling offloaded manifests, gating offload by trained-batch registry, and honoring hot-coverage safety floors +- `make hot-shard-warmup`: usage helper for hydrating active shard bins from warm to hot storage (pre-chunk or background prefetch) - `make enforce-hot-manifests`: usage helper for disabling active manifests that reference symlinked shard bins - `make fineweb-stage-shard-loop`: usage helper for rolling warm->hot stage + shard + verify + sync + purge - `make fineweb-stage-shard-watchdog`: usage helper for auto-restart watchdog around the stage/shard loop - `make lr-sweep-350bt`: usage helper for RTX 5070 LR sweep on staged 350BT shards (`2e-4..4e-4`, ctx 512) +- `make benchmark-rtx5070`: usage helper for short context/batch throughput + GPU memory benchmark sweep - `make train-350bt-v2`: usage helper for the 350BT long-run launcher profile - `make train-350bt-ctx1024`: usage helper for context-extension continuation stage - `make train-supervisor-350bt`: usage helper for auto-resume chunked training that refreshes manifest set between cycles @@ -55,8 +58,10 @@ Use the `Makefile` as the source of truth: - `make pipeline-live`: usage helper for a live terminal pipeline dashboard - `make shard-corpus-batch`: usage helper for batch sharding with a shared tokenizer - `make checkpoint-offload-prune`: usage helper for checkpoint warm sync + local prune policy +- `make checkpoint-step-offload`: usage helper for offloading older `ckpt_step_*.pt` files to warm storage while preserving newest local resume steps - `make set-swappiness`: usage helper for host swappiness tuning (root) - `make hf-download-resumable`: usage helper for self-healing Hugging Face resume-download worker +- `make hf-download-fineweb-edu-full`: usage helper for simple full FineWeb-Edu parquet sync to Ceph/local dir - `make hf-download-watchdog`: usage helper for watchdog auto-restart around stalled/exited HF download workers - `make hf-prepare-publish`: usage helper for Hugging Face release bundle/publish - `make hf-download-model`: usage helper for full Hugging Face model snapshot download @@ -69,7 +74,7 @@ CI/CD workflows: - `.github/workflows/ci.yml`: script sanity (`bash -n` + `py_compile`), lint, typecheck, tests, smoke, and gate job - `.github/workflows/wiki-sync.yml`: publishes wiki pages on `main` doc changes - `.github/dependabot.yml`: weekly dependency update PRs for `pip`, `requirements/`, and GitHub Actions -- maintenance units: `deploy/systemd/llm-checkpoint-offload-prune.service` + `.timer`, `deploy/systemd/llm-bad-parquet-revalidate.service` + `.timer`, `deploy/systemd/llm-shard-offload.service` + `.timer`, `deploy/systemd/llm-vm-swappiness.service` +- maintenance units: `deploy/systemd/llm-checkpoint-offload-prune.service` + `.timer`, `deploy/systemd/llm-checkpoint-step-offload.service` + `.timer`, `deploy/systemd/llm-bad-parquet-revalidate.service` + `.timer`, `deploy/systemd/llm-shard-offload.service` + `.timer`, `deploy/systemd/llm-vm-swappiness.service` ## Coding Style & Naming Conventions - Python 3.10+, 4-space indentation, UTF-8 files @@ -122,11 +127,16 @@ Keep PR scope narrow; split refactors and features into separate PRs. - Cleanup defaults also apply normalized dedupe keys and contamination filtering; tune with `--no-dedupe-normalized`, `--no-drop-contamination`, `--contamination-pattern`, and `--contamination-patterns-file` - Use `bash scripts/sync_warm_storage.sh /mnt/ceph/llm/data` to copy local artifacts to warm storage - `sync_warm_storage.sh` covers raw + training data: `data/raw_zim`, `data/fineweb`, `data/cleaned`, `data/extracted`, `data/shards`, `data/shards_global`, `artifacts/tokenizer`, `artifacts/checkpoints`, `artifacts/reports` +- Before wiping/rebuilding a hot workspace: `git fetch --all --prune && git status --short --branch`, push all commits, run `bash scripts/sync_warm_storage.sh /mnt/ceph/llm/data`, and write a snapshot manifest with `git ls-files --others --ignored --exclude-standard | sort > /mnt/ceph/llm/data/logs/git_untracked_all_.txt` +- Do not wipe until `sync_warm_storage.sh` exits cleanly and `/mnt/ceph/llm/data/logs/last_sync_utc.txt` has a fresh timestamp - Use `bash scripts/zim_offload_worker.sh data/raw_zim /mnt/ceph/llm/data/raw_zim 120` for continuous hot->warm raw ZIM offload - Use `bash scripts/hydrate_from_warm_storage.sh /mnt/ceph/llm/data` to restore local artifacts from warm storage -- Use `python3 scripts/offload_shard_bins_to_warm.py --disable-offloaded-manifests --require-trained-batches-file /trained_batch_names.txt --min-active-manifests [--min-active-train-tokens ]` to move only already-trained older shard bins to warm storage while keeping active manifests hot-local only +- Use `python3 scripts/offload_shard_bins_to_warm.py --disable-offloaded-manifests --require-trained-batches-file /trained_batch_names.txt,/trained_batch_names.txt --skip-if-trained-file-missing --min-manifest-unique-input-files --min-active-manifests --min-active-train-tokens ` to move only already-trained older shard bins to warm storage while keeping active manifests hot-local only +- Use `python3 scripts/reconcile_offloaded_manifests.py --trained-batches-file /trained_batch_names.txt,/trained_batch_names.txt --skip-if-trained-file-missing --min-active-unique-input-files --max-restore --warm-shards-root /mnt/ceph/llm/data/shards_global/fineweb-global-bpe-v1 --rehydrate-restored-bins --rehydrate-active-symlink-bins` before offload runs to restore untrained/under-coverage offloaded manifests and rehydrate restored/active bins back to hot storage +- Prefer `bash scripts/shard_offload_cycle.sh` in automation (pre-reconcile -> gated offload -> post-reconcile -> enforce-hot-manifests) instead of direct one-shot offload calls - For bounded external pulls (for example FineWeb samples), use `python3 scripts/pull_hf_rows.py` and write to warm storage first - For long-running Hugging Face parquet pulls, use `scripts/hf_download_resumable.sh` instead of one-shot `hf download` (prefer `--enable-hf-transfer`, `--max-workers 6`, `--skip-dry-run`, and `--attempt-timeout-seconds` for 350BT-scale pulls) +- For simple full FineWeb-Edu pulls to Ceph, use `scripts/sync_fineweb_edu_full.sh /mnt/pve/cephfs/llm/data/fineweb/fineweb-edu-full` - For unattended long pulls, prefer `scripts/hf_download_watchdog.sh` to restart stalled/exited resumable workers based on progress checks (`--stall-seconds`, `--check-interval-seconds`) - For watchdog runs, set `--exit-on-complete` with `--expected-parquet-files` and/or `--expected-bytes` so completed pulls do not restart indefinitely - `hf_download_watchdog.sh` enforces a singleton lock (`.hf_download_watchdog.lock`) in the target local-dir @@ -135,28 +145,34 @@ Keep PR scope narrow; split refactors and features into separate PRs. - Use `stage_fineweb_from_warm.sh --copy-jobs ` to parallelize warm->hot staging copies - Use `stage_fineweb_from_warm.sh --min-free-gib ` to keep a hot-disk free-space floor during staging - Use `stage_fineweb_from_warm.sh --skip-list ` to avoid re-staging known-bad parquet basenames -- For long-running 350BT ingestion on limited hot disk, use `scripts/fineweb_stage_shard_loop.sh` for staged processing and automatic hot-space reclaim +- For long-running 350BT ingestion, use `scripts/fineweb_stage_shard_loop.sh` in direct-source mode (default: read parquet from Ceph path directly) and keep shard outputs hot-local +- `fineweb_stage_shard_loop.sh` supports automatic training-focused mode after full coverage (`--expected-unique-input-files 510` + default coverage-complete mode) to pause staging/sharding churn once all inputs are represented - For unattended long 350BT ingestion, run `scripts/fineweb_stage_shard_watchdog.sh` to auto-restart stage-loop worker exits/stalls +- `fineweb_stage_shard_watchdog.sh` now holds worker restarts when manifest unique-input coverage reaches target (`--expected-unique-input-files`) - For long shard batches, use a higher stage-watchdog stall timeout (for example `--stall-seconds 5400`) to avoid false restarts mid-batch - `fineweb_stage_shard_watchdog.sh` enforces a singleton lock in the stage state dir (`watchdog.lock`) and stops worker process groups (not only the parent shell) - `fineweb_stage_shard_watchdog.sh` progress snapshot includes hot `.incomplete` file count/bytes so active warm->hot copy phases are not treated as stalls - `fineweb_stage_shard_watchdog.sh` now also cleans stale stage-loop/shard-worker processes before relaunch so only one controller remains active - `fineweb_stage_shard_watchdog.sh` can adopt an already-running stage-loop controller (default) so watchdog restarts do not leave direct loop runs unmanaged; use `--no-adopt-existing-loop` to force fresh worker launch -- Use `fineweb_stage_shard_loop.sh --hot-queue-min-files ` to keep a bounded hot parquet queue and reduce sharder copy stalls -- `stage_fineweb_from_warm.sh` now uses a per-destination lock so stage-loop and prefetch worker staging calls serialize safely +- Use `fineweb_stage_shard_loop.sh --enable-stage-copy --hot-queue-min-files ` only when you explicitly want legacy warm->hot parquet staging +- `stage_fineweb_from_warm.sh` now uses a per-destination lock so concurrent staging calls serialize safely - `stage_fineweb_from_warm.sh --lock-wait-seconds 0` (default) skips quickly when another staging call holds the lock; tune if you want blocking behavior - `stage_fineweb_from_warm.sh` only applies rsync `--contimeout` for rsync-daemon sources (`rsync://` or `::`); local/NFS paths avoid this flag -- Use `fineweb_stage_shard_loop.sh --stage-copy-jobs ` to forward parallel staging copy workers into each stage cycle -- Use `fineweb_stage_shard_loop.sh --no-auto-tune-stage-copy-jobs` to pin copy workers, or keep default copy auto-tune enabled (`--auto-tune-min-copy-jobs`, `--auto-tune-max-copy-jobs`, `--auto-tune-iowait-low-pct`, `--auto-tune-iowait-high-pct`) for queue/iowait-driven staging throughput control -- Use `fineweb_stage_shard_loop.sh --stage-min-free-gib ` so staging never drives hot disk below a free-space guardrail +- Use `fineweb_stage_shard_loop.sh --enable-stage-copy --stage-copy-jobs ` to forward parallel staging copy workers into each stage cycle +- Use `fineweb_stage_shard_loop.sh --enable-stage-copy --no-auto-tune-stage-copy-jobs` to pin copy workers, or keep default copy auto-tune enabled (`--auto-tune-min-copy-jobs`, `--auto-tune-max-copy-jobs`, `--auto-tune-iowait-low-pct`, `--auto-tune-iowait-high-pct`) for queue/iowait-driven staging throughput control +- Use `fineweb_stage_shard_loop.sh --enable-stage-copy --stage-min-free-gib ` so staging never drives hot disk below a free-space guardrail - Use `fineweb_stage_shard_loop.sh --auto-tune-shard-jobs` to adapt shard parallelism and tokenizer threads from CPU load + batch runtime - Use `fineweb_stage_shard_loop.sh --sync-background --sync-max-inflight ` to overlap warm sync with next shard batches and reduce idle wait +- Keep hot disk near target by enabling loop auto-offload (`--hot-max-used-pct 95` + `--offload-check-interval-seconds 120`), which offloads already-trained older shard batches without pausing sharding - Prefer larger shard files for throughput (`--shard-size-tokens 20000000` for FineWeb 350BT pipeline) - `fineweb_stage_shard_loop.sh` now drains background sync jobs on `INT`/`TERM` for safer restarts - Default systemd loop/watchdog templates now use stage free-space guardrails + auto-tune; override with `LLM_STAGE_SHARD_LOOP_ARGS` in `/etc/llm/llm.env` if needed +- Systemd service units pass through `LLM_*_ARGS` only when set; otherwise, script-level defaults apply - `fineweb_stage_shard_loop.sh` now preflights selected parquet files and quarantines failures into `artifacts/reports/fineweb_stage_shard_loop/quarantine_bad_parquet/` - Known-bad parquet basenames are tracked in `artifacts/reports/fineweb_stage_shard_loop/bad_parquet_files.txt` and skipped in future stage cycles -- On shard-build non-OOM failures, `fineweb_stage_shard_loop.sh` quarantines that job's parquet inputs as bad and continues processing remaining files +- On shard-build non-OOM corruption failures, `fineweb_stage_shard_loop.sh` attempts one warm->hot restage retry in stage-copy mode before quarantining inputs as bad +- Stage-loop deep-validates newly staged parquet files in stage-copy mode (`--deep-validate-max-batches`, `--deep-validate-batch-size`) and quarantines early failures as `stage_deep_validation_failed` +- Use `--parquet-validate-timeout-seconds ` to bound Ceph parquet validation/deep-validation calls and avoid startup hangs on single files - On startup, `fineweb_stage_shard_loop.sh` reconciles bad parquet entries against warm-source validity to avoid permanent false-positive skips - `fineweb_stage_shard_loop.sh` now bootstraps processed parquet basenames from existing manifests at startup, merges `processed + bad` into a stage skip list, and removes known files from hot storage before staging - Stage-loop batch guardrails now require valid report + manifest + non-empty shard files before marking files as processed/purging hot copies @@ -175,37 +191,42 @@ Keep PR scope narrow; split refactors and features into separate PRs. - Supervisor resume guardrail validates `last.pt`/`ckpt_step_*.pt` and quarantines invalid checkpoint files before retry - Use `--no-train-fail-on-eval-regression` in supervisor when you want train chunks to continue and rely on post-chunk prompt-suite gates - Supervisor baseline selection now matches the active suite (`suite_name`/`suite_path`) for both eval and generation gates so suite changes do not compare against mismatched historical reports -- For phase-1 talking quality, use `scripts/train_supervisor_phase1_english_talk.sh` (`english_talk_suite_v1` + `generation_talk_smoke_v1`) before code-specialization passes -- Phase-1 launcher writes supervisor state to `artifacts/reports/train_supervisor_phase1_talk`; pass `--supervisor-state-dir` accordingly to pipeline status tools +- For phase-1 talking quality, use `scripts/train_supervisor_phase1_english_talk.sh` (`english_talk_suite_v1` + `generation_talk_quality_v2` + fixed `english_talk_holdout_suite_v1`) before code-specialization passes +- Phase-1 launcher now defaults to recovery-oriented quality gates (`--lr-schedule constant`, `--generation-fail-below-pass-rate 0.35`, `--holdout-fail-below-pass-rate 0.35`, `--promotion-min-quality-streak 2`, `promotion_policy_talk_recovery_v2.json`) +- Phase-1 launcher writes supervisor state to `artifacts/reports/train_supervisor_phase1_talk`; pipeline status tools now auto-detect between phase1 + standard state dirs (override with `--supervisor-state-dir` when needed) - Phase-1 launcher uses lower-variance generation gating (`--generation-temperature 0.2 --generation-top-k 1`) - Supervisor now runs hot-manifest guard each loop (`scripts/enforce_hot_only_manifests.py`) to auto-disable active manifests that reference symlinked shard bins +- Supervisor now supports hot-shard warmup (`scripts/hot_shard_warmup.py`) both pre-chunk and in background during train chunks to hydrate missing/symlinked active shard bins from Ceph into hot storage (`--hot-shard-warmup-workers`, `--hot-shard-warmup-max-files`, `--hot-shard-warmup-background-interval-seconds`, `--hot-shard-warmup-background-max-files`, `--no-hot-shard-warmup`, `--no-hot-shard-warmup-background`) +- Prefer supervisor `--sampler-strategy balanced --sampler-min-full-passes ` for even shard mixing with guaranteed minimum per-shard full-pass coverage per chunk - On 12 GB RTX 5070 profiles, start supervisor with `--batch-size 12 --target-effective-batch 24 --min-batch-size 6 --max-batch-size 20 --batch-step 2` to avoid early OOM churn - Use supervisor `--train-stall-check-seconds` + `--train-stall-kill-seconds` to auto-restart stuck train chunks when step progress stops - Supervisor writes chunk trends to `artifacts/reports/train_supervisor_350bt/train_trend.tsv` and post-chunk eval trends to `artifacts/reports/train_supervisor_350bt/eval_trend.tsv` - Supervisor also writes scheduled generation-gate trends to `artifacts/reports/train_supervisor_350bt/generation_trend.tsv` +- Supervisor also writes fixed holdout-gate trends to `artifacts/reports/train_supervisor_350bt/holdout_trend.tsv` and stores frozen baseline path in `holdout_baseline_report.txt` - Supervisor also renders `artifacts/reports/train_supervisor_350bt/eval_dashboard.html` and exports `best.pt` aliases after successful eval promotions -- Supervisor now records successful-chunk manifest batch coverage in `/trained_batch_names.txt` for safe shard offload gating -- Use `--generation-suite configs/eval/generation_smoke_suite_v1.json` and `--generation-every-chunks ` to run prompt-generation drift gates every chunk (or every N chunks) +- Supervisor now records successful-chunk sampled-batch coverage in `/trained_batch_names.txt` (from `llm.cli train --sampled-shards-trace`) for safe shard offload gating +- Supervisor supports quality auto-rollback to `best.pt` after sustained regressions; tune with `--quality-rollback-streak` and `--quality-rollback-cooldown-steps` +- Use `--generation-suite configs/eval/generation_smoke_suite_v1.json` (or `generation_talk_quality_v2.json` for phase-1 talk quality) and `--generation-every-chunks ` to run prompt-generation drift gates every chunk (or every N chunks) - Use `scripts/pipeline_eta_report.py --loop` for combined ETA snapshots in `artifacts/reports/pipeline_status.{json,txt}` (includes `top`, `free -h`, `nvidia-smi`, and `df -h` captures) - `pipeline_eta_report.py` accepts `--once` for explicit single-snapshot mode - `pipeline_eta_report.py` also tracks manifest coverage quality (`manifest_unique_input_files`, overlap counts, `coverage_complete`) - `pipeline_eta_report.py` now includes per-task `RUN/STOP` state with stop reasons and `supervisor_gate` in JSON/text output - `pipeline_eta_report.py` now also reports `trainer_stall_seconds` and shard offload readiness (`offload_eligible_batches`, raw/capped counts) -- Use `scripts/pipeline_live_view.py --refresh-seconds 5` for a live-only terminal monitor (system + pipeline task status, no report writes; includes watchdog/prefetch/stage-loop/generation-gate task rows; add `--no-alt-screen` if needed) +- `pipeline_eta_report.py` now also exports `quality_heartbeat` (eval + generation + holdout) plus `status_confidence` (`coverage`, `train_eta`, `quality`, `overall_score`) +- Use `scripts/pipeline_live_view.py --refresh-seconds 5` for a live-only terminal monitor (system + pipeline task status, no report writes; includes watchdog/stage-loop/generation-gate task rows; add `--no-alt-screen` if needed) - `pipeline_live_view.py` can reuse fresh ETA report train-step rates via `--eta-status-file`/`--eta-status-max-age-seconds` to keep training ETA visible when live step deltas are flat - `pipeline_live_view.py` staging line includes `hot_parquet` + `hot_incomplete` so cache refill progress is visible during warm->hot copies - `pipeline_live_view.py` includes manifest coverage line (`unique/510`, overlap inputs/manifests, completion flag) - `pipeline_live_view.py` also shows hot-manifest state (`active`, `offloaded`, `active_symlink_manifests`) and trained-batch registry count - `pipeline_live_view.py` now also shows shard offload readiness (`offload_eligible_batches`) and training stall age (`stall=`) +- `pipeline_live_view.py` now shows a quality heartbeat (`improving`/`flat`/`regressed`/`warming`) from latest eval + generation + holdout trend files +- `pipeline_live_view.py` now also shows a confidence row (`coverage`, `train_eta`, `quality`, `overall`) to gauge status reliability - `pipeline_live_view.py` includes manifest coverage rate/ETA to gauge when coverage gates will clear - `pipeline_live_view.py` also shows supervisor gate state (for example `waiting_unique_inputs /` or `waiting_train_tokens /`) - `pipeline_live_view.py` coverage ETA/rate falls back to sharding throughput when manifest overlap is zero, so ETA remains visible between manifest-update bursts - `pipeline_live_view.py` alerts on duplicate train controllers and on stage-loop runs that are not watchdog-managed - `pipeline_eta_report.py` task process counters are root-deduped so wrapper/child shells do not inflate `RUN xN` values -- `pipeline_live_view.py` shows `STOP | ` for non-running tasks (for example `staging handled by stage-loop` for prefetch when stage-loop queue staging is enabled) -- `fineweb_prefetch_hot_queue.sh` can auto-read stage-loop skip data (`--auto-skip-state-dir artifacts/reports/fineweb_stage_shard_loop`) so it avoids restaging processed/bad parquet files -- `fineweb_prefetch_hot_queue.sh` forwards `--min-free-gib` into stage calls so prefetch and stage-loop share the same hot-space floor -- Revalidate/recover bad parquet list entries with `scripts/revalidate_bad_parquet.py`; use `--restage-valid` to copy newly validated files back into hot storage +- Revalidate/recover bad parquet list entries with `scripts/revalidate_bad_parquet.py`; use `--max-entries` for incremental backlog cleanup, `--workers` for parallel validation, and `--restage-valid` to copy newly validated files back into hot storage - `revalidate_bad_parquet.py` also prunes `quarantine_bad_parquet` by default (drops stale/validated entries; keeps newest copy per still-bad basename) - Optional automation: `llm-bad-parquet-revalidate.timer` runs `revalidate_bad_parquet.py` periodically via systemd - For checkpoint regression tracking, run `scripts/eval_checkpoint_prompts.py` with `configs/eval/standard_prompt_suite_v3.json`; use `--baseline-report` and `--promotion-policy configs/eval/promotion_policy_v1.json` to emit regression deltas + promotion verdict @@ -216,11 +237,14 @@ Keep PR scope narrow; split refactors and features into separate PRs. - On this 20-core server, use 15 parallel streams for split shard-build runs - For CUDA training throughput, prefer `llm.cli train --precision auto` (disable TF32 only if needed with `--no-tf32`) - If GPU utilization stays bursty, try `llm.cli train --compile-model --compile-mode reduce-overhead` +- `llm.cli train --compile-model` now warms the compiled graph and falls back to eager by default; add `--compile-strict` to hard-fail on compile init/warmup issues +- Use `llm.cli train --sampler-max-open-shards ` to cap open shard memmaps and reduce file-descriptor pressure - Default training architecture is `gpt_rope_rmsnorm_swiglu_v1`; use legacy profile only for old checkpoint compatibility -- Prefer `llm.cli train --lr-schedule cosine --lr-warmup-steps ` for stable first-pass runs +- Prefer `llm.cli train --lr-schedule cosine --lr-warmup-steps ` for fresh first-pass runs; prefer supervisor `--lr-schedule constant` for late-step recovery when cosine decay has already flattened LR - Use `--grad-accum-steps` when VRAM is tight and you need higher effective batch - Keep disk use bounded with `llm.cli train --checkpoint-keep-last --checkpoint-keep-every ` - Use `scripts/checkpoint_offload_prune.sh` to sync checkpoint runs to warm storage and prune older local runs (keep active + newest local) +- Use `scripts/checkpoint_step_offload.sh` to offload older `ckpt_step_*.pt` files from active runs to warm storage on a short cadence (for example every 10 minutes) while keeping newest local resume checkpoints - Tune host swap behavior with `sudo bash scripts/set_swappiness.sh --value 10 --persist` (or set `LLM_SWAPPINESS=10` in `/etc/llm/llm.env` for systemd) - For context-extension continuation, resume with `llm.cli train --allow-context-extension --context-length 1024 ...` - Use EMA for long runs with `--ema-decay 0.999 --ema-start-step ` and generate with `--use-ema` when present @@ -229,6 +253,7 @@ Keep PR scope narrow; split refactors and features into separate PRs. - For post-run smoothing, merge several checkpoints with `llm.cli average-checkpoints --state-key model_state` (or `ema_state`) - For deploy bundles, use `scripts/package_inference_bundle.py` (checksums + optional tarball) or `hf_prepare_and_publish_model.py --include-safetensors` - RTX 5070 tuned training profiles live in `configs/train/rtx5070/`; preferred 350BT launchers: `bash scripts/lr_sweep_rtx5070_fineweb_350bt_ctx512.sh` then `bash scripts/train_rtx5070_fineweb_350bt_bpe_v2.sh` +- For reproducible throughput/memory probes across context lengths, use `bash scripts/benchmark_rtx5070_context_profiles.sh` and track `summary.tsv` outputs under `artifacts/reports/rtx5070_ctx_bench_*` - Version extracted/tokenized/sharded outputs with the ZIM date stamp (for example `serverfault_2025-08`) - Keep raw ZIM archives in `/mnt/ceph/llm/data/raw_zim/` - For portable model release + offline server deploy, follow `docs/HF_RELEASE_AND_DEPLOY.md` diff --git a/Makefile b/Makefile index a22fe28..8f7b4be 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ ifneq ("$(wildcard .venv/bin/python)","") PYTHON=.venv/bin/python endif -.PHONY: setup-dev setup-train setup-infer doctor install-server-system install-systemd-services install-user-systemd-services test lint format typecheck smoke extract-zim train-tokenizer train-tokenizer-global corpus-quality-report clean-corpus-batch dataset-risk-report pull-hf-rows parquet-to-corpus fineweb-parquet-to-shards fineweb-manifest-dedupe stage-fineweb-from-warm fineweb-prefetch-hot-queue fineweb-revalidate-bad-parquet enforce-hot-manifests offload-shard-bins-warm fineweb-stage-shard-loop fineweb-stage-shard-watchdog lr-sweep-350bt train-350bt-v2 train-350bt-ctx1024 train-supervisor-350bt train-supervisor-phase1-talk pipeline-eta pipeline-live shard-corpus-batch verify-shards train generate average-checkpoints eval-checkpoint render-eval-dashboard package-inference-bundle sync-warm hydrate-warm offload-zim checkpoint-offload-prune set-swappiness hf-download-resumable hf-download-watchdog hf-prepare-publish hf-download-model serve-openai publish-wiki +.PHONY: setup-dev setup-train setup-infer doctor install-server-system install-systemd-services install-user-systemd-services test lint format typecheck smoke extract-zim train-tokenizer train-tokenizer-global corpus-quality-report clean-corpus-batch dataset-risk-report pull-hf-rows fineweb-parquet-to-shards fineweb-manifest-dedupe stage-fineweb-from-warm fineweb-revalidate-bad-parquet enforce-hot-manifests reconcile-offloaded-manifests shard-offload-cycle offload-shard-bins-warm hot-shard-warmup fineweb-stage-shard-loop fineweb-stage-shard-watchdog lr-sweep-350bt benchmark-rtx5070 train-350bt-v2 train-350bt-ctx1024 train-supervisor-350bt train-supervisor-phase1-talk pipeline-eta pipeline-live shard-corpus-batch verify-shards train generate average-checkpoints eval-checkpoint render-eval-dashboard package-inference-bundle sync-warm hydrate-warm offload-zim checkpoint-offload-prune checkpoint-step-offload set-swappiness hf-download-resumable hf-download-fineweb-edu-full hf-download-watchdog hf-prepare-publish hf-download-model serve-openai publish-wiki setup-dev: bash scripts/bootstrap_dev.sh @@ -70,10 +70,6 @@ pull-hf-rows: @echo "Usage:" @echo " python3 scripts/pull_hf_rows.py --dataset HuggingFaceFW/fineweb --config sample-350BT --split train --output /mnt/ceph/llm/data/extracted/fineweb_sample-350BT_rows100k.txt --max-rows 100000" -parquet-to-corpus: - @echo "Usage:" - @echo " python3 scripts/parquet_to_corpus.py --input-dir data/fineweb/sample-350BT --output-dir data/extracted/fineweb/sample-350BT --field text" - fineweb-parquet-to-shards: @echo "Usage:" @echo " PYTHONPATH=src $(PYTHON) scripts/fineweb_parquet_to_shards.py --input-dir data/fineweb/sample-350BT --output-dir data/shards_global/fineweb-global-bpe-v1 --tokenizer-out artifacts/tokenizer/fineweb-global-bpe-v1.json --bpe-vocab-size 32000 --field text" @@ -87,21 +83,29 @@ stage-fineweb-from-warm: @echo "Usage:" @echo " bash scripts/stage_fineweb_from_warm.sh --max-files 4 --max-gib 8 --copy-jobs 2" -fineweb-prefetch-hot-queue: - @echo "Usage:" - @echo " bash scripts/fineweb_prefetch_hot_queue.sh --queue-min-files 18 --stage-max-files 12 --sleep-seconds 30 --auto-skip-state-dir artifacts/reports/fineweb_stage_shard_loop" - fineweb-revalidate-bad-parquet: @echo "Usage:" - @echo " PYTHONPATH=src $(PYTHON) scripts/revalidate_bad_parquet.py --restage-valid --max-restage-files 15 --min-free-gib 80" + @echo " PYTHONPATH=src $(PYTHON) scripts/revalidate_bad_parquet.py --max-entries 200 --workers 8 --restage-valid --max-restage-files 15 --min-free-gib 80" enforce-hot-manifests: @echo "Usage:" @echo " PYTHONPATH=src $(PYTHON) scripts/enforce_hot_only_manifests.py --shards-root data/shards_global/fineweb-global-bpe-v1" +reconcile-offloaded-manifests: + @echo "Usage:" + @echo " PYTHONPATH=src $(PYTHON) scripts/reconcile_offloaded_manifests.py --shards-root data/shards_global/fineweb-global-bpe-v1 --trained-batches-file artifacts/reports/train_supervisor_phase1_talk/trained_batch_names.txt,artifacts/reports/train_supervisor_350bt/trained_batch_names.txt --skip-if-trained-file-missing --min-active-unique-input-files 510" + +shard-offload-cycle: + @echo "Usage:" + @echo " bash scripts/shard_offload_cycle.sh" + offload-shard-bins-warm: @echo "Usage:" - @echo " PYTHONPATH=src $(PYTHON) scripts/offload_shard_bins_to_warm.py --keep-local-batches 24 --target-free-gib 180 --max-batches 40 --disable-offloaded-manifests --require-trained-batches-file artifacts/reports/train_supervisor_phase1_talk/trained_batch_names.txt --min-active-manifests 48" + @echo " PYTHONPATH=src $(PYTHON) scripts/offload_shard_bins_to_warm.py --keep-local-batches 24 --target-free-gib 180 --max-batches 40 --disable-offloaded-manifests --require-trained-batches-file artifacts/reports/train_supervisor_phase1_talk/trained_batch_names.txt,artifacts/reports/train_supervisor_350bt/trained_batch_names.txt --skip-if-trained-file-missing --min-manifest-unique-input-files 510 --min-active-manifests 48 --min-active-train-tokens 40000000000" + +hot-shard-warmup: + @echo "Usage:" + @echo " PYTHONPATH=src $(PYTHON) scripts/hot_shard_warmup.py --shards-root data/shards_global/fineweb-global-bpe-v1 --warm-shards-root /mnt/ceph/llm/data/shards_global/fineweb-global-bpe-v1 --workers 4" fineweb-stage-shard-loop: @echo "Usage:" @@ -115,6 +119,10 @@ lr-sweep-350bt: @echo "Usage:" @echo " bash scripts/lr_sweep_rtx5070_fineweb_350bt_ctx512.sh" +benchmark-rtx5070: + @echo "Usage:" + @echo " bash scripts/benchmark_rtx5070_context_profiles.sh --max-steps 1200 --compile-model" + train-350bt-v2: @echo "Usage:" @echo " bash scripts/train_rtx5070_fineweb_350bt_bpe_v2.sh" @@ -150,7 +158,7 @@ verify-shards: train: @echo "Usage:" - @echo " PYTHONPATH=src $(PYTHON) -m llm.cli train --shards-path data/shards/ --output-dir artifacts/checkpoints/ --lr-schedule cosine --lr-warmup-steps 200 --grad-accum-steps 1 --checkpoint-keep-last 6 --checkpoint-keep-every 10000 --fail-on-eval-regression" + @echo " PYTHONPATH=src $(PYTHON) -m llm.cli train --shards-path data/shards/ --output-dir artifacts/checkpoints/ --lr-schedule cosine --lr-warmup-steps 200 --grad-accum-steps 1 --sampler-strategy balanced --sampler-min-full-passes 1 --checkpoint-keep-last 6 --checkpoint-keep-every 10000 --fail-on-eval-regression" generate: @echo "Usage:" @@ -188,6 +196,10 @@ checkpoint-offload-prune: @echo "Sync local checkpoints to warm storage and prune older local runs." @echo "Usage: bash scripts/checkpoint_offload_prune.sh --local-checkpoints-dir artifacts/checkpoints --warm-checkpoints-dir /mnt/ceph/llm/data/checkpoints --keep-local-runs 1" +checkpoint-step-offload: + @echo "Offload older ckpt_step_*.pt files to warm storage while keeping newest local steps." + @echo "Usage: bash scripts/checkpoint_step_offload.sh --local-checkpoints-dir artifacts/checkpoints --warm-checkpoints-dir /mnt/ceph/llm/data/checkpoints --keep-last-steps 6 --max-files 24" + set-swappiness: @echo "Set vm.swappiness for training hosts (root required)." @echo "Usage: sudo bash scripts/set_swappiness.sh --value 10 --persist" @@ -196,6 +208,10 @@ hf-download-resumable: @echo "Run a self-healing Hugging Face download worker with resume + retries." @echo "Usage: HF_TOKEN=hf_xxx bash scripts/hf_download_resumable.sh --dataset HuggingFaceFW/fineweb --repo-type dataset --include 'sample/350BT/*.parquet' --local-dir /mnt/ceph/llm/data/fineweb/sample-350BT --max-workers 6 --enable-hf-transfer --skip-dry-run --attempt-timeout-seconds 5400 --retry-delay-seconds 30 --max-retries 0 --log-file artifacts/reports/fineweb_350bt_download_resumable.log" +hf-download-fineweb-edu-full: + @echo "Run simple resumable sync for full FineWeb-Edu parquet set." + @echo "Usage: HF_TOKEN=hf_xxx bash scripts/sync_fineweb_edu_full.sh /mnt/pve/cephfs/llm/data/fineweb/fineweb-edu-full" + hf-download-watchdog: @echo "Run watchdog wrapper that restarts stalled or exited HF download worker." @echo "Usage: HF_TOKEN=hf_xxx bash scripts/hf_download_watchdog.sh --dataset HuggingFaceFW/fineweb --repo-type dataset --include 'sample/350BT/*.parquet' --local-dir /mnt/ceph/llm/data/fineweb/sample-350BT --max-workers 4 --enable-hf-transfer --skip-dry-run --attempt-timeout-seconds 5400 --stall-seconds 1200 --exit-on-complete --expected-parquet-files 510 --expected-bytes 1061360917731 --worker-log-file artifacts/reports/fineweb_350bt_download_resumable.log --watchdog-log-file artifacts/reports/hf_download_watchdog.log" diff --git a/README.md b/README.md index 190a7f2..72e1e15 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,7 @@ bash scripts/bootstrap_dev.sh ```bash make setup-infer # install inference/deploy dependencies make install-systemd-services # install/reload long-run systemd units -make install-user-systemd-services # install/reload user-level systemd units (no sudo) +make install-user-systemd-services # install/reload user-level systemd units (no sudo; includes shard/checkpoint timers) make test # run unit tests make lint # run Ruff checks make format # run Black formatter @@ -64,12 +64,15 @@ make pull-hf-rows # print Hugging Face rows API pull helper usage make fineweb-parquet-to-shards # print direct FineWeb parquet->token-shards usage make fineweb-manifest-dedupe # print overlap-manifest dedupe helper usage make stage-fineweb-from-warm # print warm->hot FineWeb chunk staging usage -make fineweb-prefetch-hot-queue # print hot-queue prefetch worker usage make fineweb-revalidate-bad-parquet # print bad parquet revalidate/restage usage +make reconcile-offloaded-manifests # print offloaded-manifest reconcile usage +make shard-offload-cycle # print safe reconcile->offload->reconcile usage make offload-shard-bins-warm # print shard .bin offload-to-warm usage +make hot-shard-warmup # print active-shard warm->hot hydration usage make fineweb-stage-shard-loop # print rolling stage->shard->verify->sync->purge usage make fineweb-stage-shard-watchdog # print auto-restart watchdog usage for stage/shard loop make lr-sweep-350bt # print RTX 5070 LR sweep usage for staged 350BT shards +make benchmark-rtx5070 # print short context/batch throughput benchmark usage make train-350bt-v2 # print 350BT long-run launcher usage make train-350bt-ctx1024 # print long-context continuation launcher usage make train-supervisor-350bt # print auto-resume trainer supervisor usage @@ -78,11 +81,13 @@ make pipeline-eta # print combined download/shard/train ETA reporter usage make pipeline-live # print live terminal pipeline dashboard usage make shard-corpus-batch # print shared-tokenizer batch sharding usage make hf-download-resumable # print self-healing HF resume-download worker usage +make hf-download-fineweb-edu-full # print simple full FineWeb-Edu sync usage make hf-download-watchdog # print auto-restart wrapper for stalled/exited HF downloads make sync-warm # sync raw/training data + artifacts to warm storage make hydrate-warm # hydrate hot workspace from warm storage make offload-zim # continuously move raw ZIMs hot -> warm make checkpoint-offload-prune # sync checkpoints to warm and prune older local runs +make checkpoint-step-offload # offload older ckpt_step_*.pt while keeping newest local resume steps make set-swappiness # print vm.swappiness tuning usage (root) make hf-prepare-publish # print HF bundle/publish usage make hf-download-model # print full HF model download usage @@ -221,6 +226,19 @@ Notes: - Keep 350BT parquet on warm storage and stage bounded chunks to hot storage before sharding. - For unattended runs, wrap with `scripts/hf_download_watchdog.sh` to auto-restart on stalls. +3ab. Simple full FineWeb-Edu sync script (direct to Ceph path): +```bash +# on the destination server +cd /path/to/llm-repo +export HF_TOKEN=hf_xxx # optional but recommended +bash scripts/sync_fineweb_edu_full.sh /mnt/pve/cephfs/llm/data/fineweb/fineweb-edu-full +``` +Notes: +- Downloads `HuggingFaceFW/fineweb-edu` parquet files (`data/*/*.parquet`). +- Safe to rerun; resumes partial files automatically. +- Override worker/retry behavior with env vars: + `MAX_WORKERS`, `ATTEMPT_TIMEOUT_SECONDS`, `RETRY_DELAY_SECONDS`, `MAX_RETRIES`. + 3aaa. Optional watchdog wrapper for stalled/exited downloads: ```bash bash scripts/hf_download_watchdog.sh \ @@ -242,7 +260,7 @@ bash scripts/hf_download_watchdog.sh \ Use `--exit-on-complete` with expected file and/or byte targets so the watchdog exits once download is complete (instead of looping forever and relaunching workers). -3ab. Stage FineWeb chunks from warm to hot as needed: +3ab. (Optional legacy) Stage FineWeb chunks from warm to hot: ```bash bash scripts/stage_fineweb_from_warm.sh --max-files 4 --max-gib 8 --copy-jobs 2 ``` @@ -252,13 +270,9 @@ Use `--min-free-gib ` to keep a floor of free space on hot storage while stag The staging script now copies into `*.parquet.incomplete` first and renames atomically, so sharding/preflight never reads partially written parquet files. -3ac. Run rolling warm->hot staging + sharding loop (recommended for 350BT on limited hot disk): +3ac. Run direct-from-Ceph sharding loop (recommended baseline): ```bash bash scripts/fineweb_stage_shard_loop.sh \ - --hot-queue-min-files 18 \ - --stage-max-files 12 \ - --stage-copy-jobs 2 \ - --stage-min-free-gib 80 \ --process-max-files 12 \ --shard-jobs 2 \ --auto-tune-shard-jobs \ @@ -269,12 +283,16 @@ bash scripts/fineweb_stage_shard_loop.sh \ --shard-size-tokens 20000000 \ --sync-background \ --sync-max-inflight 2 \ + --hot-max-used-pct 95 \ --sleep-seconds 60 \ --shard-min-batch-size 512 ``` -This loop stages bounded parquet files to hot storage, builds verified shard batches under -`data/shards_global/fineweb-global-bpe-v1/`, syncs those batches back to warm storage, -and purges processed hot parquet files. +This loop reads parquet directly from Ceph (`/mnt/ceph/llm/data/...` by default), +builds verified shard batches under `data/shards_global/fineweb-global-bpe-v1/`, +and syncs each successful batch back to warm storage immediately. +It keeps sharding moving while automatically offloading older already-trained shard batches +when hot-disk usage exceeds `--hot-max-used-pct` (default 80), rather than pausing sharding. +Use `--enable-stage-copy` only if you explicitly want warm->hot parquet staging. Before sharding each batch, the loop now runs a parquet preflight check (row groups/rows/field), quarantines failing hot files, and records their basenames in `artifacts/reports/fineweb_stage_shard_loop/bad_parquet_files.txt` so they are skipped in future staging. @@ -283,25 +301,31 @@ builds a combined stage skip list (`processed + bad`), and removes already-known so restarted loops continue forward instead of re-staging the earliest parquet files. It also reconciles `bad_parquet_files.txt` against warm-source parquet validity on startup, so transient hot-copy failures do not permanently blacklist valid warm files. -`--hot-queue-min-files` keeps a small parquet queue staged locally so shard building is less likely to idle on copy waits. -`--stage-copy-jobs` controls warm->hot copy parallelism for staging throughput. -`--stage-min-free-gib` prevents staging from filling hot disk below a safety floor. +In direct-source mode, `--hot-queue-min-files`/`--stage-*` options are inactive unless +`--enable-stage-copy` is set. `--auto-tune-shard-jobs` adapts `--shard-jobs` (and matching tokenizer threads) from loadavg + batch runtime. `--sync-background` overlaps warm-storage sync with the next shard batch to reduce idle gaps. `--shard-size-tokens 20000000` reduces shard file-count overhead vs the old 5M-token default. If a shard build fails with OOM-like errors, the loop retries automatically with a smaller batch size. Batch guardrails now require valid report/manifest + non-empty shard outputs before files are marked processed or purged from hot storage. -If a shard build fails with non-OOM errors (for example parquet decode errors), that job's input files -are quarantined as bad and the loop continues with remaining files. +If a shard build fails with non-OOM corruption errors (for example parquet decode errors), +the loop can attempt a one-time warm->hot restage retry in stage-copy mode. +If retry still fails, inputs are quarantined as bad and the loop continues with remaining files. +Additionally, newly staged parquet files are deep-validated before shard build +(configurable via `--deep-validate-max-batches` and `--deep-validate-batch-size`) +to catch decode corruption earlier and quarantine files sooner. +Use `--parquet-validate-timeout-seconds` to cap validation calls and avoid hangs on a single Ceph file. Guardrail checks are implemented in `src/llm/fineweb_guardrails.py` and are unit-tested. For 20-core hosts, `--shard-jobs 2 --tokenizer-threads 10 --encode-batch-size 1024` is the current high-throughput profile. +After full coverage (`--expected-unique-input-files`, default `510`), the loop now +switches into training-focused mode and pauses additional stage/shard churn. 3ad. Optional watchdog for stage/shard loop auto-restart on exit/stall: ```bash bash scripts/fineweb_stage_shard_watchdog.sh \ - --worker-args "--hot-queue-min-files 18 --stage-max-files 12 --stage-copy-jobs 2 --process-max-files 12 --shard-jobs 2 --tokenizer-threads 10 --encode-batch-size 1024 --sleep-seconds 60 --shard-min-batch-size 512" \ + --worker-args "--process-max-files 12 --shard-jobs 2 --tokenizer-threads 10 --encode-batch-size 1024 --hot-max-used-pct 95 --sleep-seconds 60 --shard-min-batch-size 512" \ --check-interval-seconds 120 \ --stall-seconds 5400 ``` @@ -312,6 +336,8 @@ watchdog restarts do not leave direct loop runs unmanaged. Use `--no-adopt-exist to force launching a fresh worker process. Watchdog progress snapshots now include hot `.incomplete` file count/bytes, so long warm->hot copy phases are treated as active progress (not false stalls). +Watchdog now also suppresses stall restarts after coverage completion +(`--expected-unique-input-files`). 3ad. Build tokenizer + token shards directly from FineWeb parquet: ```bash @@ -408,6 +434,8 @@ Use a global tokenizer + `shard-corpus-batch` output root for multi-dataset runs For higher sustained GPU utilization on CUDA, use `--precision auto` and keep validation less frequent (`--eval-interval 500 --eval-steps 10`). If utilization is still bursty on smaller models, test `--compile-model`. +`--compile-model` now warms the compiled graph and safely falls back to eager mode on +backend/warmup failures; use `--compile-strict` to hard-fail instead. Training now supports: - warmup + cosine LR schedule (`--lr-schedule`, `--lr-warmup-steps`, `--lr-min-ratio`) - gradient accumulation (`--grad-accum-steps`) @@ -416,6 +444,10 @@ Training now supports: - checkpoint retention pruning (`--checkpoint-keep-last`, `--checkpoint-keep-every`) - optional EMA weights (`--ema-decay`, `--ema-update-every`, `--ema-start-step`) - optional weights-only export (`--export-safetensors`) +- shard sampler FD guardrail (`--sampler-max-open-shards`) to avoid too-many-open-files +- balanced shuffled shard cycling (`--sampler-strategy balanced`) for even shard usage +- minimum full-pass gate (`--sampler-min-full-passes `) to enforce at-least-X passes +- sampled shard trace export (`--sampled-shards-trace`) for per-chunk true coverage accounting 9. Generate text from a checkpoint: ```bash @@ -496,9 +528,6 @@ bash scripts/train_rtx5070_fineweb_350bt_bpe_v2_ctx1024.sh ``` This path resumes from the base run and uses `--allow-context-extension`. -Optional text-first path still exists for inspection-heavy runs: -`parquet_to_corpus -> clean-corpus-batch -> train-tokenizer-global -> shard-corpus-batch`. - ### Incremental FineWeb Adds While Training You can start training on a subset, then add new parquet files with the same tokenizer and resume: @@ -543,16 +572,16 @@ On this 20-core host, default FineWeb shard splitting should use `15` parallel s ## RTX 5070 Tuned Profiles - Tuned profile docs: `docs/RTX5070_TUNING.md` - Saved JSON profiles: - - `configs/train/rtx5070/fineweb_global_bpe_v1_big.json` (recommended, BPE) - `configs/train/rtx5070/fineweb_350bt_bpe_v2_longrun.json` (350BT long-run preset) -- Launch tuned big profile: -```bash -bash scripts/train_rtx5070_fineweb_bpe_v1_big.sh -``` +- `configs/train/rtx5070/fineweb_350bt_bpe_v2_ctx1024_stage.json` (ctx1024 continuation preset) - 350BT-first LR sweep (ctx 512, LR `2e-4..4e-4`): ```bash bash scripts/lr_sweep_rtx5070_fineweb_350bt_ctx512.sh ``` +- Reproducible context/batch benchmark sweep (tok/s + GPU memory/util summary): +```bash +bash scripts/benchmark_rtx5070_context_profiles.sh --max-steps 1200 --compile-model +``` - 350BT-first long run launcher: ```bash bash scripts/train_rtx5070_fineweb_350bt_bpe_v2.sh @@ -565,6 +594,8 @@ bash scripts/train_supervisor_rtx5070_350bt.sh \ --batch-size 12 \ --target-effective-batch 24 \ --min-unique-input-files 510 \ + --sampler-strategy balanced \ + --sampler-min-full-passes 1 \ --min-batch-size 6 \ --max-batch-size 20 \ --batch-step 2 \ @@ -576,18 +607,29 @@ bash scripts/train_supervisor_rtx5070_350bt.sh \ bash scripts/train_supervisor_phase1_english_talk.sh ``` This uses `configs/eval/english_talk_suite_v1.json`, -`configs/eval/generation_talk_smoke_v1.json`, and -`configs/eval/promotion_policy_talk_v1.json`. +`configs/eval/generation_talk_quality_v2.json`, and +`configs/eval/promotion_policy_talk_recovery_v2.json`. +It also runs a fixed holdout gate using +`configs/eval/english_talk_holdout_suite_v1.json`. It also uses a dedicated state dir (`artifacts/reports/train_supervisor_phase1_talk`) and lower-variance generation-gate settings (`--generation-temperature 0.2 --generation-top-k 1`). +Phase-1 launcher now defaults to recovery-friendly guardrails: +`--lr-schedule constant`, `--generation-fail-below-pass-rate 0.35`, +`--holdout-fail-below-pass-rate 0.35`, and `--promotion-min-quality-streak 2`. Successful chunks update `artifacts/reports/train_supervisor_phase1_talk/trained_batch_names.txt`, which can be used to gate shard offload so only already-trained batches move to warm storage. On each supervisor loop, hot-only manifest guard now runs automatically and disables any active manifest that references symlinked shard bins. +Supervisor runs hot-shard warmup before each chunk and also a background warmup loop +during chunk training by default, hydrating missing active shard bins from Ceph into +hot storage (`scripts/hot_shard_warmup.py`) while GPU work is running. When monitoring this profile, point status tools at that state dir: `PYTHONPATH=src .venv/bin/python scripts/pipeline_live_view.py --supervisor-state-dir artifacts/reports/train_supervisor_phase1_talk` and `PYTHONPATH=src .venv/bin/python scripts/pipeline_eta_report.py --supervisor-state-dir artifacts/reports/train_supervisor_phase1_talk`. +If `--supervisor-state-dir` is omitted, both tools now auto-detect the newest existing +state dir between `artifacts/reports/train_supervisor_phase1_talk` and +`artifacts/reports/train_supervisor_350bt`. For continuous 350BT ingestion/training, keep exactly one stage watchdog and one train supervisor running. Avoid launching one-off `llm.cli train --max-steps ...` jobs in parallel with the supervisor. Stage watchdog now performs stale worker cleanup before relaunch, so restarted controllers @@ -599,6 +641,14 @@ Use `--no-dedupe-overlap-manifests` to disable, or `--dedupe-dry-run` to audit w Use `--dedupe-report-keep ` to cap saved dedupe report/log artifacts during long waits. Use `--min-unique-input-files ` to hold training until enough unique parquet inputs are represented in manifests. Use `--min-train-tokens ` to gate startup by total train-token coverage instead of raw file count. +Use `--lr-schedule constant` in supervisor for late-step recovery runs where cosine decay is too aggressive. +Use `--sampler-strategy balanced --sampler-min-full-passes ` to keep shard exposure +mixed and evenly distributed while guaranteeing minimum per-shard pass coverage each chunk. +Tune Ceph warm->hot hydration with `--hot-shard-warmup-workers ` and +`--hot-shard-warmup-max-files ` (or disable with `--no-hot-shard-warmup`). +Tune concurrent prefetch with `--hot-shard-warmup-background-interval-seconds `, +`--hot-shard-warmup-background-max-files `, or disable with +`--no-hot-shard-warmup-background`. Supervisor enforces a singleton lock at `artifacts/reports/train_supervisor_350bt/supervisor.lock`. Add `--no-train-fail-on-eval-regression` if you want chunk runs to continue even when @@ -608,10 +658,18 @@ Supervisor resume guardrails now validate `last.pt`/`ckpt_step_*.pt` before resu quarantine invalid checkpoint files automatically, then continue from the newest valid one. When post-chunk eval passes promotion logic (or beats prior pass-rate baseline), supervisor also exports `best.pt`, `best_eval_report.json`, and safetensors best aliases. +Promotion discipline supports stricter gating by requiring eval policy promotion, +generation pass, holdout pass, and a minimum consecutive quality streak before best promotion. +Supervisor now updates `trained_batch_names.txt` from sampled shard traces produced by +`llm.cli train --sampled-shards-trace ...` (actual touched batches), not full manifest lists. +Supervisor can also auto-rollback to `best.pt` after sustained quality regressions; tune with +`--quality-rollback-streak ` and `--quality-rollback-cooldown-steps ` +(`0` streak disables rollback). Supervisor outputs: - `artifacts/reports/train_supervisor_350bt/train_trend.tsv` (per-chunk train telemetry) - `artifacts/reports/train_supervisor_350bt/eval_trend.tsv` (post-chunk eval trend, including regression/promotion columns) - `artifacts/reports/train_supervisor_350bt/generation_trend.tsv` (scheduled generation-gate trend, with regression columns) +- `artifacts/reports/train_supervisor_350bt/holdout_trend.tsv` (fixed holdout gate trend) - `artifacts/reports/train_supervisor_350bt/eval_dashboard.html` (rendered trend dashboard) - `artifacts/reports/train_supervisor_350bt/eval_dashboard_summary.json` (dashboard summary JSON) The supervisor now auto-selects the latest successful eval baseline from the same suite @@ -628,10 +686,13 @@ Outputs: - `artifacts/reports/pipeline_status.txt` Includes embedded snapshots of `top -b -n1`, `free -h`, `nvidia-smi`, and `df -h`. Also reports manifest coverage metrics (`manifest_unique_input_files`, overlap counts, `coverage_complete`). +Coverage metrics now include both active and offloaded manifests, so progress does not drop after safe offload. Also reports hot-manifest metrics (`active_manifests`, `offloaded_manifests`, `active_manifests_with_symlink_bins`, `trained_batch_names_count`). Also reports `trainer_stall_seconds` and shard offload eligibility (`offload_eligible_batches`, raw/capped counts, trained-registry presence). +Also reports `quality_heartbeat` (eval/gen/holdout trend state) and `status_confidence` +(`coverage`, `train_eta`, `quality`, `overall_score`). Also includes per-task `RUN/STOP` state with stop reasons (for example `download complete`, `staging handled by stage-loop`, `idle between chunks/eval`, or gate waits). Task process counts are root-deduped (controller processes), so wrapper/child shells do not inflate `RUN xN`. @@ -649,6 +710,8 @@ This is a live-only monitor (no report/status files written) and includes: - manifest coverage status (`unique/510`, overlap inputs/manifests, coverage rate + ETA, completion flag) - supervisor gate status (for example waiting on `min_unique_input_files`) - training row includes `stall=` for direct trainer stall visibility +- quality heartbeat line (`improving`/`flat`/`regressed`/`warming`) based on latest eval + generation trends +- confidence line (`coverage`, `train_eta`, `quality`, `overall`) to quickly judge status reliability - running project task states with pid/runtime/cpu/mem summaries - explicit stop reasons for tasks that are not running - alert rows for stage-controller health and shard-manifest stall conditions @@ -695,26 +758,17 @@ Templates: - `deploy/systemd/llm-vm-swappiness.service` - user equivalents under `deploy/systemd/user/` -Note: prefetch is optional when stage-loop already uses hot-queue staging flags -(`--hot-queue-min-files`, `--stage-max-files`, `--stage-copy-jobs`, `--stage-min-free-gib`). -Install/enable the prefetch unit only when you explicitly want separate queue prefetching: - -```bash -bash scripts/install_systemd_services.sh --install-watchdog --install-prefetch -``` - -If you run prefetch with stage-loop, keep auto skip enabled so it respects -processed/bad parquet lists from `artifacts/reports/fineweb_stage_shard_loop/`. -Prefetch now forwards `--min-free-gib` to `stage_fineweb_from_warm.sh`, so -stage-loop and prefetch apply the same hot-disk free-space guardrail. - Revalidate and optionally restore bad parquet files: ```bash PYTHONPATH=src .venv/bin/python scripts/revalidate_bad_parquet.py \ --restage-valid \ + --max-entries 200 \ + --workers 8 \ --max-restage-files 15 \ --min-free-gib 80 ``` +Use `--max-entries` for incremental backlog cleanup; unprocessed entries stay on the bad list. +Use `--workers` to parallelize warm parquet validation on large bad-file backlogs. This also prunes `artifacts/reports/fineweb_stage_shard_loop/quarantine_bad_parquet` by default: - removes quarantine copies for files no longer marked bad - for still-bad files, keeps only the newest copy per basename (`--quarantine-keep-per-name 1`) @@ -724,30 +778,66 @@ Offload older shard binaries to warm storage while keeping training hot-only: ```bash PYTHONPATH=src .venv/bin/python scripts/offload_shard_bins_to_warm.py \ --keep-local-batches 24 \ - --target-free-gib 180 \ + --target-free-gib 47 \ --max-batches 40 \ --disable-offloaded-manifests \ - --require-trained-batches-file artifacts/reports/train_supervisor_phase1_talk/trained_batch_names.txt \ - --min-active-manifests 48 + --require-trained-batches-file artifacts/reports/train_supervisor_phase1_talk/trained_batch_names.txt,artifacts/reports/train_supervisor_350bt/trained_batch_names.txt \ + --skip-if-trained-file-missing \ + --min-manifest-unique-input-files 510 \ + --min-active-manifests 48 \ + --min-active-train-tokens 40000000000 ``` This replaces older local shard `.bin` files with warm-storage symlinks and renames their `manifest.json` to `manifest.offloaded.json`, so `llm.cli train` only sees local hot-disk manifests while disk usage stays bounded. The `--require-trained-batches-file` guard prevents offloading any batch that has not yet been included in a successful supervisor training chunk. -Use `--min-active-manifests` (and optional `--min-active-train-tokens`) as an offload +You can pass a comma-separated fallback list of trained-batch registry files +(for example phase1 + standard state dirs). +Use `--min-manifest-unique-input-files` to block offload until dataset coverage is complete. +Use `--min-active-manifests` and `--min-active-train-tokens` as offload safety floor so hot-local training coverage never drops below your target. +Before offloading, reconcile previously offloaded manifests and re-enable any +batch not proven trained (plus optional hot rehydrate of active symlink bins): +```bash +PYTHONPATH=src .venv/bin/python scripts/reconcile_offloaded_manifests.py \ + --shards-root data/shards_global/fineweb-global-bpe-v1 \ + --trained-batches-file artifacts/reports/train_supervisor_phase1_talk/trained_batch_names.txt,artifacts/reports/train_supervisor_350bt/trained_batch_names.txt \ + --skip-if-trained-file-missing \ + --min-active-unique-input-files 240 \ + --max-restore 4 \ + --warm-shards-root /mnt/ceph/llm/data/shards_global/fineweb-global-bpe-v1 \ + --rehydrate-restored-bins \ + --rehydrate-active-symlink-bins +``` +Use a realistic `--min-active-unique-input-files` floor for your hot-disk budget. +On ~800-900 GiB hot disks, `240` is a practical rolling target. + + +For timer automation, use the safe cycle wrapper +(reconcile -> offload -> reconcile -> enforce-hot-manifests): +```bash +bash scripts/shard_offload_cycle.sh +``` + Environment template: - `deploy/systemd/llm.env.example` (installed to `/etc/llm/llm.env`) +- Service units now pass through `LLM_*_ARGS` only when set; if unset, the underlying + scripts use their built-in defaults. Recommended `LLM_STAGE_SHARD_LOOP_ARGS` baseline for 20-core hosts: ```bash -LLM_STAGE_SHARD_LOOP_ARGS="--hot-queue-min-files 10 --stage-max-files 8 --stage-copy-jobs 4 --stage-min-free-gib 80 --process-max-files 15 --shard-jobs 2 --auto-tune-shard-jobs --auto-tune-min-shard-jobs 2 --auto-tune-max-shard-jobs 3 --auto-tune-low-load-pct 80 --auto-tune-high-load-pct 95 --auto-tune-min-batch-seconds 300 --tokenizer-threads 10 --encode-batch-size 1024 --shard-size-tokens 20000000 --sync-background --sync-max-inflight 2 --sleep-seconds 60 --shard-min-batch-size 512" +LLM_STAGE_SHARD_LOOP_ARGS="--process-max-files 15 --shard-jobs 2 --auto-tune-shard-jobs --auto-tune-min-shard-jobs 2 --auto-tune-max-shard-jobs 3 --auto-tune-low-load-pct 80 --auto-tune-high-load-pct 95 --auto-tune-min-batch-seconds 300 --tokenizer-threads 10 --encode-batch-size 1024 --shard-size-tokens 20000000 --expected-unique-input-files 510 --coverage-complete-sleep-seconds 300 --sync-background --sync-max-inflight 2 --hot-max-used-pct 95 --offload-check-interval-seconds 120 --parquet-validate-timeout-seconds 180 --sleep-seconds 60 --shard-min-batch-size 512" ``` Recommended stage watchdog wrapper: ```bash -LLM_STAGE_SHARD_WATCHDOG_ARGS="--worker-args \"${LLM_STAGE_SHARD_LOOP_ARGS}\" --check-interval-seconds 120 --stall-seconds 5400 --watchdog-log-file artifacts/reports/fineweb_stage_shard_loop/watchdog.log" +LLM_STAGE_SHARD_WATCHDOG_ARGS="--worker-args \"${LLM_STAGE_SHARD_LOOP_ARGS}\" --expected-unique-input-files 510 --check-interval-seconds 120 --stall-seconds 5400 --watchdog-log-file artifacts/reports/fineweb_stage_shard_loop/watchdog.log" +``` +Recommended shard-offload cycle overrides: +```bash +LLM_SHARD_OFFLOAD_RECONCILE_ARGS="--shards-root data/shards_global/fineweb-global-bpe-v1 --trained-batches-file artifacts/reports/train_supervisor_phase1_talk/trained_batch_names.txt,artifacts/reports/train_supervisor_350bt/trained_batch_names.txt --skip-if-trained-file-missing --min-active-unique-input-files 240 --max-restore 4 --warm-shards-root /mnt/ceph/llm/data/shards_global/fineweb-global-bpe-v1 --rehydrate-restored-bins --rehydrate-active-symlink-bins" +LLM_SHARD_OFFLOAD_ARGS="--shards-root data/shards_global/fineweb-global-bpe-v1 --warm-shards-root /mnt/ceph/llm/data/shards_global/fineweb-global-bpe-v1 --keep-local-batches 24 --target-free-gib 47 --max-batches 16 --disable-offloaded-manifests --require-trained-batches-file artifacts/reports/train_supervisor_phase1_talk/trained_batch_names.txt,artifacts/reports/train_supervisor_350bt/trained_batch_names.txt --skip-if-trained-file-missing --min-manifest-unique-input-files 510 --min-active-manifests 48 --min-active-train-tokens 40000000000" ``` ## Inference Bundle Packaging @@ -787,6 +877,18 @@ This now syncs training-critical inputs/outputs including: `data/raw_zim`, `data/fineweb`, `data/cleaned`, `data/extracted`, `data/shards`, `data/shards_global`, `artifacts/tokenizer`, `artifacts/checkpoints`, and `artifacts/reports`. +- Pre-wipe safety checklist (before deleting/rebuilding hot disk contents): +```bash +git fetch --all --prune +git status --short --branch +git push +bash scripts/sync_warm_storage.sh /mnt/ceph/llm/data +stamp=$(date -u +%Y%m%dT%H%M%SZ) +git ls-files --others --ignored --exclude-standard | sort \ + > /mnt/ceph/llm/data/logs/git_untracked_all_${stamp}.txt +``` +Wait for sync completion, then confirm `/mnt/ceph/llm/data/logs/last_sync_utc.txt` +was updated in the current run window. - Periodic checkpoint offload + local prune: ```bash bash scripts/checkpoint_offload_prune.sh \ @@ -821,7 +923,7 @@ bash scripts/hydrate_from_warm_storage.sh /mnt/ceph/llm/data - Batch corpus sharding (`shard-corpus-batch`) with one shared tokenizer. - Baseline GPT training (`train`) with checkpoint save/resume. - Default architecture: RoPE + RMSNorm + SwiGLU (`gpt_rope_rmsnorm_swiglu_v1`). - - Includes AdamW no-decay param groups, warmup/cosine LR, and grad accumulation. + - Includes AdamW no-decay param groups, constant or warmup/cosine LR, and grad accumulation. - Checkpoint-based text generation (`generate`) with temperature/top-k sampling. - Optional safetensors export for deployment (`--export-safetensors`). - Unit tests for tokenizer round-trips and unknown token behavior. @@ -829,7 +931,7 @@ bash scripts/hydrate_from_warm_storage.sh /mnt/ceph/llm/data ## Next Milestones 1. Expand checkpoint eval suite and track regressions in CI. 2. Add tokenizer-aware dataset manifests for long-running incremental FineWeb phases. -3. Add larger-context training profiles and memory/throughput benchmarking. +3. Raise context length beyond 1024 and benchmark memory/throughput tradeoffs. 4. Add finetuning flows for classification and instruction datasets. ## References diff --git a/configs/eval/english_talk_holdout_suite_v1.json b/configs/eval/english_talk_holdout_suite_v1.json new file mode 100644 index 0000000..914d3d5 --- /dev/null +++ b/configs/eval/english_talk_holdout_suite_v1.json @@ -0,0 +1,91 @@ +{ + "name": "english_talk_holdout_suite_v1", + "description": "Frozen holdout suite for phase-1 English conversation quality. Keep prompts stable across runs for promotion discipline.", + "cases": [ + { + "id": "holdout_small_talk_weather", + "category": "conversation", + "prompt": "User: Good morning. It looks rainy today. Reply naturally in plain English.\nAssistant:", + "checks": { + "min_completion_chars": 40, + "max_url_count": 0, + "max_repeated_char_run": 8, + "max_non_ascii_ratio": 0.05 + } + }, + { + "id": "holdout_translation_guten_tag", + "category": "translation", + "prompt": "Translate to English: Guten Tag.", + "checks": { + "min_completion_chars": 4, + "expect_any": ["good day", "hello"], + "max_url_count": 0 + } + }, + { + "id": "holdout_instruction_polite_email", + "category": "instruction_following", + "prompt": "Write two short sentences asking a teacher for an extension on homework in polite English.", + "checks": { + "min_completion_chars": 70, + "expect_any": ["please", "extension", "homework"], + "max_url_count": 0, + "max_repeated_char_run": 8 + } + }, + { + "id": "holdout_factual_ocean", + "category": "factual_qa", + "prompt": "User: Which ocean is the largest on Earth?\nAssistant:", + "checks": { + "min_completion_chars": 7, + "expect_any": ["pacific"], + "max_url_count": 0 + } + }, + { + "id": "holdout_rephrase_simple", + "category": "paraphrase", + "prompt": "Rewrite this in simpler English: The conference was postponed due to severe weather conditions.", + "checks": { + "min_completion_chars": 45, + "expect_any": ["conference", "weather", "delayed", "postponed"], + "max_url_count": 0 + } + }, + { + "id": "holdout_supportive_reply", + "category": "conversation", + "prompt": "User: I am overwhelmed with work this week. Give a short supportive response.\nAssistant:", + "checks": { + "min_completion_chars": 45, + "expect_any": ["sorry", "break", "help", "plan"], + "max_url_count": 0, + "max_repeated_char_run": 8 + } + }, + { + "id": "holdout_clean_english_only", + "category": "clean_output", + "prompt": "Write two short natural-English sentences about learning to cook.", + "checks": { + "min_completion_chars": 60, + "reject_any": ["```", "def ", "class ", "import "], + "max_url_count": 0, + "max_non_ascii_ratio": 0.03 + } + }, + { + "id": "holdout_follow_up_question", + "category": "conversation", + "prompt": "User: I started reading more books this year. Respond positively and ask one follow-up question.\nAssistant:", + "checks": { + "min_completion_chars": 50, + "expect_any": ["?", "great", "nice", "what"], + "max_url_count": 0, + "max_repeated_char_run": 8 + } + } + ] +} diff --git a/configs/eval/generation_talk_quality_v2.json b/configs/eval/generation_talk_quality_v2.json new file mode 100644 index 0000000..bd7b247 --- /dev/null +++ b/configs/eval/generation_talk_quality_v2.json @@ -0,0 +1,140 @@ +{ + "name": "generation_talk_quality_v2", + "description": "Stronger phase-1 generation gate for English fluency, factuality basics, and anti-gibberish behavior.", + "cases": [ + { + "id": "talk_natural_greeting", + "category": "conversation", + "prompt": "User: Hi, I had a rough day. Reply with a short friendly message and ask one follow-up question.\nAssistant:", + "checks": { + "min_completion_chars": 70, + "expect_any": ["?", "sorry", "hope", "tomorrow"], + "reject_any": ["i can't", "i cannot", "i'm unable"], + "max_repeated_char_run": 7, + "max_non_ascii_ratio": 0.03, + "max_url_count": 0 + } + }, + { + "id": "talk_translate_short", + "category": "translation", + "prompt": "Translate to English: Muchas gracias por todo.", + "checks": { + "min_completion_chars": 14, + "expect_any": ["thank", "everything", "very much"], + "max_url_count": 0 + } + }, + { + "id": "talk_factual_days_week", + "category": "factual_qa", + "prompt": "How many days are in a week?", + "checks": { + "min_completion_chars": 3, + "expect_any": ["7", "seven"], + "max_url_count": 0 + } + }, + { + "id": "talk_factual_capital_japan", + "category": "factual_qa", + "prompt": "What is the capital city of Japan?", + "checks": { + "min_completion_chars": 5, + "expect_any": ["tokyo"], + "max_url_count": 0 + } + }, + { + "id": "talk_factual_largest_ocean", + "category": "factual_qa", + "prompt": "Which ocean is the largest on Earth?", + "checks": { + "min_completion_chars": 7, + "expect_any": ["pacific"], + "max_url_count": 0 + } + }, + { + "id": "talk_simple_math", + "category": "reasoning", + "prompt": "Compute 18 + 27 and answer with a short sentence.", + "checks": { + "min_completion_chars": 2, + "expect_any": ["45"], + "max_url_count": 0 + } + }, + { + "id": "talk_instruction_two_sentences", + "category": "instruction_following", + "prompt": "In two short sentences, explain why sleep helps memory.", + "checks": { + "min_completion_chars": 85, + "expect_any": ["sleep", "memory", "brain", "learn"], + "max_repeated_char_run": 7, + "max_non_ascii_ratio": 0.03, + "max_url_count": 0 + } + }, + { + "id": "talk_instruction_three_bullets", + "category": "instruction_following", + "prompt": "Give exactly three bullet points with simple tips for improving spoken English.", + "checks": { + "min_completion_chars": 90, + "expect_any": ["-", "*", "1."], + "max_repeated_char_run": 7, + "max_non_ascii_ratio": 0.03, + "max_url_count": 0 + } + }, + { + "id": "talk_short_summary", + "category": "summarization", + "prompt": "Summarize in one sentence: \"The package arrived early, but one item was missing. Support shipped a replacement the same day.\"", + "checks": { + "min_completion_chars": 70, + "expect_any": ["package", "missing", "replacement"], + "max_repeated_char_run": 7, + "max_non_ascii_ratio": 0.03, + "max_url_count": 0 + } + }, + { + "id": "talk_paraphrase_plain_english", + "category": "paraphrase", + "prompt": "Rewrite this in simpler English: \"The committee postponed the meeting due to severe weather.\"", + "checks": { + "min_completion_chars": 55, + "expect_any": ["meeting", "weather", "postponed", "delayed"], + "max_url_count": 0 + } + }, + { + "id": "talk_clean_no_code", + "category": "clean_output", + "prompt": "Write one short paragraph in plain English about why daily walking is healthy.", + "checks": { + "min_completion_chars": 95, + "reject_any": ["```", "def ", "class ", "import ", "SELECT ", "function("], + "max_repeated_char_run": 7, + "max_non_ascii_ratio": 0.03, + "max_url_count": 0 + } + }, + { + "id": "talk_supportive_response", + "category": "conversation", + "prompt": "User: I failed an interview and feel discouraged. Give a brief supportive response.\nAssistant:", + "checks": { + "min_completion_chars": 75, + "expect_any": ["sorry", "next", "practice", "improve", "you can"], + "reject_any": ["i can't", "i cannot", "i'm unable"], + "max_repeated_char_run": 7, + "max_non_ascii_ratio": 0.03, + "max_url_count": 0 + } + } + ] +} diff --git a/configs/eval/promotion_policy_talk_recovery_v2.json b/configs/eval/promotion_policy_talk_recovery_v2.json new file mode 100644 index 0000000..fe66167 --- /dev/null +++ b/configs/eval/promotion_policy_talk_recovery_v2.json @@ -0,0 +1,19 @@ +{ + "name": "checkpoint_promotion_talk_recovery_v2", + "description": "Recovery policy for early/weak phase-1 checkpoints: allow gradual promotion while still blocking clear regressions.", + "absolute": { + "min_pass_rate": 0.30, + "min_check_pass_rate": 0.65, + "min_avg_case_score": 0.60 + }, + "regression": { + "max_pass_rate_drop": 0.03, + "max_check_pass_rate_drop": 0.03, + "max_avg_case_score_drop": 0.02 + }, + "improvement": { + "min_pass_rate_gain": 0.0, + "min_check_pass_rate_gain": 0.0, + "min_avg_case_score_gain": 0.002 + } +} diff --git a/configs/train/rtx5070/fineweb_global_bpe_v1_big.json b/configs/train/rtx5070/fineweb_global_bpe_v1_big.json deleted file mode 100644 index c1ffefa..0000000 --- a/configs/train/rtx5070/fineweb_global_bpe_v1_big.json +++ /dev/null @@ -1,40 +0,0 @@ -{ - "profile_id": "rtx5070-fineweb-global-bpe-v1-big", - "created_on": "2026-03-07", - "hardware": { - "gpu": "NVIDIA GeForce RTX 5070 Ti Laptop GPU", - "vram_mib": 12227 - }, - "dataset": { - "shards_path": "data/shards_global/fineweb-global-bpe-v1" - }, - "model": { - "n_layers": 12, - "n_heads": 12, - "d_model": 768, - "context_length": 512 - }, - "train": { - "device": "cuda", - "max_steps": 50000, - "batch_size": 34, - "learning_rate": 0.00015, - "lr_schedule": "cosine", - "lr_warmup_steps": 1000, - "lr_min_ratio": 0.1, - "eval_interval": 2000, - "eval_steps": 5, - "fail_on_eval_regression": true, - "eval_regression_tolerance": 0.2, - "log_interval": 100, - "precision": "auto", - "tf32": true, - "compile_model": false, - "pytorch_alloc_conf": "expandable_segments:True" - }, - "notes": [ - "RTX 5070 profile tuned for BPE-tokenized FineWeb shards.", - "Model architecture default is modern RoPE + RMSNorm + SwiGLU." - ], - "command": "PYTORCH_ALLOC_CONF=expandable_segments:True PYTHONPATH=src .venv/bin/python -m llm.cli train --shards-path data/shards_global/fineweb-global-bpe-v1 --output-dir artifacts/checkpoints/fineweb-global-bpe-v1-big-run1 --device cuda --max-steps 50000 --batch-size 34 --context-length 512 --n-layers 12 --n-heads 12 --d-model 768 --learning-rate 1.5e-4 --lr-schedule cosine --lr-warmup-steps 1000 --lr-min-ratio 0.10 --eval-interval 2000 --eval-steps 5 --fail-on-eval-regression --eval-regression-tolerance 0.20 --log-interval 100 --precision auto" -} diff --git a/deploy/systemd/llm-bad-parquet-revalidate.service b/deploy/systemd/llm-bad-parquet-revalidate.service index 31e27fb..d238baa 100644 --- a/deploy/systemd/llm-bad-parquet-revalidate.service +++ b/deploy/systemd/llm-bad-parquet-revalidate.service @@ -8,6 +8,5 @@ Type=oneshot User=__USER__ WorkingDirectory=__REPO_DIR__ EnvironmentFile=-/etc/llm/llm.env -ExecStart=/bin/bash -lc 'cd __REPO_DIR__ && PYTHONPATH=src .venv/bin/python scripts/revalidate_bad_parquet.py ${LLM_BAD_PARQUET_REVALIDATE_ARGS:-"--restage-valid --max-restage-files 8 --min-free-gib 80"}' +ExecStart=/bin/bash -lc 'cd __REPO_DIR__ && PYTHONPATH=src .venv/bin/python scripts/revalidate_bad_parquet.py ${LLM_BAD_PARQUET_REVALIDATE_ARGS:-}' NoNewPrivileges=true - diff --git a/deploy/systemd/llm-checkpoint-offload-prune.service b/deploy/systemd/llm-checkpoint-offload-prune.service index f6ea09c..ba33b35 100644 --- a/deploy/systemd/llm-checkpoint-offload-prune.service +++ b/deploy/systemd/llm-checkpoint-offload-prune.service @@ -8,7 +8,7 @@ Type=oneshot User=__USER__ WorkingDirectory=__REPO_DIR__ EnvironmentFile=-/etc/llm/llm.env -ExecStart=/bin/bash -lc 'cd __REPO_DIR__ && exec bash scripts/checkpoint_offload_prune.sh ${LLM_CHECKPOINT_OFFLOAD_ARGS:-"--local-checkpoints-dir artifacts/checkpoints --warm-checkpoints-dir /mnt/ceph/llm/data/checkpoints --keep-local-runs 1"}' +ExecStart=/bin/bash -lc 'cd __REPO_DIR__ && exec bash scripts/checkpoint_offload_prune.sh ${LLM_CHECKPOINT_OFFLOAD_ARGS:-}' Nice=10 IOSchedulingClass=idle NoNewPrivileges=true diff --git a/deploy/systemd/llm-checkpoint-step-offload.service b/deploy/systemd/llm-checkpoint-step-offload.service new file mode 100644 index 0000000..c32f77b --- /dev/null +++ b/deploy/systemd/llm-checkpoint-step-offload.service @@ -0,0 +1,14 @@ +[Unit] +Description=LLM Checkpoint Step Offload To Warm Storage +After=network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +User=__USER__ +WorkingDirectory=__REPO_DIR__ +EnvironmentFile=-/etc/llm/llm.env +ExecStart=/bin/bash -lc 'cd __REPO_DIR__ && mkdir -p artifacts/reports && exec flock -n artifacts/reports/checkpoint_step_offload.lock bash scripts/checkpoint_step_offload.sh ${LLM_CHECKPOINT_STEP_OFFLOAD_ARGS:-}' +Nice=10 +IOSchedulingClass=idle +NoNewPrivileges=true diff --git a/deploy/systemd/llm-checkpoint-step-offload.timer b/deploy/systemd/llm-checkpoint-step-offload.timer new file mode 100644 index 0000000..16300b8 --- /dev/null +++ b/deploy/systemd/llm-checkpoint-step-offload.timer @@ -0,0 +1,12 @@ +[Unit] +Description=LLM Checkpoint Step Offload Schedule + +[Timer] +OnBootSec=5m +OnUnitActiveSec=10m +AccuracySec=1m +Persistent=true +Unit=llm-checkpoint-step-offload.service + +[Install] +WantedBy=timers.target diff --git a/deploy/systemd/llm-fineweb-prefetch.service b/deploy/systemd/llm-fineweb-prefetch.service deleted file mode 100644 index 708f930..0000000 --- a/deploy/systemd/llm-fineweb-prefetch.service +++ /dev/null @@ -1,19 +0,0 @@ -[Unit] -Description=LLM FineWeb Warm->Hot Prefetch Worker -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=__USER__ -WorkingDirectory=__REPO_DIR__ -EnvironmentFile=-/etc/llm/llm.env -ExecStart=/bin/bash -lc 'cd __REPO_DIR__ && exec bash scripts/fineweb_prefetch_hot_queue.sh ${LLM_PREFETCH_ARGS:-"--queue-min-files 12 --stage-max-files 8 --sleep-seconds 60 --min-free-gib 50"}' -Restart=always -RestartSec=10 -KillSignal=SIGINT -TimeoutStopSec=60 -NoNewPrivileges=true - -[Install] -WantedBy=multi-user.target diff --git a/deploy/systemd/llm-fineweb-stage-shard-loop.service b/deploy/systemd/llm-fineweb-stage-shard-loop.service index 35dee15..343b8b6 100644 --- a/deploy/systemd/llm-fineweb-stage-shard-loop.service +++ b/deploy/systemd/llm-fineweb-stage-shard-loop.service @@ -9,7 +9,7 @@ Type=simple User=__USER__ WorkingDirectory=__REPO_DIR__ EnvironmentFile=-/etc/llm/llm.env -ExecStart=/bin/bash -lc 'cd __REPO_DIR__ && exec bash scripts/fineweb_stage_shard_loop.sh ${LLM_STAGE_SHARD_LOOP_ARGS:-"--hot-queue-min-files 10 --stage-max-files 8 --stage-copy-jobs 4 --stage-min-free-gib 80 --process-max-files 15 --shard-jobs 2 --auto-tune-shard-jobs --auto-tune-min-shard-jobs 2 --auto-tune-max-shard-jobs 3 --auto-tune-low-load-pct 80 --auto-tune-high-load-pct 95 --auto-tune-min-batch-seconds 300 --tokenizer-threads 10 --encode-batch-size 1024 --shard-size-tokens 20000000 --sync-background --sync-max-inflight 2 --sleep-seconds 60 --shard-min-batch-size 512"}' +ExecStart=/bin/bash -lc 'cd __REPO_DIR__ && exec bash scripts/fineweb_stage_shard_loop.sh ${LLM_STAGE_SHARD_LOOP_ARGS:-}' Restart=always RestartSec=10 KillSignal=SIGINT diff --git a/deploy/systemd/llm-fineweb-stage-shard-watchdog.service b/deploy/systemd/llm-fineweb-stage-shard-watchdog.service index 84829e9..abaf537 100644 --- a/deploy/systemd/llm-fineweb-stage-shard-watchdog.service +++ b/deploy/systemd/llm-fineweb-stage-shard-watchdog.service @@ -9,7 +9,7 @@ Type=simple User=__USER__ WorkingDirectory=__REPO_DIR__ EnvironmentFile=-/etc/llm/llm.env -ExecStart=/bin/bash -lc 'cd __REPO_DIR__ && exec bash scripts/fineweb_stage_shard_watchdog.sh ${LLM_STAGE_SHARD_WATCHDOG_ARGS:-"--worker-args \"--hot-queue-min-files 10 --stage-max-files 8 --stage-copy-jobs 4 --stage-min-free-gib 80 --process-max-files 15 --shard-jobs 2 --auto-tune-shard-jobs --auto-tune-min-shard-jobs 2 --auto-tune-max-shard-jobs 3 --auto-tune-low-load-pct 80 --auto-tune-high-load-pct 95 --auto-tune-min-batch-seconds 300 --tokenizer-threads 10 --encode-batch-size 1024 --shard-size-tokens 20000000 --sync-background --sync-max-inflight 2 --sleep-seconds 60 --shard-min-batch-size 512\" --check-interval-seconds 120 --stall-seconds 5400 --watchdog-log-file artifacts/reports/fineweb_stage_shard_loop/watchdog.log"}' +ExecStart=/bin/bash -lc 'cd __REPO_DIR__ && exec bash scripts/fineweb_stage_shard_watchdog.sh ${LLM_STAGE_SHARD_WATCHDOG_ARGS:-}' Restart=always RestartSec=10 KillSignal=SIGINT diff --git a/deploy/systemd/llm-hf-download-watchdog.service b/deploy/systemd/llm-hf-download-watchdog.service index 0428580..d18312f 100644 --- a/deploy/systemd/llm-hf-download-watchdog.service +++ b/deploy/systemd/llm-hf-download-watchdog.service @@ -8,7 +8,7 @@ Type=simple User=__USER__ WorkingDirectory=__REPO_DIR__ EnvironmentFile=-/etc/llm/llm.env -ExecStart=/bin/bash -lc 'cd __REPO_DIR__ && if [[ -z "${HF_TOKEN:-}" ]]; then echo "HF_TOKEN missing; set in /etc/llm/llm.env"; exit 2; fi; exec bash scripts/hf_download_watchdog.sh ${LLM_HF_WATCHDOG_ARGS:-"--dataset HuggingFaceFW/fineweb --repo-type dataset --include \"sample/350BT/*.parquet\" --local-dir /mnt/ceph/llm/data/fineweb/sample-350BT --max-workers 4 --enable-hf-transfer --skip-dry-run --attempt-timeout-seconds 5400 --stall-seconds 1200 --worker-log-file artifacts/reports/fineweb_350bt_download_resumable.log --watchdog-log-file artifacts/reports/hf_download_watchdog.log"}' +ExecStart=/bin/bash -lc 'cd __REPO_DIR__ && if [[ -z "${HF_TOKEN:-}" ]]; then echo "HF_TOKEN missing; set in /etc/llm/llm.env"; exit 2; fi; exec bash scripts/hf_download_watchdog.sh ${LLM_HF_WATCHDOG_ARGS:-}' Restart=always RestartSec=30 KillSignal=SIGINT diff --git a/deploy/systemd/llm-shard-offload.service b/deploy/systemd/llm-shard-offload.service index 5679d10..96179af 100644 --- a/deploy/systemd/llm-shard-offload.service +++ b/deploy/systemd/llm-shard-offload.service @@ -8,7 +8,7 @@ Type=oneshot User=__USER__ WorkingDirectory=__REPO_DIR__ EnvironmentFile=-/etc/llm/llm.env -ExecStart=/bin/bash -lc 'cd __REPO_DIR__ && mkdir -p artifacts/reports && exec flock -n artifacts/reports/shard_offload.lock env PYTHONPATH=src .venv/bin/python scripts/offload_shard_bins_to_warm.py ${LLM_SHARD_OFFLOAD_ARGS:-"--shards-root data/shards_global/fineweb-global-bpe-v1 --warm-shards-root /mnt/ceph/llm/data/shards_global/fineweb-global-bpe-v1 --keep-local-batches 24 --target-free-gib 180 --max-batches 16 --disable-offloaded-manifests --require-trained-batches-file artifacts/reports/train_supervisor_350bt/trained_batch_names.txt --min-active-manifests 48"}' +ExecStart=/bin/bash -lc 'cd __REPO_DIR__ && mkdir -p artifacts/reports && exec flock -n artifacts/reports/shard_offload.lock bash scripts/shard_offload_cycle.sh' Nice=10 IOSchedulingClass=idle NoNewPrivileges=true diff --git a/deploy/systemd/llm-train-supervisor.service b/deploy/systemd/llm-train-supervisor.service index 2498b3e..eb28877 100644 --- a/deploy/systemd/llm-train-supervisor.service +++ b/deploy/systemd/llm-train-supervisor.service @@ -8,7 +8,7 @@ Type=simple User=__USER__ WorkingDirectory=__REPO_DIR__ EnvironmentFile=-/etc/llm/llm.env -ExecStart=/bin/bash -lc 'cd __REPO_DIR__ && exec bash scripts/train_supervisor_rtx5070_350bt.sh ${LLM_TRAIN_SUPERVISOR_ARGS:-"--step-chunk 2000 --poll-seconds 60 --batch-size 12 --target-effective-batch 24 --min-batch-size 6 --max-batch-size 20 --batch-step 2 --gpu-sample-seconds 1 --checkpoint-keep-last 6 --checkpoint-keep-every 10000 --ema-decay 0.999 --ema-update-every 1 --ema-start-step 12000 --eval-suite configs/eval/standard_prompt_suite_v3.json --generation-suite configs/eval/generation_smoke_suite_v1.json --generation-every-chunks 1 --no-train-fail-on-eval-regression"}' +ExecStart=/bin/bash -lc 'cd __REPO_DIR__ && exec bash scripts/train_supervisor_rtx5070_350bt.sh ${LLM_TRAIN_SUPERVISOR_ARGS:-}' Restart=always RestartSec=10 KillSignal=SIGINT diff --git a/deploy/systemd/llm.env.example b/deploy/systemd/llm.env.example index 92c191a..99d47f9 100644 --- a/deploy/systemd/llm.env.example +++ b/deploy/systemd/llm.env.example @@ -5,12 +5,13 @@ # Optional overrides used by unit files # LLM_TRAIN_SUPERVISOR_ARGS="--step-chunk 2000 --poll-seconds 60 --batch-size 12 --target-effective-batch 24 --min-train-tokens 40000000000 --min-batch-size 6 --max-batch-size 20 --batch-step 2 --gpu-sample-seconds 1 --checkpoint-keep-last 6 --checkpoint-keep-every 10000 --ema-decay 0.999 --ema-update-every 1 --ema-start-step 12000 --eval-suite configs/eval/standard_prompt_suite_v3.json --generation-suite configs/eval/generation_smoke_suite_v1.json --generation-every-chunks 1 --no-train-fail-on-eval-regression" -# LLM_TRAIN_SUPERVISOR_ARGS="--state-dir artifacts/reports/train_supervisor_phase1_talk --step-chunk 2000 --poll-seconds 60 --batch-size 12 --target-effective-batch 24 --min-train-tokens 30000000000 --min-batch-size 6 --max-batch-size 20 --batch-step 2 --checkpoint-keep-last 6 --checkpoint-keep-every 10000 --ema-decay 0.999 --dedupe-report-keep 240 --eval-suite configs/eval/english_talk_suite_v1.json --eval-promotion-policy configs/eval/promotion_policy_talk_v1.json --generation-suite configs/eval/generation_talk_smoke_v1.json --generation-temperature 0.2 --generation-top-k 1 --generation-every-chunks 1 --no-train-fail-on-eval-regression" -# LLM_PREFETCH_ARGS="--queue-min-files 24 --stage-max-files 12 --stage-copy-jobs 4 --sleep-seconds 30 --min-free-gib 80 --auto-skip-state-dir artifacts/reports/fineweb_stage_shard_loop" -# LLM_STAGE_SHARD_LOOP_ARGS="--hot-queue-min-files 10 --stage-max-files 8 --stage-copy-jobs 4 --stage-min-free-gib 80 --process-max-files 15 --shard-jobs 2 --auto-tune-shard-jobs --auto-tune-min-shard-jobs 2 --auto-tune-max-shard-jobs 3 --auto-tune-low-load-pct 80 --auto-tune-high-load-pct 95 --auto-tune-min-batch-seconds 300 --tokenizer-threads 10 --encode-batch-size 1024 --shard-size-tokens 20000000 --sync-background --sync-max-inflight 2 --sleep-seconds 60 --shard-min-batch-size 512" -# LLM_STAGE_SHARD_WATCHDOG_ARGS="--worker-args \"--hot-queue-min-files 10 --stage-max-files 8 --stage-copy-jobs 4 --stage-min-free-gib 80 --process-max-files 15 --shard-jobs 2 --auto-tune-shard-jobs --auto-tune-min-shard-jobs 2 --auto-tune-max-shard-jobs 3 --auto-tune-low-load-pct 80 --auto-tune-high-load-pct 95 --auto-tune-min-batch-seconds 300 --tokenizer-threads 10 --encode-batch-size 1024 --shard-size-tokens 20000000 --sync-background --sync-max-inflight 2 --sleep-seconds 60 --shard-min-batch-size 512\" --check-interval-seconds 120 --stall-seconds 5400 --watchdog-log-file artifacts/reports/fineweb_stage_shard_loop/watchdog.log" +# LLM_TRAIN_SUPERVISOR_ARGS="--state-dir artifacts/reports/train_supervisor_phase1_talk --step-chunk 2000 --poll-seconds 60 --batch-size 12 --target-effective-batch 24 --lr-schedule constant --min-train-tokens 30000000000 --min-batch-size 6 --max-batch-size 20 --batch-step 2 --checkpoint-keep-last 6 --checkpoint-keep-every 10000 --ema-decay 0.999 --dedupe-report-keep 240 --eval-suite configs/eval/english_talk_suite_v1.json --eval-promotion-policy configs/eval/promotion_policy_talk_recovery_v2.json --generation-suite configs/eval/generation_talk_quality_v2.json --generation-temperature 0.2 --generation-top-k 1 --generation-fail-below-pass-rate 0.35 --holdout-suite configs/eval/english_talk_holdout_suite_v1.json --holdout-fail-below-pass-rate 0.35 --generation-every-chunks 1 --no-train-fail-on-eval-regression" +# LLM_STAGE_SHARD_LOOP_ARGS="--process-max-files 15 --shard-jobs 2 --auto-tune-shard-jobs --auto-tune-min-shard-jobs 2 --auto-tune-max-shard-jobs 3 --auto-tune-low-load-pct 80 --auto-tune-high-load-pct 95 --auto-tune-min-batch-seconds 300 --tokenizer-threads 10 --encode-batch-size 1024 --shard-size-tokens 20000000 --expected-unique-input-files 510 --coverage-complete-sleep-seconds 300 --sync-background --sync-max-inflight 2 --hot-max-used-pct 95 --offload-check-interval-seconds 120 --parquet-validate-timeout-seconds 180 --sleep-seconds 60 --shard-min-batch-size 512" +# LLM_STAGE_SHARD_WATCHDOG_ARGS="--worker-args \"${LLM_STAGE_SHARD_LOOP_ARGS}\" --expected-unique-input-files 510 --check-interval-seconds 120 --stall-seconds 5400 --watchdog-log-file artifacts/reports/fineweb_stage_shard_loop/watchdog.log" # LLM_HF_WATCHDOG_ARGS="--dataset HuggingFaceFW/fineweb --repo-type dataset --include \"sample/350BT/*.parquet\" --local-dir /mnt/ceph/llm/data/fineweb/sample-350BT --max-workers 4 --enable-hf-transfer --skip-dry-run --attempt-timeout-seconds 5400 --stall-seconds 1200 --worker-log-file artifacts/reports/fineweb_350bt_download_resumable.log --watchdog-log-file artifacts/reports/hf_download_watchdog.log" # LLM_CHECKPOINT_OFFLOAD_ARGS="--local-checkpoints-dir artifacts/checkpoints --warm-checkpoints-dir /mnt/ceph/llm/data/checkpoints --keep-local-runs 1" +# LLM_CHECKPOINT_STEP_OFFLOAD_ARGS="--local-checkpoints-dir artifacts/checkpoints --warm-checkpoints-dir /mnt/ceph/llm/data/checkpoints --keep-last-steps 6 --max-files 24" # LLM_BAD_PARQUET_REVALIDATE_ARGS="--restage-valid --max-restage-files 8 --min-free-gib 80" -# LLM_SHARD_OFFLOAD_ARGS="--shards-root data/shards_global/fineweb-global-bpe-v1 --warm-shards-root /mnt/ceph/llm/data/shards_global/fineweb-global-bpe-v1 --keep-local-batches 24 --target-free-gib 180 --max-batches 16 --disable-offloaded-manifests --require-trained-batches-file artifacts/reports/train_supervisor_350bt/trained_batch_names.txt --min-active-manifests 48" +# LLM_SHARD_OFFLOAD_ARGS="--shards-root data/shards_global/fineweb-global-bpe-v1 --warm-shards-root /mnt/ceph/llm/data/shards_global/fineweb-global-bpe-v1 --keep-local-batches 24 --target-free-gib 47 --max-batches 24 --disable-offloaded-manifests --require-trained-batches-file artifacts/reports/train_supervisor_phase1_talk/trained_batch_names.txt,artifacts/reports/train_supervisor_350bt/trained_batch_names.txt --skip-if-trained-file-missing --min-active-manifests 48 --min-active-train-tokens 40000000000" +# LLM_SHARD_OFFLOAD_RECONCILE_ARGS="--shards-root data/shards_global/fineweb-global-bpe-v1 --trained-batches-file artifacts/reports/train_supervisor_phase1_talk/trained_batch_names.txt,artifacts/reports/train_supervisor_350bt/trained_batch_names.txt --skip-if-trained-file-missing --min-active-unique-input-files 240 --max-restore 4 --warm-shards-root /mnt/ceph/llm/data/shards_global/fineweb-global-bpe-v1 --rehydrate-restored-bins --rehydrate-active-symlink-bins" # LLM_SWAPPINESS=10 diff --git a/deploy/systemd/user/llm-checkpoint-offload-prune.service b/deploy/systemd/user/llm-checkpoint-offload-prune.service new file mode 100644 index 0000000..413ebbe --- /dev/null +++ b/deploy/systemd/user/llm-checkpoint-offload-prune.service @@ -0,0 +1,13 @@ +[Unit] +Description=LLM Checkpoint Offload And Local Prune +After=network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +WorkingDirectory=__REPO_DIR__ +EnvironmentFile=-%h/.config/llm/llm.env +ExecStart=/bin/bash -lc 'cd __REPO_DIR__ && exec bash scripts/checkpoint_offload_prune.sh ${LLM_CHECKPOINT_OFFLOAD_ARGS:-}' +Nice=10 +IOSchedulingClass=idle +NoNewPrivileges=true diff --git a/deploy/systemd/user/llm-checkpoint-offload-prune.timer b/deploy/systemd/user/llm-checkpoint-offload-prune.timer new file mode 100644 index 0000000..8dd2d72 --- /dev/null +++ b/deploy/systemd/user/llm-checkpoint-offload-prune.timer @@ -0,0 +1,12 @@ +[Unit] +Description=LLM Checkpoint Offload/Prune Schedule + +[Timer] +OnBootSec=10m +OnUnitActiveSec=30m +AccuracySec=1m +Persistent=true +Unit=llm-checkpoint-offload-prune.service + +[Install] +WantedBy=timers.target diff --git a/deploy/systemd/user/llm-checkpoint-step-offload.service b/deploy/systemd/user/llm-checkpoint-step-offload.service new file mode 100644 index 0000000..0a79246 --- /dev/null +++ b/deploy/systemd/user/llm-checkpoint-step-offload.service @@ -0,0 +1,13 @@ +[Unit] +Description=LLM Checkpoint Step Offload To Warm Storage +After=network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +WorkingDirectory=__REPO_DIR__ +EnvironmentFile=-%h/.config/llm/llm.env +ExecStart=/bin/bash -lc 'cd __REPO_DIR__ && mkdir -p artifacts/reports && exec flock -n artifacts/reports/checkpoint_step_offload.lock bash scripts/checkpoint_step_offload.sh ${LLM_CHECKPOINT_STEP_OFFLOAD_ARGS:-}' +Nice=10 +IOSchedulingClass=idle +NoNewPrivileges=true diff --git a/deploy/systemd/user/llm-checkpoint-step-offload.timer b/deploy/systemd/user/llm-checkpoint-step-offload.timer new file mode 100644 index 0000000..16300b8 --- /dev/null +++ b/deploy/systemd/user/llm-checkpoint-step-offload.timer @@ -0,0 +1,12 @@ +[Unit] +Description=LLM Checkpoint Step Offload Schedule + +[Timer] +OnBootSec=5m +OnUnitActiveSec=10m +AccuracySec=1m +Persistent=true +Unit=llm-checkpoint-step-offload.service + +[Install] +WantedBy=timers.target diff --git a/deploy/systemd/user/llm-fineweb-stage-shard-watchdog.service b/deploy/systemd/user/llm-fineweb-stage-shard-watchdog.service index 12e8524..87512f6 100644 --- a/deploy/systemd/user/llm-fineweb-stage-shard-watchdog.service +++ b/deploy/systemd/user/llm-fineweb-stage-shard-watchdog.service @@ -8,7 +8,7 @@ Conflicts=llm-fineweb-stage-shard-loop.service Type=simple WorkingDirectory=__REPO_DIR__ EnvironmentFile=-%h/.config/llm/llm.env -ExecStart=/bin/bash -lc 'cd __REPO_DIR__ && exec bash scripts/fineweb_stage_shard_watchdog.sh ${LLM_STAGE_SHARD_WATCHDOG_ARGS:-"--worker-args \"--hot-queue-min-files 10 --stage-max-files 8 --stage-copy-jobs 4 --stage-min-free-gib 80 --process-max-files 15 --shard-jobs 2 --auto-tune-shard-jobs --auto-tune-min-shard-jobs 2 --auto-tune-max-shard-jobs 3 --auto-tune-low-load-pct 80 --auto-tune-high-load-pct 95 --auto-tune-min-batch-seconds 300 --tokenizer-threads 10 --encode-batch-size 1024 --shard-size-tokens 20000000 --sync-background --sync-max-inflight 2 --sleep-seconds 60 --shard-min-batch-size 512\" --check-interval-seconds 120 --stall-seconds 5400 --watchdog-log-file artifacts/reports/fineweb_stage_shard_loop/watchdog.log"}' +ExecStart=/bin/bash -lc 'cd __REPO_DIR__ && exec bash scripts/fineweb_stage_shard_watchdog.sh ${LLM_STAGE_SHARD_WATCHDOG_ARGS:-}' Restart=always RestartSec=10 KillSignal=SIGINT diff --git a/deploy/systemd/user/llm-hf-download-watchdog.service b/deploy/systemd/user/llm-hf-download-watchdog.service index 006fc31..f7c67e6 100644 --- a/deploy/systemd/user/llm-hf-download-watchdog.service +++ b/deploy/systemd/user/llm-hf-download-watchdog.service @@ -7,7 +7,7 @@ Wants=network-online.target Type=simple WorkingDirectory=__REPO_DIR__ EnvironmentFile=-%h/.config/llm/llm.env -ExecStart=/bin/bash -lc 'cd __REPO_DIR__ && if [[ -z "${HF_TOKEN:-}" ]]; then echo "HF_TOKEN missing; set in ~/.config/llm/llm.env"; exit 2; fi; exec bash scripts/hf_download_watchdog.sh ${LLM_HF_WATCHDOG_ARGS:-"--dataset HuggingFaceFW/fineweb --repo-type dataset --include \"sample/350BT/*.parquet\" --local-dir /mnt/ceph/llm/data/fineweb/sample-350BT --max-workers 4 --enable-hf-transfer --skip-dry-run --attempt-timeout-seconds 5400 --stall-seconds 1200 --worker-log-file artifacts/reports/fineweb_350bt_download_resumable.log --watchdog-log-file artifacts/reports/hf_download_watchdog.log"}' +ExecStart=/bin/bash -lc 'cd __REPO_DIR__ && if [[ -z "${HF_TOKEN:-}" ]]; then echo "HF_TOKEN missing; set in ~/.config/llm/llm.env"; exit 2; fi; exec bash scripts/hf_download_watchdog.sh ${LLM_HF_WATCHDOG_ARGS:-}' Restart=always RestartSec=30 KillSignal=SIGINT diff --git a/deploy/systemd/user/llm-shard-offload.service b/deploy/systemd/user/llm-shard-offload.service index dfb063d..c8a6be0 100644 --- a/deploy/systemd/user/llm-shard-offload.service +++ b/deploy/systemd/user/llm-shard-offload.service @@ -7,7 +7,7 @@ Wants=network-online.target Type=oneshot WorkingDirectory=__REPO_DIR__ EnvironmentFile=-%h/.config/llm/llm.env -ExecStart=/bin/bash -lc 'cd __REPO_DIR__ && mkdir -p artifacts/reports && exec flock -n artifacts/reports/shard_offload.lock env PYTHONPATH=src .venv/bin/python scripts/offload_shard_bins_to_warm.py ${LLM_SHARD_OFFLOAD_ARGS:-"--shards-root data/shards_global/fineweb-global-bpe-v1 --warm-shards-root /mnt/ceph/llm/data/shards_global/fineweb-global-bpe-v1 --keep-local-batches 24 --target-free-gib 180 --max-batches 16 --disable-offloaded-manifests --require-trained-batches-file artifacts/reports/train_supervisor_350bt/trained_batch_names.txt --min-active-manifests 48"}' +ExecStart=/bin/bash -lc 'cd __REPO_DIR__ && mkdir -p artifacts/reports && exec flock -n artifacts/reports/shard_offload.lock bash scripts/shard_offload_cycle.sh' Nice=10 IOSchedulingClass=idle NoNewPrivileges=true diff --git a/deploy/systemd/user/llm-train-supervisor.service b/deploy/systemd/user/llm-train-supervisor.service index b566b14..d6e0ccb 100644 --- a/deploy/systemd/user/llm-train-supervisor.service +++ b/deploy/systemd/user/llm-train-supervisor.service @@ -7,7 +7,7 @@ Wants=network-online.target Type=simple WorkingDirectory=__REPO_DIR__ EnvironmentFile=-%h/.config/llm/llm.env -ExecStart=/bin/bash -lc 'cd __REPO_DIR__ && exec bash scripts/train_supervisor_rtx5070_350bt.sh ${LLM_TRAIN_SUPERVISOR_ARGS:-"--step-chunk 2000 --poll-seconds 60 --batch-size 12 --target-effective-batch 24 --min-batch-size 6 --max-batch-size 20 --batch-step 2 --gpu-sample-seconds 1 --checkpoint-keep-last 6 --checkpoint-keep-every 10000 --ema-decay 0.999 --ema-update-every 1 --ema-start-step 12000 --eval-suite configs/eval/standard_prompt_suite_v3.json --generation-suite configs/eval/generation_smoke_suite_v1.json --generation-every-chunks 1 --no-train-fail-on-eval-regression"}' +ExecStart=/bin/bash -lc 'cd __REPO_DIR__ && exec bash scripts/train_supervisor_rtx5070_350bt.sh ${LLM_TRAIN_SUPERVISOR_ARGS:-}' Restart=always RestartSec=10 KillSignal=SIGINT diff --git a/docs/HF_RELEASE_AND_DEPLOY.md b/docs/HF_RELEASE_AND_DEPLOY.md index 6a9d515..955e3fb 100644 --- a/docs/HF_RELEASE_AND_DEPLOY.md +++ b/docs/HF_RELEASE_AND_DEPLOY.md @@ -12,13 +12,13 @@ export HF_TOKEN=hf_xxx # bundle only (local) .venv/bin/python scripts/hf_prepare_and_publish_model.py \ --repo-id aditaa/llm-from-scratch-v1 \ - --checkpoint artifacts/checkpoints/fineweb-global-bpe-v1-big-run1/last.pt \ + --checkpoint artifacts/checkpoints/fineweb-350bt-bpe-v2-run1/best.pt \ --include-safetensors # bundle + push to HF model repo .venv/bin/python scripts/hf_prepare_and_publish_model.py \ --repo-id aditaa/llm-from-scratch-v1 \ - --checkpoint artifacts/checkpoints/fineweb-global-bpe-v1-big-run1/last.pt \ + --checkpoint artifacts/checkpoints/fineweb-350bt-bpe-v2-run1/best.pt \ --include-safetensors \ --push ``` diff --git a/docs/RTX5070_TUNING.md b/docs/RTX5070_TUNING.md index 1def7aa..8c9a11f 100644 --- a/docs/RTX5070_TUNING.md +++ b/docs/RTX5070_TUNING.md @@ -51,7 +51,7 @@ Measured with this profile: `avg_util=99.1%`, `max_util=100%`, `min_util=96%`, ` ## Launch ```bash -bash scripts/train_rtx5070_fineweb_bpe_v1_big.sh +bash scripts/train_rtx5070_fineweb_350bt_bpe_v2.sh ``` 350BT-first sweep and long-run launchers: @@ -61,5 +61,20 @@ bash scripts/train_rtx5070_fineweb_350bt_bpe_v2.sh ``` Saved profile JSON: -- `configs/train/rtx5070/fineweb_global_bpe_v1_big.json` - `configs/train/rtx5070/fineweb_350bt_bpe_v2_longrun.json` +- `configs/train/rtx5070/fineweb_350bt_bpe_v2_ctx1024_stage.json` + +## Reproducible Context/Batch Benchmark +Run a short standardized sweep that records per-profile token throughput and GPU memory/utilization: + +```bash +bash scripts/benchmark_rtx5070_context_profiles.sh --max-steps 1200 --compile-model +``` + +Default profile set: +- `512:34:1` +- `768:16:2` +- `1024:6:4` + +Output: +- `artifacts/reports/rtx5070_ctx_bench_/summary.tsv` diff --git a/docs/SERVER_SETUP.md b/docs/SERVER_SETUP.md index e1fe4d1..7f75051 100644 --- a/docs/SERVER_SETUP.md +++ b/docs/SERVER_SETUP.md @@ -91,6 +91,28 @@ Rehydrate from warm storage: bash scripts/hydrate_from_warm_storage.sh /mnt/ceph/llm/data ``` +Simple full FineWeb-Edu sync to Ceph (resumable): + +```bash +export HF_TOKEN=hf_xxx # optional but recommended +bash scripts/sync_fineweb_edu_full.sh /mnt/pve/cephfs/llm/data/fineweb/fineweb-edu-full +``` + +Pre-wipe safety checklist (before deleting/rebuilding hot disk contents): + +```bash +git fetch --all --prune +git status --short --branch +git push +bash scripts/sync_warm_storage.sh /mnt/ceph/llm/data +stamp=$(date -u +%Y%m%dT%H%M%SZ) +git ls-files --others --ignored --exclude-standard | sort \ + > /mnt/ceph/llm/data/logs/git_untracked_all_${stamp}.txt +``` + +Do not wipe until the sync command exits successfully and +`/mnt/ceph/llm/data/logs/last_sync_utc.txt` reflects this run. + ## 8) Pre-Training Shard Integrity Check Run integrity checks before training starts: @@ -115,6 +137,8 @@ Installed units: - `llm-hf-download-watchdog.service` (optional but recommended for long HF pulls) - `llm-checkpoint-offload-prune.service` - `llm-checkpoint-offload-prune.timer` +- `llm-checkpoint-step-offload.service` +- `llm-checkpoint-step-offload.timer` - `llm-bad-parquet-revalidate.service` - `llm-bad-parquet-revalidate.timer` - `llm-vm-swappiness.service` @@ -129,16 +153,14 @@ Environment file: - Set `LLM_TRAIN_SUPERVISOR_ARGS` to tune training readiness gates (for example `--min-train-tokens 40000000000` to gate by token coverage instead of only file count) - Set `LLM_CHECKPOINT_OFFLOAD_ARGS` for warm-sync + prune policy (for example keep newest local run only) +- Set `LLM_CHECKPOINT_STEP_OFFLOAD_ARGS` for frequent older-step offload while keeping recent local resume checkpoints - Set `LLM_BAD_PARQUET_REVALIDATE_ARGS` for periodic bad-parquet recovery/restage policy - Set `LLM_SWAPPINESS=10` to reduce swap churn; `llm-vm-swappiness.service` applies this at boot -- Optional prefetch unit (only when you explicitly want separate prefetching in addition to stage-loop staging): - `bash scripts/install_systemd_services.sh --install-watchdog --install-prefetch` - installs `llm-fineweb-prefetch.service` - Useful commands: ```bash sudo systemctl status llm-train-supervisor.service sudo systemctl status llm-checkpoint-offload-prune.timer +sudo systemctl status llm-checkpoint-step-offload.timer sudo systemctl status llm-bad-parquet-revalidate.timer sudo systemctl restart llm-train-supervisor.service sudo journalctl -u llm-train-supervisor.service -f diff --git a/scripts/benchmark_rtx5070_context_profiles.sh b/scripts/benchmark_rtx5070_context_profiles.sh new file mode 100755 index 0000000..5d729af --- /dev/null +++ b/scripts/benchmark_rtx5070_context_profiles.sh @@ -0,0 +1,348 @@ +#!/usr/bin/env bash +set -euo pipefail + +usage() { + cat <<'USAGE' +Usage: + bash scripts/benchmark_rtx5070_context_profiles.sh [options] + +Short throughput/memory benchmark sweep for RTX 5070 profile sizing. +Each profile runs a bounded training probe and writes a summary TSV. + +Options: + --shards-path PATH Shards root or manifest path + (default: data/shards_global/fineweb-global-bpe-v1) + --out-root PATH Output root for benchmark runs + (default: artifacts/reports/rtx5070_ctx_bench_) + --profiles CSV Comma-separated context:batch:grad_accum entries + (default: 512:34:1,768:16:2,1024:6:4) + --max-steps N Max train steps per profile (default: 1200) + --learning-rate FLOAT Learning rate for probe runs (default: 1.5e-4) + --lr-warmup-steps N LR warmup steps (default: 200) + --lr-min-ratio FLOAT Cosine floor ratio (default: 0.10) + --eval-interval N Eval interval (default: 300) + --eval-steps N Eval steps (default: 6) + --log-interval N Train log interval (default: 50) + --sample-seconds N GPU sampling interval seconds (default: 2) + --n-layers N Model layers (default: 12) + --n-heads N Attention heads (default: 12) + --d-model N Hidden width (default: 768) + --device DEVICE Train device (default: cuda) + --precision MODE Train precision (default: auto) + --compile-model Enable torch.compile during probes + --help Show this help + +Outputs: + /summary.tsv + //train.log + //gpu_samples.csv +USAGE +} + +SHARDS_PATH="data/shards_global/fineweb-global-bpe-v1" +OUT_ROOT="artifacts/reports/rtx5070_ctx_bench_$(date +%Y%m%d_%H%M%S)" +PROFILES="512:34:1,768:16:2,1024:6:4" +MAX_STEPS=1200 +LEARNING_RATE="1.5e-4" +LR_WARMUP_STEPS=200 +LR_MIN_RATIO="0.10" +EVAL_INTERVAL=300 +EVAL_STEPS=6 +LOG_INTERVAL=50 +SAMPLE_SECONDS=2 +N_LAYERS=12 +N_HEADS=12 +D_MODEL=768 +DEVICE="cuda" +PRECISION="auto" +COMPILE_MODEL=0 + +while [[ $# -gt 0 ]]; do + case "$1" in + --shards-path) + SHARDS_PATH="$2" + shift 2 + ;; + --out-root) + OUT_ROOT="$2" + shift 2 + ;; + --profiles) + PROFILES="$2" + shift 2 + ;; + --max-steps) + MAX_STEPS="$2" + shift 2 + ;; + --learning-rate) + LEARNING_RATE="$2" + shift 2 + ;; + --lr-warmup-steps) + LR_WARMUP_STEPS="$2" + shift 2 + ;; + --lr-min-ratio) + LR_MIN_RATIO="$2" + shift 2 + ;; + --eval-interval) + EVAL_INTERVAL="$2" + shift 2 + ;; + --eval-steps) + EVAL_STEPS="$2" + shift 2 + ;; + --log-interval) + LOG_INTERVAL="$2" + shift 2 + ;; + --sample-seconds) + SAMPLE_SECONDS="$2" + shift 2 + ;; + --n-layers) + N_LAYERS="$2" + shift 2 + ;; + --n-heads) + N_HEADS="$2" + shift 2 + ;; + --d-model) + D_MODEL="$2" + shift 2 + ;; + --device) + DEVICE="$2" + shift 2 + ;; + --precision) + PRECISION="$2" + shift 2 + ;; + --compile-model) + COMPILE_MODEL=1 + shift + ;; + --help|-h) + usage + exit 0 + ;; + *) + echo "error: unknown option: $1" >&2 + usage >&2 + exit 2 + ;; + esac +done + +if [[ ! -x ".venv/bin/python" ]]; then + echo "error: .venv/bin/python not found; run make setup-train first" >&2 + exit 1 +fi + +if [[ ! -e "$SHARDS_PATH" ]]; then + echo "error: shards path not found: $SHARDS_PATH" >&2 + exit 1 +fi + +if ! [[ "$MAX_STEPS" =~ ^[0-9]+$ ]] || [[ "$MAX_STEPS" -le 0 ]]; then + echo "error: --max-steps must be a positive integer" >&2 + exit 2 +fi + +if ! [[ "$SAMPLE_SECONDS" =~ ^[0-9]+$ ]] || [[ "$SAMPLE_SECONDS" -le 0 ]]; then + echo "error: --sample-seconds must be a positive integer" >&2 + exit 2 +fi + +has_nvidia_smi=0 +if command -v nvidia-smi >/dev/null 2>&1; then + has_nvidia_smi=1 +fi + +mkdir -p "$OUT_ROOT" +SUMMARY_TSV="$OUT_ROOT/summary.tsv" + +echo -e "profile\tcontext\tbatch\tgrad_accum\teffective_batch\tavg_toks_per_sec\tmax_toks_per_sec\tbest_val_ppl\tavg_gpu_util\tmax_gpu_util\tavg_gpu_mem_mib\tpeak_gpu_mem_mib\tgpu_samples\texit_code\trun_dir" > "$SUMMARY_TSV" + +parse_metrics() { + local log_path="$1" + local gpu_csv="$2" + python3 - <<'PY' "$log_path" "$gpu_csv" +import csv +import re +import statistics +import sys +from pathlib import Path + +log_path = Path(sys.argv[1]) +gpu_path = Path(sys.argv[2]) + +tok_values = [] +best_val = None +tok_pat = re.compile(r"toks_per_sec=([0-9]+(?:\.[0-9]+)?)") +best_pat = re.compile(r"best_val_ppl=([0-9]+(?:\.[0-9]+)?)") +for line in log_path.read_text(encoding="utf-8", errors="replace").splitlines(): + tok_match = tok_pat.search(line) + if tok_match: + tok_values.append(float(tok_match.group(1))) + best_match = best_pat.search(line) + if best_match: + best_val = float(best_match.group(1)) + +if tok_values: + avg_tok = f"{statistics.fmean(tok_values):.2f}" + max_tok = f"{max(tok_values):.2f}" +else: + avg_tok = "NA" + max_tok = "NA" + +util_values = [] +mem_values = [] +if gpu_path.exists(): + with gpu_path.open("r", encoding="utf-8", errors="replace") as f: + reader = csv.reader(f) + for row in reader: + if len(row) < 3: + continue + try: + util = float(str(row[1]).strip()) + mem = float(str(row[2]).strip()) + except ValueError: + continue + util_values.append(util) + mem_values.append(mem) + +if util_values: + avg_util = f"{statistics.fmean(util_values):.1f}" + max_util = f"{max(util_values):.1f}" +else: + avg_util = "NA" + max_util = "NA" + +if mem_values: + avg_mem = f"{statistics.fmean(mem_values):.0f}" + peak_mem = f"{max(mem_values):.0f}" +else: + avg_mem = "NA" + peak_mem = "NA" + +best_text = f"{best_val:.4f}" if best_val is not None else "NA" +print( + "\t".join( + [ + avg_tok, + max_tok, + best_text, + avg_util, + max_util, + avg_mem, + peak_mem, + str(len(util_values)), + ] + ) +) +PY +} + +IFS=',' read -r -a profile_items <<< "$PROFILES" +for raw_profile in "${profile_items[@]}"; do + profile="$(echo "$raw_profile" | xargs)" + if [[ -z "$profile" ]]; then + continue + fi + + IFS=':' read -r context batch grad_accum <<< "$profile" + if [[ -z "${context:-}" || -z "${batch:-}" || -z "${grad_accum:-}" ]]; then + echo "error: invalid --profiles entry '$profile' (expected context:batch:grad_accum)" >&2 + exit 2 + fi + if ! [[ "$context" =~ ^[0-9]+$ && "$batch" =~ ^[0-9]+$ && "$grad_accum" =~ ^[0-9]+$ ]]; then + echo "error: non-integer profile entry '$profile'" >&2 + exit 2 + fi + if [[ "$context" -le 0 || "$batch" -le 0 || "$grad_accum" -le 0 ]]; then + echo "error: profile values must be > 0 ('$profile')" >&2 + exit 2 + fi + + run_slug="ctx${context}_b${batch}_ga${grad_accum}" + run_dir="$OUT_ROOT/$run_slug" + log_path="$run_dir/train.log" + gpu_csv="$run_dir/gpu_samples.csv" + mkdir -p "$run_dir" + + echo "[ctx-bench] start profile=$run_slug run_dir=$run_dir" + + sampler_pid="" + if [[ "$has_nvidia_smi" -eq 1 ]]; then + : > "$gpu_csv" + ( + while true; do + ts="$(date +%s)" + line="$(nvidia-smi --query-gpu=utilization.gpu,memory.used --format=csv,noheader,nounits 2>/dev/null | head -n 1 || true)" + if [[ -n "$line" ]]; then + util="$(echo "$line" | cut -d',' -f1 | tr -d ' ')" + mem="$(echo "$line" | cut -d',' -f2 | tr -d ' ')" + echo "$ts,$util,$mem" >> "$gpu_csv" + fi + sleep "$SAMPLE_SECONDS" + done + ) & + sampler_pid=$! + fi + + cmd=( + .venv/bin/python -u -m llm.cli train + --shards-path "$SHARDS_PATH" + --output-dir "$run_dir" + --device "$DEVICE" + --max-steps "$MAX_STEPS" + --batch-size "$batch" + --grad-accum-steps "$grad_accum" + --context-length "$context" + --n-layers "$N_LAYERS" + --n-heads "$N_HEADS" + --d-model "$D_MODEL" + --learning-rate "$LEARNING_RATE" + --lr-schedule cosine + --lr-warmup-steps "$LR_WARMUP_STEPS" + --lr-min-ratio "$LR_MIN_RATIO" + --eval-interval "$EVAL_INTERVAL" + --eval-steps "$EVAL_STEPS" + --log-interval "$LOG_INTERVAL" + --precision "$PRECISION" + --checkpoint-keep-last 1 + --checkpoint-keep-every 0 + ) + + if [[ "$COMPILE_MODEL" -eq 1 ]]; then + cmd+=(--compile-model) + fi + + set +e + PYTORCH_ALLOC_CONF=expandable_segments:True \ + PYTHONPATH=src \ + "${cmd[@]}" > "$log_path" 2>&1 + rc=$? + set -e + + if [[ -n "$sampler_pid" ]]; then + kill "$sampler_pid" >/dev/null 2>&1 || true + wait "$sampler_pid" 2>/dev/null || true + fi + + metric_line="$(parse_metrics "$log_path" "$gpu_csv")" + IFS=$'\t' read -r avg_tok max_tok best_val avg_util max_util avg_mem peak_mem gpu_samples <<< "$metric_line" + effective_batch=$((batch * grad_accum)) + + echo -e "${run_slug}\t${context}\t${batch}\t${grad_accum}\t${effective_batch}\t${avg_tok}\t${max_tok}\t${best_val}\t${avg_util}\t${max_util}\t${avg_mem}\t${peak_mem}\t${gpu_samples}\t${rc}\t${run_dir}" >> "$SUMMARY_TSV" + + echo "[ctx-bench] done profile=$run_slug rc=$rc avg_toks_per_sec=$avg_tok max_toks_per_sec=$max_tok peak_gpu_mem_mib=$peak_mem" +done + +echo "[ctx-bench] summary=$SUMMARY_TSV" diff --git a/scripts/checkpoint_step_offload.sh b/scripts/checkpoint_step_offload.sh new file mode 100755 index 0000000..b3f3deb --- /dev/null +++ b/scripts/checkpoint_step_offload.sh @@ -0,0 +1,239 @@ +#!/usr/bin/env bash +set -euo pipefail + +usage() { + cat <<'USAGE' +Usage: + bash scripts/checkpoint_step_offload.sh [options] + +Offload older checkpoint step files (ckpt_step_*.pt) to warm storage while +keeping the newest step files local for fast resume. + +Options: + --local-checkpoints-dir DIR Local checkpoints root (default: artifacts/checkpoints) + --warm-checkpoints-dir DIR Warm checkpoints root (default: /mnt/ceph/llm/data/checkpoints) + --keep-last-steps N Keep newest N step checkpoints per run locally (default: 6) + --max-files N Maximum candidate files to process this run (default: 24, 0 = unlimited) + --run-name NAME Restrict to one run directory name (repeatable) + --dry-run Print planned actions without copy/delete + -h, --help Show help + +Notes: + - Only ckpt_step_*.pt files are offloaded. + - last.pt, best.pt, rollback checkpoints, and safetensors are untouched. + - A file is deleted locally only after warm copy size matches. +USAGE +} + +LOCAL_CHECKPOINTS_DIR="artifacts/checkpoints" +WARM_CHECKPOINTS_DIR="/mnt/ceph/llm/data/checkpoints" +KEEP_LAST_STEPS=6 +MAX_FILES=24 +DRY_RUN=0 +RUN_NAMES=() + +while [[ $# -gt 0 ]]; do + case "$1" in + --local-checkpoints-dir) + LOCAL_CHECKPOINTS_DIR="${2:-}" + shift 2 + ;; + --warm-checkpoints-dir) + WARM_CHECKPOINTS_DIR="${2:-}" + shift 2 + ;; + --keep-last-steps) + KEEP_LAST_STEPS="${2:-}" + shift 2 + ;; + --max-files) + MAX_FILES="${2:-}" + shift 2 + ;; + --run-name) + RUN_NAMES+=("${2:-}") + shift 2 + ;; + --dry-run) + DRY_RUN=1 + shift + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "error: unknown argument: $1" >&2 + usage >&2 + exit 2 + ;; + esac +done + +if ! [[ "$KEEP_LAST_STEPS" =~ ^[0-9]+$ ]]; then + echo "error: --keep-last-steps must be a non-negative integer" >&2 + exit 2 +fi +if ! [[ "$MAX_FILES" =~ ^[0-9]+$ ]]; then + echo "error: --max-files must be a non-negative integer" >&2 + exit 2 +fi + +timestamp() { + date '+%Y-%m-%d %H:%M:%S' +} + +log() { + printf '[%s] %s\n' "$(timestamp)" "$1" +} + +to_abs() { + if command -v realpath >/dev/null 2>&1; then + realpath -m "$1" + else + readlink -f "$1" 2>/dev/null || echo "$1" + fi +} + +LOCAL_CHECKPOINTS_DIR="$(to_abs "$LOCAL_CHECKPOINTS_DIR")" +WARM_CHECKPOINTS_DIR="$(to_abs "$WARM_CHECKPOINTS_DIR")" + +if [[ ! -d "$LOCAL_CHECKPOINTS_DIR" ]]; then + log "skip local_checkpoints_missing path=$LOCAL_CHECKPOINTS_DIR" + exit 0 +fi + +if [[ "$DRY_RUN" -ne 1 ]]; then + mkdir -p "$WARM_CHECKPOINTS_DIR" +else + log "dry_run mkdir -p $WARM_CHECKPOINTS_DIR" +fi + +tmp_dir="$(mktemp -d)" +cleanup() { + rm -rf "$tmp_dir" +} +trap cleanup EXIT + +candidates_tsv="$tmp_dir/candidates.tsv" +: > "$candidates_tsv" + +declare -a run_dirs=() +if [[ "${#RUN_NAMES[@]}" -gt 0 ]]; then + for run_name in "${RUN_NAMES[@]}"; do + run_dir="$LOCAL_CHECKPOINTS_DIR/$run_name" + if [[ -d "$run_dir" ]]; then + run_dirs+=("$run_dir") + else + log "skip run_missing name=$run_name path=$run_dir" + fi + done +else + while IFS= read -r line; do + [[ -z "$line" ]] && continue + run_dirs+=("$line") + done < <(find "$LOCAL_CHECKPOINTS_DIR" -mindepth 1 -maxdepth 1 -type d | sort) +fi + +if [[ "${#run_dirs[@]}" -eq 0 ]]; then + log "skip no_run_dirs" + exit 0 +fi + +extract_step_num() { + local fname="$1" + local stem="${fname#ckpt_step_}" + stem="${stem%.pt}" + echo $((10#$stem)) +} + +for run_dir in "${run_dirs[@]}"; do + run_name="$(basename "$run_dir")" + mapfile -t entries < <( + find "$run_dir" -maxdepth 1 -type f -name 'ckpt_step_*.pt' -printf '%f\t%p\t%s\n' | sort -t$'\t' -k1,1 + ) + count="${#entries[@]}" + if [[ "$count" -le "$KEEP_LAST_STEPS" ]]; then + log "skip run=$run_name reason=insufficient_step_files count=$count keep_last_steps=$KEEP_LAST_STEPS" + continue + fi + + candidate_count=$((count - KEEP_LAST_STEPS)) + for ((idx=0; idx> "$candidates_tsv" + done +done + +if [[ ! -s "$candidates_tsv" ]]; then + log "skip no_step_candidates" + exit 0 +fi + +sort -n -t$'\t' -k1,1 "$candidates_tsv" > "$tmp_dir/candidates.sorted.tsv" + +processed=0 +copied=0 +removed=0 +skipped_existing=0 +failed=0 + +while IFS=$'\t' read -r step_num src run_name src_size; do + [[ -z "$src" ]] && continue + if [[ "$MAX_FILES" -gt 0 && "$processed" -ge "$MAX_FILES" ]]; then + break + fi + processed=$((processed + 1)) + + if [[ ! -f "$src" ]]; then + log "skip source_missing run=$run_name step=$step_num path=$src" + failed=$((failed + 1)) + continue + fi + + dst_dir="$WARM_CHECKPOINTS_DIR/$run_name" + dst="$dst_dir/$(basename "$src")" + dst_tmp="${dst}.tmp_offload" + action="copy" + + if [[ -f "$dst" ]]; then + dst_size="$(stat -c%s "$dst" 2>/dev/null || echo -1)" + if [[ "$dst_size" -eq "$src_size" ]]; then + action="reuse" + fi + fi + + if [[ "$DRY_RUN" -eq 1 ]]; then + log "dry_run step_offload run=$run_name step=$step_num action=$action src=$src dst=$dst" + continue + fi + + mkdir -p "$dst_dir" + if [[ "$action" == "copy" ]]; then + rm -f "$dst_tmp" + if ! rsync -a "$src" "$dst_tmp"; then + log "copy_fail run=$run_name step=$step_num src=$src dst_tmp=$dst_tmp" + failed=$((failed + 1)) + rm -f "$dst_tmp" + continue + fi + mv -f "$dst_tmp" "$dst" + copied=$((copied + 1)) + else + skipped_existing=$((skipped_existing + 1)) + fi + + dst_size_after="$(stat -c%s "$dst" 2>/dev/null || echo -1)" + if [[ "$dst_size_after" -ne "$src_size" ]]; then + log "verify_fail run=$run_name step=$step_num src_size=$src_size dst_size=$dst_size_after src=$src dst=$dst" + failed=$((failed + 1)) + continue + fi + + rm -f "$src" + removed=$((removed + 1)) + log "offload_done run=$run_name step=$step_num src_removed=1 dst=$dst" +done < "$tmp_dir/candidates.sorted.tsv" + +log "done processed=$processed copied=$copied removed=$removed reused_existing=$skipped_existing failed=$failed dry_run=$DRY_RUN keep_last_steps=$KEEP_LAST_STEPS max_files=$MAX_FILES" diff --git a/scripts/fineweb_prefetch_hot_queue.sh b/scripts/fineweb_prefetch_hot_queue.sh deleted file mode 100755 index f3b1179..0000000 --- a/scripts/fineweb_prefetch_hot_queue.sh +++ /dev/null @@ -1,263 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -WARM_PARQUET_DIR="/mnt/ceph/llm/data/fineweb/sample-350BT/sample/350BT" -HOT_PARQUET_DIR="data/fineweb/sample-350BT/sample/350BT" -QUEUE_MIN_FILES=12 -STAGE_MAX_FILES=8 -STAGE_MAX_GIB=0 -STAGE_MIN_AGE_SECONDS=180 -STAGE_COPY_JOBS=1 -MIN_FREE_GIB=50 -SLEEP_SECONDS=60 -ITERATIONS=0 -SKIP_LIST="" -AUTO_SKIP_STATE_DIR="artifacts/reports/fineweb_stage_shard_loop" -AUTO_SKIP=1 -DRY_RUN=0 -STATE_DIR="artifacts/reports/fineweb_prefetch_hot_queue" - -usage() { - cat <<'USAGE' -Usage: - bash scripts/fineweb_prefetch_hot_queue.sh [options] - -Maintain a warm->hot parquet queue by staging additional files when hot queue -count drops below --queue-min-files. - -Options: - --warm-parquet-dir DIR Warm source parquet directory - --hot-parquet-dir DIR Hot parquet directory - --queue-min-files N Target minimum number of hot parquet files - (default: 12) - --stage-max-files N Max files to stage in one cycle (default: 8) - --stage-copy-jobs N Parallel warm->hot copy workers per stage cycle - (default: 1) - --stage-max-gib N Max GiB to stage in one cycle (default: 0 unlimited) - --stage-min-age-seconds N Ignore warm files newer than this age (default: 180) - --min-free-gib N Skip staging if hot filesystem free GiB is below this - threshold (default: 50) - --sleep-seconds N Sleep between cycles (default: 60) - --iterations N Number of cycles (default: 0 infinite) - --skip-list FILE Optional parquet basename skip list - --auto-skip-state-dir DIR Stage-loop state dir for auto skip list - (default: artifacts/reports/fineweb_stage_shard_loop) - --no-auto-skip Disable auto skip-list discovery/merge - --state-dir DIR Log/state directory - --dry-run Print actions without copying files - -h, --help Show help - -Example: - bash scripts/fineweb_prefetch_hot_queue.sh \ - --queue-min-files 16 \ - --stage-max-files 10 \ - --sleep-seconds 45 -USAGE -} - -while [[ $# -gt 0 ]]; do - case "$1" in - --warm-parquet-dir) - WARM_PARQUET_DIR="$2" - shift 2 - ;; - --hot-parquet-dir) - HOT_PARQUET_DIR="$2" - shift 2 - ;; - --queue-min-files) - QUEUE_MIN_FILES="$2" - shift 2 - ;; - --stage-max-files) - STAGE_MAX_FILES="$2" - shift 2 - ;; - --stage-copy-jobs) - STAGE_COPY_JOBS="$2" - shift 2 - ;; - --stage-max-gib) - STAGE_MAX_GIB="$2" - shift 2 - ;; - --stage-min-age-seconds) - STAGE_MIN_AGE_SECONDS="$2" - shift 2 - ;; - --min-free-gib) - MIN_FREE_GIB="$2" - shift 2 - ;; - --sleep-seconds) - SLEEP_SECONDS="$2" - shift 2 - ;; - --iterations) - ITERATIONS="$2" - shift 2 - ;; - --skip-list) - SKIP_LIST="$2" - shift 2 - ;; - --auto-skip-state-dir) - AUTO_SKIP_STATE_DIR="$2" - shift 2 - ;; - --no-auto-skip) - AUTO_SKIP=0 - shift - ;; - --state-dir) - STATE_DIR="$2" - shift 2 - ;; - --dry-run) - DRY_RUN=1 - shift - ;; - -h|--help) - usage - exit 0 - ;; - *) - echo "unknown argument: $1" >&2 - usage - exit 2 - ;; - esac -done - -if [[ ! -d "$WARM_PARQUET_DIR" ]]; then - echo "error: warm parquet dir not found: $WARM_PARQUET_DIR" >&2 - exit 1 -fi -if [[ ! -d "$HOT_PARQUET_DIR" ]]; then - echo "error: hot parquet dir not found: $HOT_PARQUET_DIR" >&2 - exit 1 -fi -if [[ "$QUEUE_MIN_FILES" -lt 1 ]]; then - echo "error: queue-min-files must be >= 1" >&2 - exit 1 -fi -if ! [[ "$STAGE_COPY_JOBS" =~ ^[1-9][0-9]*$ ]]; then - echo "error: stage-copy-jobs must be a positive integer" >&2 - exit 1 -fi -if [[ -n "$SKIP_LIST" && ! -f "$SKIP_LIST" ]]; then - echo "error: skip-list not found: $SKIP_LIST" >&2 - exit 1 -fi -if [[ "$SLEEP_SECONDS" -lt 1 ]]; then - echo "error: sleep-seconds must be >= 1" >&2 - exit 1 -fi - -mkdir -p "$STATE_DIR" -LOCK_FILE="$STATE_DIR/prefetch.lock" -exec 9>"$LOCK_FILE" -if ! flock -n 9; then - echo "error: another fineweb_prefetch_hot_queue instance is already running" >&2 - exit 1 -fi - -LOG_FILE="$STATE_DIR/prefetch_$(date +%Y%m%d_%H%M%S).log" -AUTO_SKIP_FILE="$STATE_DIR/auto_skip_list.txt" -log() { - echo "[$(date -Iseconds)] $*" | tee -a "$LOG_FILE" -} - -hot_count() { - find "$HOT_PARQUET_DIR" -maxdepth 1 -type f -name '*.parquet' | wc -l | tr -d ' ' -} - -hot_free_gib() { - local avail_kib - avail_kib="$(df -Pk "$HOT_PARQUET_DIR" | awk 'NR==2 {print $4}')" - awk -v kib="$avail_kib" 'BEGIN { printf "%.2f", kib/1024/1024 }' -} - -resolve_skip_list() { - if [[ -n "$SKIP_LIST" ]]; then - echo "$SKIP_LIST" - return - fi - if [[ "$AUTO_SKIP" -ne 1 ]]; then - echo "" - return - fi - - local direct_file="$AUTO_SKIP_STATE_DIR/stage_skip_list.txt" - if [[ -f "$direct_file" ]]; then - echo "$direct_file" - return - fi - - : > "$AUTO_SKIP_FILE" - local src - for src in \ - "$AUTO_SKIP_STATE_DIR/processed_parquet_files.txt" \ - "$AUTO_SKIP_STATE_DIR/bad_parquet_files.txt" - do - if [[ -s "$src" ]]; then - cat "$src" >> "$AUTO_SKIP_FILE" - fi - done - if [[ -s "$AUTO_SKIP_FILE" ]]; then - sort -u "$AUTO_SKIP_FILE" -o "$AUTO_SKIP_FILE" - echo "$AUTO_SKIP_FILE" - return - fi - echo "" -} - -cycle=0 -log "prefetch_start warm=$WARM_PARQUET_DIR hot=$HOT_PARQUET_DIR queue_min_files=$QUEUE_MIN_FILES stage_max_files=$STAGE_MAX_FILES stage_copy_jobs=$STAGE_COPY_JOBS stage_max_gib=$STAGE_MAX_GIB min_free_gib=$MIN_FREE_GIB auto_skip=$AUTO_SKIP auto_skip_state_dir=$AUTO_SKIP_STATE_DIR dry_run=$DRY_RUN" - -while true; do - cycle=$((cycle + 1)) - current_hot="$(hot_count)" - free_gib="$(hot_free_gib)" - - if awk "BEGIN {exit !($free_gib < $MIN_FREE_GIB)}"; then - log "cycle=$cycle action=skip reason=low_free_space free_gib=$free_gib threshold=$MIN_FREE_GIB hot_files=$current_hot" - elif [[ "$current_hot" -lt "$QUEUE_MIN_FILES" ]]; then - need=$((QUEUE_MIN_FILES - current_hot)) - stage_files="$STAGE_MAX_FILES" - if [[ "$stage_files" -gt "$need" ]]; then - stage_files="$need" - fi - - stage_cmd=(bash scripts/stage_fineweb_from_warm.sh - --src-dir "$WARM_PARQUET_DIR" - --dest-dir "$HOT_PARQUET_DIR" - --max-files "$stage_files" - --copy-jobs "$STAGE_COPY_JOBS" - --max-gib "$STAGE_MAX_GIB" - --min-age-seconds "$STAGE_MIN_AGE_SECONDS" - --min-free-gib "$MIN_FREE_GIB" - ) - resolved_skip_list="$(resolve_skip_list)" - if [[ -n "$resolved_skip_list" ]]; then - stage_cmd+=(--skip-list "$resolved_skip_list") - fi - if [[ "$DRY_RUN" -eq 1 ]]; then - stage_cmd+=(--dry-run) - fi - - log "cycle=$cycle action=stage hot_files=$current_hot need=$need stage_files=$stage_files stage_copy_jobs=$STAGE_COPY_JOBS free_gib=$free_gib skip_list=${resolved_skip_list:-none}" - "${stage_cmd[@]}" >> "$LOG_FILE" 2>&1 || { - log "cycle=$cycle action=stage_failed" - } - else - log "cycle=$cycle action=idle hot_files=$current_hot free_gib=$free_gib" - fi - - if [[ "$ITERATIONS" -gt 0 && "$cycle" -ge "$ITERATIONS" ]]; then - log "prefetch_done cycles=$cycle" - break - fi - - sleep "$SLEEP_SECONDS" -done diff --git a/scripts/fineweb_stage_shard_loop.sh b/scripts/fineweb_stage_shard_loop.sh index f39be4a..898fe74 100755 --- a/scripts/fineweb_stage_shard_loop.sh +++ b/scripts/fineweb_stage_shard_loop.sh @@ -3,6 +3,8 @@ set -euo pipefail WARM_PARQUET_DIR="/mnt/ceph/llm/data/fineweb/sample-350BT/sample/350BT" HOT_PARQUET_DIR="data/fineweb/sample-350BT/sample/350BT" +SOURCE_PARQUET_DIR="$WARM_PARQUET_DIR" +USE_STAGE_COPY=0 SHARDS_ROOT="data/shards_global/fineweb-global-bpe-v1" TOKENIZER_PATH="artifacts/tokenizer/fineweb-global-bpe-v1.json" STATE_DIR="artifacts/reports/fineweb_stage_shard_loop" @@ -16,6 +18,10 @@ ENCODE_BATCH_SIZE=1024 TOKENIZER_THREADS=10 SHARD_JOBS=1 SHARD_RETRY_ON_OOM=1 +SHARD_RESTAGE_ON_CORRUPT=1 +DEEP_VALIDATE_STAGE_NEW_FILES=1 +DEEP_VALIDATE_MAX_BATCHES=8 +DEEP_VALIDATE_BATCH_SIZE=65536 SHARD_MIN_BATCH_SIZE=512 SHARD_SIZE_TOKENS=20000000 VAL_RATIO=0.01 @@ -31,6 +37,9 @@ STAGE_COPY_JOBS=1 STAGE_MIN_FREE_GIB=0 HOT_QUEUE_MIN_FILES=8 PROCESS_MAX_FILES=10 +EXPECTED_UNIQUE_INPUT_FILES=510 +COVERAGE_COMPLETE_MODE=1 +COVERAGE_COMPLETE_SLEEP_SECONDS=300 AUTO_TUNE_SHARD_JOBS=0 AUTO_TUNE_MIN_SHARD_JOBS=1 @@ -51,7 +60,11 @@ ITERATIONS=0 SYNC_TO_WARM=1 SYNC_BACKGROUND=0 SYNC_MAX_INFLIGHT=2 -PURGE_HOT=1 +PURGE_HOT=0 +AUTO_OFFLOAD_SHARDS=1 +HOT_MAX_USED_PCT=80 +OFFLOAD_CHECK_INTERVAL_SECONDS=120 +PARQUET_VALIDATE_TIMEOUT_SECONDS=180 DRY_RUN=0 CPU_CORES=1 LAST_BATCH_SECONDS=0 @@ -68,6 +81,10 @@ and purge processed hot parquet files. Options: --warm-parquet-dir DIR Warm source parquet dir --hot-parquet-dir DIR Hot local parquet dir + --source-parquet-dir DIR Parquet source directory used by shard builder + (default: warm parquet dir) + --enable-stage-copy Enable warm->hot parquet staging (legacy mode) + instead of direct source reads --shards-root DIR Root for new shard batch directories --tokenizer-path FILE Shared tokenizer path (train once, then reuse) --state-dir DIR State/log directory @@ -86,6 +103,12 @@ Options: --hot-queue-min-files N Try to keep at least N parquet files staged on hot disk (default: 8) --process-max-files N Max staged files to shard per batch (default: 10) + --expected-unique-input-files N + Coverage target for manifest unique input files + (default: 510, 0 disables) + --no-coverage-complete-mode Disable training-focused mode switch after full coverage + --coverage-complete-sleep-seconds N + Sleep interval after coverage completion (default: 300) --field NAME Parquet text field (default: text) --batch-size N Parquet read batch size (default: 8192) @@ -119,6 +142,18 @@ Options: --auto-tune-iowait-high-pct N Force copy scale-down when iowait >= N (default: 30) --no-shard-retry-on-oom Disable automatic shard build retry on OOM/memory errors + --no-shard-restage-on-corrupt + Disable one-time warm->hot restage retry on + corrupt-parquet shard failures + --no-deep-validate-stage-new-files + Skip deep parquet content checks on newly staged + files (default: enabled) + --deep-validate-max-batches N + Per-file batch scan cap for deep validation + (default: 8, 0 = full scan) + --deep-validate-batch-size N + Batch size for deep parquet validation scan + (default: 65536) --shard-min-batch-size N Lower bound for batch-size backoff on OOM (default: 512) --shard-size-tokens N Tokens per output shard (default: 5000000) --val-ratio X Validation ratio (default: 0.01) @@ -133,6 +168,16 @@ Options: --sync-background Run warm sync in background (non-blocking per batch) --sync-max-inflight N Max in-flight background warm sync jobs (default: 2) --no-purge-hot Keep processed parquet files on hot storage + --no-auto-offload-shards Disable automatic shard offload when hot usage + exceeds threshold + --hot-max-used-pct N Trigger shard offload when hot disk used percent + is above N (default: 80) + --offload-check-interval-seconds N + Minimum seconds between offload checks/triggers + (default: 120) + --parquet-validate-timeout-seconds N + Timeout for parquet validation/deep-validation + calls (default: 180) --dry-run Print actions without executing shard build/deletes -h, --help Show help @@ -156,6 +201,14 @@ while [[ $# -gt 0 ]]; do HOT_PARQUET_DIR="$2" shift 2 ;; + --source-parquet-dir) + SOURCE_PARQUET_DIR="$2" + shift 2 + ;; + --enable-stage-copy) + USE_STAGE_COPY=1 + shift + ;; --shards-root) SHARDS_ROOT="$2" shift 2 @@ -208,6 +261,18 @@ while [[ $# -gt 0 ]]; do PROCESS_MAX_FILES="$2" shift 2 ;; + --expected-unique-input-files) + EXPECTED_UNIQUE_INPUT_FILES="$2" + shift 2 + ;; + --no-coverage-complete-mode) + COVERAGE_COMPLETE_MODE=0 + shift + ;; + --coverage-complete-sleep-seconds) + COVERAGE_COMPLETE_SLEEP_SECONDS="$2" + shift 2 + ;; --field) FIELD="$2" shift 2 @@ -284,6 +349,22 @@ while [[ $# -gt 0 ]]; do SHARD_RETRY_ON_OOM=0 shift ;; + --no-shard-restage-on-corrupt) + SHARD_RESTAGE_ON_CORRUPT=0 + shift + ;; + --no-deep-validate-stage-new-files) + DEEP_VALIDATE_STAGE_NEW_FILES=0 + shift + ;; + --deep-validate-max-batches) + DEEP_VALIDATE_MAX_BATCHES="$2" + shift 2 + ;; + --deep-validate-batch-size) + DEEP_VALIDATE_BATCH_SIZE="$2" + shift 2 + ;; --shard-min-batch-size) SHARD_MIN_BATCH_SIZE="$2" shift 2 @@ -336,6 +417,22 @@ while [[ $# -gt 0 ]]; do PURGE_HOT=0 shift ;; + --no-auto-offload-shards) + AUTO_OFFLOAD_SHARDS=0 + shift + ;; + --hot-max-used-pct) + HOT_MAX_USED_PCT="$2" + shift 2 + ;; + --offload-check-interval-seconds) + OFFLOAD_CHECK_INTERVAL_SECONDS="$2" + shift 2 + ;; + --parquet-validate-timeout-seconds) + PARQUET_VALIDATE_TIMEOUT_SECONDS="$2" + shift 2 + ;; --dry-run) DRY_RUN=1 shift @@ -377,10 +474,14 @@ require_positive_int "stage-copy-jobs" "$STAGE_COPY_JOBS" require_nonneg_int "stage-min-free-gib" "$STAGE_MIN_FREE_GIB" require_nonneg_int "hot-queue-min-files" "$HOT_QUEUE_MIN_FILES" require_nonneg_int "process-max-files" "$PROCESS_MAX_FILES" +require_nonneg_int "expected-unique-input-files" "$EXPECTED_UNIQUE_INPUT_FILES" +require_positive_int "coverage-complete-sleep-seconds" "$COVERAGE_COMPLETE_SLEEP_SECONDS" require_positive_int "batch-size" "$BATCH_SIZE" require_positive_int "encode-batch-size" "$ENCODE_BATCH_SIZE" require_positive_int "tokenizer-threads" "$TOKENIZER_THREADS" require_positive_int "shard-jobs" "$SHARD_JOBS" +require_nonneg_int "deep-validate-max-batches" "$DEEP_VALIDATE_MAX_BATCHES" +require_positive_int "deep-validate-batch-size" "$DEEP_VALIDATE_BATCH_SIZE" require_positive_int "shard-min-batch-size" "$SHARD_MIN_BATCH_SIZE" require_positive_int "shard-size-tokens" "$SHARD_SIZE_TOKENS" require_nonneg_int "min-chars" "$MIN_CHARS" @@ -389,6 +490,9 @@ require_nonneg_int "max-rows-per-file" "$MAX_ROWS_PER_FILE" require_positive_int "sleep-seconds" "$SLEEP_SECONDS" require_nonneg_int "iterations" "$ITERATIONS" require_positive_int "sync-max-inflight" "$SYNC_MAX_INFLIGHT" +require_nonneg_int "hot-max-used-pct" "$HOT_MAX_USED_PCT" +require_positive_int "offload-check-interval-seconds" "$OFFLOAD_CHECK_INTERVAL_SECONDS" +require_positive_int "parquet-validate-timeout-seconds" "$PARQUET_VALIDATE_TIMEOUT_SECONDS" require_positive_int "auto-tune-min-shard-jobs" "$AUTO_TUNE_MIN_SHARD_JOBS" require_positive_int "auto-tune-max-shard-jobs" "$AUTO_TUNE_MAX_SHARD_JOBS" @@ -426,11 +530,27 @@ if [[ "$AUTO_TUNE_IOWAIT_HIGH_PCT" -gt 100 ]]; then echo "error: auto-tune-iowait-high-pct must be <= 100" >&2 exit 2 fi +if [[ "$HOT_MAX_USED_PCT" -gt 100 ]]; then + echo "error: hot-max-used-pct must be <= 100" >&2 + exit 2 +fi + +if [[ "$USE_STAGE_COPY" -ne 1 ]]; then + PURGE_HOT=0 + SHARD_RESTAGE_ON_CORRUPT=0 +fi if [[ ! -d "$WARM_PARQUET_DIR" ]]; then echo "error: warm parquet dir not found: $WARM_PARQUET_DIR" >&2 exit 1 fi +if [[ ! -d "$SOURCE_PARQUET_DIR" ]]; then + echo "error: source parquet dir not found: $SOURCE_PARQUET_DIR" >&2 + exit 1 +fi +if [[ "$USE_STAGE_COPY" -eq 1 ]] && [[ ! -d "$HOT_PARQUET_DIR" ]]; then + mkdir -p "$HOT_PARQUET_DIR" +fi if [[ ! -x ".venv/bin/python" ]]; then echo "error: .venv/bin/python not found; run make setup-train first" >&2 @@ -502,9 +622,26 @@ warm_tokenizer_dir="$WARM_ROOT/tokenizer" warm_reports_dir="$WARM_ROOT/reports/fineweb_stage_shard_loop" declare -a SYNC_PIDS=() declare -a ACTIVE_SHARD_PIDS=() +OFFLOAD_PID="" +OFFLOAD_LAST_CHECK_EPOCH=0 SHOULD_STOP=0 IOWAIT_PREV_TOTAL="" IOWAIT_PREV_WAIT="" +COVERAGE_COMPLETE_REACHED=0 + +OFFLOAD_ARGS_DEFAULT="--shards-root $SHARDS_ROOT --warm-shards-root $WARM_ROOT/shards_global/$(basename "$SHARDS_ROOT") --keep-local-batches 24 --target-free-gib 180 --max-batches 24 --disable-offloaded-manifests --require-trained-batches-file artifacts/reports/train_supervisor_phase1_talk/trained_batch_names.txt,artifacts/reports/train_supervisor_350bt/trained_batch_names.txt --skip-if-trained-file-missing --min-active-manifests 48 --min-active-train-tokens 40000000000" +OFFLOAD_ARGS_RAW="${LLM_SHARD_OFFLOAD_ARGS:-$OFFLOAD_ARGS_DEFAULT}" + +expand_args() { + local raw="$1" + local -n out_ref="$2" + out_ref=() + eval "set -- $raw" + while [[ $# -gt 0 ]]; do + out_ref+=("$1") + shift + done +} retune_tokenizer_threads() { local jobs="$1" @@ -653,6 +790,54 @@ maybe_auto_tune_stage_copy_jobs() { log "auto_tune_copy_update reason=$reason eligible_hot_count=$eligible_hot_count queue_min=$HOT_QUEUE_MIN_FILES deficit=$deficit iowait_pct=$iowait_pct copy_jobs=$old_jobs->$STAGE_COPY_JOBS" } +manifest_unique_input_count() { + PYTHONPATH=src .venv/bin/python - "$SHARDS_ROOT" <<'PY' +import json +import sys +from pathlib import Path + +root = Path(sys.argv[1]) +unique: set[str] = set() +manifests = list(root.rglob("manifest.json")) +manifests.extend(root.rglob("manifest.offloaded.json")) +for manifest_path in manifests: + try: + payload = json.loads(manifest_path.read_text(encoding="utf-8")) + except Exception: + continue + raw_files = payload.get("input_files", []) + if not isinstance(raw_files, list): + continue + for raw in raw_files: + name = Path(str(raw)).name + if name: + unique.add(name) +print(len(unique)) +PY +} + +maybe_enter_coverage_complete_mode() { + if [[ "$COVERAGE_COMPLETE_MODE" -ne 1 ]]; then + return + fi + if [[ "$EXPECTED_UNIQUE_INPUT_FILES" -le 0 ]]; then + return + fi + local unique_inputs + unique_inputs="$(manifest_unique_input_count)" + if ! [[ "$unique_inputs" =~ ^[0-9]+$ ]]; then + return + fi + if [[ "$unique_inputs" -lt "$EXPECTED_UNIQUE_INPUT_FILES" ]]; then + return + fi + if [[ "$COVERAGE_COMPLETE_REACHED" -eq 1 ]]; then + return + fi + COVERAGE_COMPLETE_REACHED=1 + log "coverage_complete_mode_enter unique_inputs=$unique_inputs expected=$EXPECTED_UNIQUE_INPUT_FILES action=pause_stage_and_shard" +} + is_bad_parquet_name() { local name="$1" grep -Fqx "$name" "$BAD_PARQUET_FILE" @@ -722,6 +907,9 @@ PY } purge_hot_known_files() { + if [[ "$USE_STAGE_COPY" -ne 1 ]]; then + return + fi local removed=0 mapfile -t hot_files < <(find "$HOT_PARQUET_DIR" -maxdepth 1 -type f -name '*.parquet' -printf '%f\n' | sort) for name in "${hot_files[@]}"; do @@ -740,7 +928,7 @@ purge_hot_known_files() { count_hot_eligible_files() { local count=0 - mapfile -t hot_files < <(find "$HOT_PARQUET_DIR" -maxdepth 1 -type f -name '*.parquet' -printf '%f\n' | sort) + mapfile -t hot_files < <(find "$SOURCE_PARQUET_DIR" -maxdepth 1 -type f -name '*.parquet' -printf '%f\n' | sort) for name in "${hot_files[@]}"; do if is_bad_parquet_name "$name" || is_processed_parquet_name "$name"; then continue @@ -750,6 +938,19 @@ count_hot_eligible_files() { echo "$count" } +list_hot_eligible_files() { + local -n out_ref="$1" + out_ref=() + mapfile -t hot_files < <(find "$SOURCE_PARQUET_DIR" -maxdepth 1 -type f -name '*.parquet' -printf '%f\n' | sort) + local name + for name in "${hot_files[@]}"; do + if is_bad_parquet_name "$name" || is_processed_parquet_name "$name"; then + continue + fi + out_ref+=("$name") + done +} + mark_bad_parquet() { local parquet_path="$1" local reason="$2" @@ -758,7 +959,7 @@ mark_bad_parquet() { printf '%s\n' "$name" >> "$BAD_PARQUET_FILE" sort -u "$BAD_PARQUET_FILE" -o "$BAD_PARQUET_FILE" - if [[ -f "$parquet_path" ]]; then + if [[ "$USE_STAGE_COPY" -eq 1 ]] && [[ -f "$parquet_path" ]]; then local ts ts="$(date +%Y%m%d_%H%M%S)" local quarantine_path="$QUARANTINE_DIR/${name}.bad_${ts}" @@ -778,8 +979,85 @@ mark_bad_from_files_list() { local name while IFS= read -r name; do [[ -z "$name" ]] && continue - mark_bad_parquet "$HOT_PARQUET_DIR/$name" "$reason" + mark_bad_parquet "$SOURCE_PARQUET_DIR/$name" "$reason" + done < "$files_list" +} + +shard_log_has_corrupt_error() { + local shard_attempt_log="$1" + grep -Eqi \ + 'Corrupt snappy compressed data|parquet.*corrupt|snappy.*corrupt|invalid compressed data|page header' \ + "$shard_attempt_log" +} + +restage_files_from_list() { + if [[ "$USE_STAGE_COPY" -ne 1 ]]; then + log "restage_skip reason=stage_copy_disabled" + return 1 + fi + local files_list="$1" + local reason="$2" + if [[ ! -f "$files_list" ]]; then + return 1 + fi + + local requested + requested="$(awk 'NF {c+=1} END {print c+0}' "$files_list")" + local restaged=0 + local failed=0 + local missing=0 + local name + while IFS= read -r name; do + [[ -z "$name" ]] && continue + local warm_file="$WARM_PARQUET_DIR/$name" + local hot_file="$HOT_PARQUET_DIR/$name" + local tmp_file="$HOT_PARQUET_DIR/${name}.restage_tmp" + + if [[ ! -f "$warm_file" ]]; then + log "restage_missing_warm file=$name reason=$reason warm_file=$warm_file" + missing=$((missing + 1)) + failed=$((failed + 1)) + continue + fi + + rm -f "$tmp_file" + if ! ( + exec 9>&- + rsync -ah --partial --inplace --timeout=900 "$warm_file" "$tmp_file" + ) >> "$LOG_FILE" 2>&1; then + log "restage_copy_failed file=$name reason=$reason" + rm -f "$tmp_file" + failed=$((failed + 1)) + continue + fi + + local warm_size + local tmp_size + warm_size="$(stat -c %s "$warm_file" 2>/dev/null || echo 0)" + tmp_size="$(stat -c %s "$tmp_file" 2>/dev/null || echo 0)" + if [[ "$warm_size" -le 0 || "$tmp_size" -ne "$warm_size" ]]; then + log "restage_size_mismatch file=$name reason=$reason warm_size=$warm_size tmp_size=$tmp_size" + rm -f "$tmp_file" + failed=$((failed + 1)) + continue + fi + + mv -f "$tmp_file" "$hot_file" + if ! validate_parquet_file "$hot_file" "$FIELD" >> "$LOG_FILE" 2>&1; then + log "restage_validate_failed file=$name reason=$reason" + failed=$((failed + 1)) + continue + fi + + restaged=$((restaged + 1)) + log "restage_ok file=$name reason=$reason size_bytes=$warm_size" done < "$files_list" + + log "restage_summary reason=$reason requested=$requested restaged=$restaged failed=$failed missing_warm=$missing" + if [[ "$restaged" -le 0 || "$failed" -ne 0 ]]; then + return 1 + fi + return 0 } reconcile_bad_parquet_from_warm() { @@ -811,7 +1089,11 @@ reconcile_bad_parquet_from_warm() { validate_parquet_file() { local parquet_path="$1" local field_name="$2" - PYTHONPATH=src .venv/bin/python - "$parquet_path" "$field_name" <<'PY' + local -a cmd=(env PYTHONPATH=src .venv/bin/python - "$parquet_path" "$field_name") + if command -v timeout >/dev/null 2>&1; then + cmd=(timeout --signal=TERM "${PARQUET_VALIDATE_TIMEOUT_SECONDS}s" "${cmd[@]}") + fi + "${cmd[@]}" <<'PY' import sys from pyarrow import parquet as pq @@ -829,6 +1111,46 @@ if field not in table.schema.names: PY } +validate_parquet_file_deep() { + local parquet_path="$1" + local field_name="$2" + local scan_batch_size="$3" + local scan_max_batches="$4" + local -a cmd=(env PYTHONPATH=src .venv/bin/python - "$parquet_path" "$field_name" "$scan_batch_size" "$scan_max_batches") + if command -v timeout >/dev/null 2>&1; then + cmd=(timeout --signal=TERM "${PARQUET_VALIDATE_TIMEOUT_SECONDS}s" "${cmd[@]}") + fi + "${cmd[@]}" <<'PY' +import sys +from pyarrow import parquet as pq + +path = sys.argv[1] +field = sys.argv[2] +batch_size = int(sys.argv[3]) +max_batches = int(sys.argv[4]) + +table = pq.ParquetFile(path) +meta = table.metadata +if meta is None or meta.num_row_groups <= 0: + raise RuntimeError("missing row groups") +if meta.num_rows <= 0: + raise RuntimeError("no rows") +if field not in table.schema.names: + raise RuntimeError(f"missing field '{field}'") + +batches = 0 +for batch in table.iter_batches(columns=[field], batch_size=batch_size): + # Force decode of payload buffers so snappy/page corruption is surfaced early. + _ = batch.column(0).to_pylist() + batches += 1 + if max_batches > 0 and batches >= max_batches: + break + +if batches <= 0: + raise RuntimeError("no batches decoded") +PY +} + preflight_selected_files() { local -n in_ref="$1" local -n out_ref="$2" @@ -839,7 +1161,7 @@ preflight_selected_files() { log "preflight_skip_known_bad file=$name" continue fi - local hot_path="$HOT_PARQUET_DIR/$name" + local hot_path="$SOURCE_PARQUET_DIR/$name" if [[ ! -f "$hot_path" ]]; then log "preflight_skip_missing file=$name" continue @@ -896,13 +1218,24 @@ validate_batch_guardrails() { } stage_once() { + if [[ "$USE_STAGE_COPY" -ne 1 ]]; then + return 0 + fi refresh_stage_skip_list local hot_count local eligible_hot_count + local -a eligible_before=() + local -a eligible_after=() + local -A eligible_before_set=() hot_count="$(find "$HOT_PARQUET_DIR" -maxdepth 1 -type f -name '*.parquet' | wc -l | tr -d ' ')" eligible_hot_count="$(count_hot_eligible_files)" local stage_files="$STAGE_MAX_FILES" + list_hot_eligible_files eligible_before + local name + for name in "${eligible_before[@]}"; do + eligible_before_set["$name"]=1 + done if [[ "$HOT_QUEUE_MIN_FILES" -gt 0 ]]; then if [[ "$eligible_hot_count" -ge "$HOT_QUEUE_MIN_FILES" ]]; then @@ -954,12 +1287,39 @@ stage_once() { ) fi log "stage_done" + + if [[ "$DRY_RUN" -eq 1 || "$DEEP_VALIDATE_STAGE_NEW_FILES" -ne 1 ]]; then + return 0 + fi + + list_hot_eligible_files eligible_after + local checked=0 + local failed=0 + for name in "${eligible_after[@]}"; do + if [[ -n "${eligible_before_set[$name]+x}" ]]; then + continue + fi + local hot_path="$HOT_PARQUET_DIR/$name" + checked=$((checked + 1)) + if validate_parquet_file_deep "$hot_path" "$FIELD" "$DEEP_VALIDATE_BATCH_SIZE" "$DEEP_VALIDATE_MAX_BATCHES" >> "$LOG_FILE" 2>&1; then + log "stage_deep_validate_ok file=$name batches=$DEEP_VALIDATE_MAX_BATCHES batch_size=$DEEP_VALIDATE_BATCH_SIZE" + continue + fi + failed=$((failed + 1)) + mark_bad_parquet "$hot_path" "stage_deep_validation_failed" + done + if [[ "$checked" -gt 0 ]]; then + log "stage_deep_validate_done checked=$checked failed=$failed max_batches=$DEEP_VALIDATE_MAX_BATCHES batch_size=$DEEP_VALIDATE_BATCH_SIZE" + if [[ "$failed" -gt 0 ]]; then + refresh_stage_skip_list + fi + fi } select_unprocessed_files() { local -n out_ref="$1" out_ref=() - mapfile -t all_files < <(find "$HOT_PARQUET_DIR" -maxdepth 1 -type f -name '*.parquet' -printf '%f\n' | sort) + mapfile -t all_files < <(find "$SOURCE_PARQUET_DIR" -maxdepth 1 -type f -name '*.parquet' -printf '%f\n' | sort) for name in "${all_files[@]}"; do if grep -Fqx "$name" "$PROCESSED_FILE"; then continue @@ -1064,6 +1424,70 @@ drain_sync_jobs() { log "batch_sync_drain_done" } +prune_offload_job() { + if [[ -z "$OFFLOAD_PID" ]]; then + return + fi + if kill -0 "$OFFLOAD_PID" 2>/dev/null; then + return + fi + if wait "$OFFLOAD_PID"; then + log "offload_done pid=$OFFLOAD_PID" + else + log "offload_failed pid=$OFFLOAD_PID" + fi + OFFLOAD_PID="" +} + +hot_disk_used_pct() { + local pct + pct="$(df -P "$SHARDS_ROOT" 2>/dev/null | awk 'NR==2 {gsub(/%/, "", $5); print $5}')" + if [[ "$pct" =~ ^[0-9]+$ ]]; then + echo "$pct" + return + fi + echo "-1" +} + +maybe_trigger_shard_offload() { + if [[ "$AUTO_OFFLOAD_SHARDS" -ne 1 || "$DRY_RUN" -eq 1 ]]; then + return + fi + local now_epoch + now_epoch="$(date +%s)" + if [[ "$OFFLOAD_LAST_CHECK_EPOCH" -gt 0 ]]; then + local elapsed=$((now_epoch - OFFLOAD_LAST_CHECK_EPOCH)) + if [[ "$elapsed" -lt "$OFFLOAD_CHECK_INTERVAL_SECONDS" ]]; then + return + fi + fi + OFFLOAD_LAST_CHECK_EPOCH="$now_epoch" + + prune_offload_job + if [[ -n "$OFFLOAD_PID" ]]; then + return + fi + + local used_pct + used_pct="$(hot_disk_used_pct)" + if ! [[ "$used_pct" =~ ^[0-9]+$ ]] || [[ "$used_pct" -lt 0 ]]; then + log "offload_skip reason=disk_pct_unknown" + return + fi + if [[ "$used_pct" -le "$HOT_MAX_USED_PCT" ]]; then + return + fi + + local -a offload_args=() + expand_args "$OFFLOAD_ARGS_RAW" offload_args + ( + exec 9>&- + PYTHONPATH=src .venv/bin/python scripts/offload_shard_bins_to_warm.py "${offload_args[@]}" + ) >> "$LOG_FILE" 2>&1 & + OFFLOAD_PID="$!" + log "offload_start pid=$OFFLOAD_PID used_pct=$used_pct threshold_pct=$HOT_MAX_USED_PCT args=$OFFLOAD_ARGS_RAW" +} + terminate_active_shard_jobs() { if [[ "${#ACTIVE_SHARD_PIDS[@]}" -eq 0 ]]; then return @@ -1091,6 +1515,10 @@ on_loop_signal() { log "loop_signal signal=$sig action=terminate_shard_jobs_drain_sync_then_exit" terminate_active_shard_jobs drain_sync_jobs + prune_offload_job + if [[ -n "$OFFLOAD_PID" ]]; then + kill -TERM "$OFFLOAD_PID" 2>/dev/null || true + fi exit 0 } @@ -1116,6 +1544,7 @@ run_shard_job() { shard_batch_size="$BATCH_SIZE" local shard_attempt=1 local shard_rc=0 + local restage_retry_used=0 while true; do local shard_attempt_log="$STATE_DIR/${job_id}.shard_attempt_${shard_attempt}.log" log "shard_build_attempt id=$job_id attempt=$shard_attempt batch_size=$shard_batch_size encode_batch_size=$ENCODE_BATCH_SIZE tokenizer_threads=$TOKENIZER_THREADS" @@ -1125,7 +1554,7 @@ run_shard_job() { # Prevent shard subprocesses (including tee) from inheriting the loop lock FD. exec 9>&- TOKENIZERS_PARALLELISM=true RAYON_NUM_THREADS="$TOKENIZER_THREADS" PYTHONPATH=src .venv/bin/python scripts/fineweb_parquet_to_shards.py \ - --input-dir "$HOT_PARQUET_DIR" \ + --input-dir "$SOURCE_PARQUET_DIR" \ --files-list "$files_list" \ --output-dir "$output_dir" \ "$tok_arg_flag" "$tok_arg_path" \ @@ -1154,6 +1583,16 @@ run_shard_job() { fi if ! grep -Eqi 'out of memory|memoryerror|arrowmemoryerror|bad_alloc|cannot allocate memory|std::bad_alloc' "$shard_attempt_log"; then + if [[ "$SHARD_RESTAGE_ON_CORRUPT" -eq 1 && "$restage_retry_used" -eq 0 ]] && shard_log_has_corrupt_error "$shard_attempt_log"; then + log "shard_build_restage_retry id=$job_id attempt=$shard_attempt rc=$shard_rc reason=corrupt_error_detected" + if restage_files_from_list "$files_list" "shard_corrupt_retry"; then + restage_retry_used=1 + shard_attempt=$((shard_attempt + 1)) + continue + fi + restage_retry_used=1 + log "shard_build_restage_retry_failed id=$job_id attempt=$shard_attempt rc=$shard_rc reason=corrupt_error_detected" + fi log "shard_build_failed id=$job_id attempt=$shard_attempt rc=$shard_rc reason=non_oom_error" mark_bad_from_files_list "$files_list" "shard_build_failed_non_oom_rc_${shard_rc}" return "$shard_rc" @@ -1353,8 +1792,8 @@ process_batch() { log "batch_done id=$batch_id" } -log "loop_start warm_parquet_dir=$WARM_PARQUET_DIR hot_parquet_dir=$HOT_PARQUET_DIR" -log "loop_config shards_root=$SHARDS_ROOT tokenizer_path=$TOKENIZER_PATH iterations=$ITERATIONS hot_queue_min_files=$HOT_QUEUE_MIN_FILES stage_copy_jobs=$STAGE_COPY_JOBS stage_min_free_gib=$STAGE_MIN_FREE_GIB shard_jobs=$SHARD_JOBS tokenizer_threads=$TOKENIZER_THREADS shard_size_tokens=$SHARD_SIZE_TOKENS sync_to_warm=$SYNC_TO_WARM sync_background=$SYNC_BACKGROUND sync_max_inflight=$SYNC_MAX_INFLIGHT auto_tune_shard_jobs=$AUTO_TUNE_SHARD_JOBS auto_tune_bounds=${AUTO_TUNE_MIN_SHARD_JOBS}-${AUTO_TUNE_MAX_SHARD_JOBS} auto_tune_load_pct=${AUTO_TUNE_LOW_LOAD_PCT}-${AUTO_TUNE_HIGH_LOAD_PCT} auto_tune_core_budget=$AUTO_TUNE_CORE_BUDGET auto_tune_stage_copy_jobs=$AUTO_TUNE_STAGE_COPY_JOBS auto_tune_copy_bounds=${AUTO_TUNE_MIN_COPY_JOBS}-${AUTO_TUNE_MAX_COPY_JOBS} auto_tune_iowait_pct=${AUTO_TUNE_IOWAIT_LOW_PCT}-${AUTO_TUNE_IOWAIT_HIGH_PCT} bad_parquet_file=$BAD_PARQUET_FILE quarantine_dir=$QUARANTINE_DIR" +log "loop_start source_parquet_dir=$SOURCE_PARQUET_DIR warm_parquet_dir=$WARM_PARQUET_DIR hot_parquet_dir=$HOT_PARQUET_DIR" +log "loop_config shards_root=$SHARDS_ROOT tokenizer_path=$TOKENIZER_PATH iterations=$ITERATIONS use_stage_copy=$USE_STAGE_COPY hot_queue_min_files=$HOT_QUEUE_MIN_FILES stage_copy_jobs=$STAGE_COPY_JOBS stage_min_free_gib=$STAGE_MIN_FREE_GIB shard_jobs=$SHARD_JOBS tokenizer_threads=$TOKENIZER_THREADS shard_size_tokens=$SHARD_SIZE_TOKENS shard_retry_on_oom=$SHARD_RETRY_ON_OOM shard_restage_on_corrupt=$SHARD_RESTAGE_ON_CORRUPT deep_validate_stage_new_files=$DEEP_VALIDATE_STAGE_NEW_FILES deep_validate_max_batches=$DEEP_VALIDATE_MAX_BATCHES deep_validate_batch_size=$DEEP_VALIDATE_BATCH_SIZE parquet_validate_timeout_seconds=$PARQUET_VALIDATE_TIMEOUT_SECONDS sync_to_warm=$SYNC_TO_WARM sync_background=$SYNC_BACKGROUND sync_max_inflight=$SYNC_MAX_INFLIGHT auto_offload_shards=$AUTO_OFFLOAD_SHARDS hot_max_used_pct=$HOT_MAX_USED_PCT offload_check_interval_seconds=$OFFLOAD_CHECK_INTERVAL_SECONDS auto_tune_shard_jobs=$AUTO_TUNE_SHARD_JOBS auto_tune_bounds=${AUTO_TUNE_MIN_SHARD_JOBS}-${AUTO_TUNE_MAX_SHARD_JOBS} auto_tune_load_pct=${AUTO_TUNE_LOW_LOAD_PCT}-${AUTO_TUNE_HIGH_LOAD_PCT} auto_tune_core_budget=$AUTO_TUNE_CORE_BUDGET auto_tune_stage_copy_jobs=$AUTO_TUNE_STAGE_COPY_JOBS auto_tune_copy_bounds=${AUTO_TUNE_MIN_COPY_JOBS}-${AUTO_TUNE_MAX_COPY_JOBS} auto_tune_iowait_pct=${AUTO_TUNE_IOWAIT_LOW_PCT}-${AUTO_TUNE_IOWAIT_HIGH_PCT} expected_unique_input_files=$EXPECTED_UNIQUE_INPUT_FILES coverage_complete_mode=$COVERAGE_COMPLETE_MODE coverage_complete_sleep_seconds=$COVERAGE_COMPLETE_SLEEP_SECONDS bad_parquet_file=$BAD_PARQUET_FILE quarantine_dir=$QUARANTINE_DIR" trap 'on_loop_signal INT' INT trap 'on_loop_signal TERM' TERM reconcile_bad_parquet_from_warm @@ -1367,9 +1806,17 @@ while true; do log "loop_stop_requested reason=signal" break fi + maybe_trigger_shard_offload if [[ "$SYNC_BACKGROUND" -eq 1 ]]; then prune_sync_jobs fi + maybe_enter_coverage_complete_mode + if [[ "$COVERAGE_COMPLETE_REACHED" -eq 1 ]]; then + drain_sync_jobs + log "coverage_complete_idle sleep_seconds=$COVERAGE_COMPLETE_SLEEP_SECONDS" + sleep "$COVERAGE_COMPLETE_SLEEP_SECONDS" + continue + fi purge_hot_known_files selected=() select_unprocessed_files selected @@ -1414,10 +1861,12 @@ while true; do completed_batches=$((completed_batches + 1)) maybe_auto_tune_parallelism "$completed_batches" stage_once + maybe_trigger_shard_offload if [[ "$ITERATIONS" -gt 0 && "$completed_batches" -ge "$ITERATIONS" ]]; then log "loop_done reason=iterations_reached completed_batches=$completed_batches" drain_sync_jobs + prune_offload_job break fi done diff --git a/scripts/fineweb_stage_shard_watchdog.sh b/scripts/fineweb_stage_shard_watchdog.sh index a3380ed..f60a770 100755 --- a/scripts/fineweb_stage_shard_watchdog.sh +++ b/scripts/fineweb_stage_shard_watchdog.sh @@ -19,19 +19,22 @@ Options: --hot-parquet-dir DIR Hot parquet directory for progress snapshot --shards-root DIR Shards root for progress snapshot --processed-file FILE Stage-loop processed parquet state file + --expected-unique-input-files N + Coverage target for manifest unique input files (default: 510, 0 disables) --no-adopt-existing-loop Always launch a new stage-loop worker (do not adopt existing) --no-cleanup-stale-workers Do not terminate stale loop/shard workers before relaunch -h, --help Show help USAGE } -WORKER_ARGS="--hot-queue-min-files 10 --stage-max-files 8 --stage-copy-jobs 4 --stage-min-free-gib 80 --process-max-files 15 --shard-jobs 2 --auto-tune-shard-jobs --auto-tune-min-shard-jobs 2 --auto-tune-max-shard-jobs 3 --auto-tune-low-load-pct 80 --auto-tune-high-load-pct 95 --auto-tune-min-batch-seconds 300 --tokenizer-threads 10 --encode-batch-size 1024 --shard-size-tokens 20000000 --sync-background --sync-max-inflight 2 --sleep-seconds 60 --shard-min-batch-size 512" +WORKER_ARGS="--hot-queue-min-files 10 --stage-max-files 8 --stage-copy-jobs 4 --stage-min-free-gib 80 --process-max-files 15 --shard-jobs 2 --auto-tune-shard-jobs --auto-tune-min-shard-jobs 2 --auto-tune-max-shard-jobs 3 --auto-tune-low-load-pct 80 --auto-tune-high-load-pct 95 --auto-tune-min-batch-seconds 300 --tokenizer-threads 10 --encode-batch-size 1024 --shard-size-tokens 20000000 --expected-unique-input-files 510 --coverage-complete-sleep-seconds 300 --sync-background --sync-max-inflight 2 --sleep-seconds 60 --shard-min-batch-size 512" WATCHDOG_LOG_FILE="artifacts/reports/fineweb_stage_shard_loop/watchdog.log" CHECK_INTERVAL_SECONDS=120 STALL_SECONDS=1800 HOT_PARQUET_DIR="data/fineweb/sample-350BT/sample/350BT" SHARDS_ROOT="data/shards_global/fineweb-global-bpe-v1" PROCESSED_FILE="artifacts/reports/fineweb_stage_shard_loop/processed_parquet_files.txt" +EXPECTED_UNIQUE_INPUT_FILES=510 LOCK_FILE="" ADOPT_EXISTING_LOOP=1 CLEANUP_STALE_WORKERS=1 @@ -70,6 +73,10 @@ while [[ $# -gt 0 ]]; do PROCESSED_FILE="${2:-}" shift 2 ;; + --expected-unique-input-files) + EXPECTED_UNIQUE_INPUT_FILES="${2:-}" + shift 2 + ;; --no-adopt-existing-loop) ADOPT_EXISTING_LOOP=0 shift @@ -100,6 +107,10 @@ if ! command -v flock >/dev/null 2>&1; then echo "error: required command not found: flock" >&2 exit 1 fi +if ! [[ "$EXPECTED_UNIQUE_INPUT_FILES" =~ ^[0-9]+$ ]]; then + echo "error: --expected-unique-input-files must be >= 0" >&2 + exit 2 +fi exec 8>"$LOCK_FILE" if ! flock -n 8; then @@ -135,6 +146,32 @@ progress_snapshot() { "$incomplete_bytes" } +manifest_unique_input_count() { + PYTHONPATH=src .venv/bin/python - "$SHARDS_ROOT" <<'PY' +import json +import sys +from pathlib import Path + +root = Path(sys.argv[1]) +unique: set[str] = set() +manifests = list(root.rglob("manifest.json")) +manifests.extend(root.rglob("manifest.offloaded.json")) +for manifest_path in manifests: + try: + payload = json.loads(manifest_path.read_text(encoding="utf-8")) + except Exception: + continue + raw_files = payload.get("input_files", []) + if not isinstance(raw_files, list): + continue + for raw in raw_files: + name = Path(str(raw)).name + if name: + unique.add(name) +print(len(unique)) +PY +} + WORKER_PID="" find_stage_loop_controller_pid() { @@ -335,7 +372,7 @@ if ! start_worker; then fi last_snapshot="$(progress_snapshot)" last_progress_epoch="$(date +%s)" -log "watchdog_start interval_seconds=$CHECK_INTERVAL_SECONDS stall_seconds=$STALL_SECONDS lock_file=$LOCK_FILE adopt_existing_loop=$ADOPT_EXISTING_LOOP snapshot=$last_snapshot" +log "watchdog_start interval_seconds=$CHECK_INTERVAL_SECONDS stall_seconds=$STALL_SECONDS lock_file=$LOCK_FILE adopt_existing_loop=$ADOPT_EXISTING_LOOP expected_unique_input_files=$EXPECTED_UNIQUE_INPUT_FILES snapshot=$last_snapshot" while true; do sleep "$CHECK_INTERVAL_SECONDS" @@ -363,6 +400,14 @@ while true; do fi if (( now_epoch - last_progress_epoch >= STALL_SECONDS )); then + if [[ "$EXPECTED_UNIQUE_INPUT_FILES" -gt 0 ]]; then + unique_inputs="$(manifest_unique_input_count)" + if [[ "$unique_inputs" =~ ^[0-9]+$ ]] && [[ "$unique_inputs" -ge "$EXPECTED_UNIQUE_INPUT_FILES" ]]; then + last_progress_epoch="$now_epoch" + log "worker_hold reason=coverage_complete unique_inputs=$unique_inputs expected=$EXPECTED_UNIQUE_INPUT_FILES" + continue + fi + fi log "worker_stalled elapsed=$((now_epoch - last_progress_epoch))s snapshot=$current_snapshot restarting" stop_worker "stall_timeout" sleep 5 diff --git a/scripts/hf_download_watchdog.sh b/scripts/hf_download_watchdog.sh index 42170d3..ca84f7a 100755 --- a/scripts/hf_download_watchdog.sh +++ b/scripts/hf_download_watchdog.sh @@ -5,10 +5,10 @@ usage() { cat <<'USAGE' Usage: bash scripts/hf_download_watchdog.sh \ - --dataset HuggingFaceFW/fineweb \ - --repo-type dataset \ - --include "sample/350BT/*.parquet" \ - --local-dir /mnt/ceph/llm/data/fineweb/sample-350BT \ + [--dataset HuggingFaceFW/fineweb] \ + [--repo-type dataset] \ + [--include "sample/350BT/*.parquet"] \ + [--local-dir /mnt/ceph/llm/data/fineweb/sample-350BT] \ [--max-workers 4] \ [--enable-hf-transfer | --disable-hf-transfer] \ [--skip-dry-run] \ @@ -27,6 +27,7 @@ Notes: - Restarts the worker if it exits unexpectedly. - Detects stalls from unchanged parquet/incomplete bytes+counts and restarts. - Optional completion exit can stop watchdog once expected file/byte targets are reached. + - Dataset/include/local-dir default to FineWeb 350BT paths if not provided. USAGE } @@ -38,10 +39,10 @@ require_cmd() { fi } -DATASET="" +DATASET="HuggingFaceFW/fineweb" REPO_TYPE="dataset" -INCLUDE_PATTERN="" -LOCAL_DIR="" +INCLUDE_PATTERN="sample/350BT/*.parquet" +LOCAL_DIR="/mnt/ceph/llm/data/fineweb/sample-350BT" MAX_WORKERS=4 ENABLE_HF_TRANSFER="auto" SKIP_DRY_RUN=0 @@ -149,7 +150,7 @@ while [[ $# -gt 0 ]]; do done if [[ -z "$DATASET" || -z "$INCLUDE_PATTERN" || -z "$LOCAL_DIR" ]]; then - echo "error: --dataset, --include, and --local-dir are required" >&2 + echo "error: dataset/include/local-dir cannot be empty" >&2 usage >&2 exit 2 fi diff --git a/scripts/hot_shard_warmup.py b/scripts/hot_shard_warmup.py new file mode 100644 index 0000000..8b8d683 --- /dev/null +++ b/scripts/hot_shard_warmup.py @@ -0,0 +1,278 @@ +#!/usr/bin/env python3 +"""Hydrate active shard binaries from warm storage into hot-local storage.""" + +from __future__ import annotations + +import argparse +import json +import os +import time +from concurrent.futures import ThreadPoolExecutor, as_completed +from pathlib import Path +from typing import Any + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--shards-root", + default="data/shards_global/fineweb-global-bpe-v1", + help="Local shards root containing active manifest.json files", + ) + parser.add_argument( + "--warm-shards-root", + default="/mnt/ceph/llm/data/shards_global/fineweb-global-bpe-v1", + help="Warm shards root mirroring local batch directory names", + ) + parser.add_argument( + "--include-offloaded-manifests", + action="store_true", + help="Also inspect manifest.offloaded.json files (default: active manifests only)", + ) + parser.add_argument( + "--splits", + default="train,val", + help="Comma-separated manifest splits to include (default: train,val)", + ) + parser.add_argument( + "--workers", + type=int, + default=4, + help="Parallel copy workers for warm->hot hydration (default: 4)", + ) + parser.add_argument( + "--max-files", + type=int, + default=0, + help="Optional cap on hydration file count (0 = no cap)", + ) + parser.add_argument( + "--allow-missing-warm", + action="store_true", + help="Do not fail the command when warm copies are missing", + ) + parser.add_argument( + "--report-output", + default="", + help="Optional JSON report path", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Preview hydration actions without copying files", + ) + return parser.parse_args() + + +def _load_manifest(path: Path) -> dict[str, Any] | None: + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + return None + if not isinstance(payload, dict): + return None + return payload + + +def _manifest_relpaths(payload: dict[str, Any], *, splits: set[str]) -> list[Path]: + out: list[Path] = [] + for split in sorted(splits): + split_meta = payload.get(split, {}) + if not isinstance(split_meta, dict): + continue + shards = split_meta.get("shards", []) + if not isinstance(shards, list): + continue + for row in shards: + if not isinstance(row, dict): + continue + rel = row.get("path") + if isinstance(rel, str) and rel.strip(): + out.append(Path(rel)) + seen: set[Path] = set() + uniq: list[Path] = [] + for rel in out: + if rel in seen: + continue + seen.add(rel) + uniq.append(rel) + return uniq + + +def _copy_warm_to_hot(*, warm_file: Path, local_file: Path) -> tuple[bool, int, str]: + if not warm_file.exists(): + return (False, 0, "missing_warm_file") + try: + size = int(warm_file.stat().st_size) + except OSError as exc: + return (False, 0, f"warm_stat_error:{exc}") + + try: + local_file.parent.mkdir(parents=True, exist_ok=True) + tmp_copy = local_file.with_name(local_file.name + ".warmup_tmp") + if tmp_copy.exists() or tmp_copy.is_symlink(): + tmp_copy.unlink() + with warm_file.open("rb") as src, tmp_copy.open("wb") as dst: + while True: + chunk = src.read(4 * 1024 * 1024) + if not chunk: + break + dst.write(chunk) + dst.flush() + os.fsync(dst.fileno()) + os.replace(tmp_copy, local_file) + except OSError as exc: + return (False, 0, f"copy_error:{exc}") + return (True, size, "ok") + + +def main() -> int: + args = parse_args() + shards_root = Path(args.shards_root) + warm_root = Path(args.warm_shards_root) + if not shards_root.exists(): + raise FileNotFoundError(f"shards-root not found: {shards_root}") + if not warm_root.exists(): + raise FileNotFoundError(f"warm-shards-root not found: {warm_root}") + + requested_splits = {part.strip() for part in str(args.splits).split(",") if part.strip()} + if not requested_splits: + raise ValueError("at least one split is required in --splits") + + ts = time.strftime("%Y%m%d_%H%M%S") + report_path = ( + Path(args.report_output) + if args.report_output + else Path(f"artifacts/reports/hot_shard_warmup_{ts}.json") + ) + report_path.parent.mkdir(parents=True, exist_ok=True) + + manifests = sorted(p for p in shards_root.rglob("manifest.json") if p.is_file()) + if args.include_offloaded_manifests: + manifests.extend( + sorted(p for p in shards_root.rglob("manifest.offloaded.json") if p.is_file()) + ) + manifest_total = len(manifests) + + parse_errors = 0 + total_refs = 0 + hot_ready = 0 + skipped_cap = 0 + missing_warm = 0 + failed = 0 + hydrated = 0 + hydrated_bytes = 0 + inspected: dict[str, dict[str, str]] = {} + to_hydrate: list[tuple[Path, Path]] = [] + + max_files = max(0, int(args.max_files)) + for manifest_path in manifests: + payload = _load_manifest(manifest_path) + if payload is None: + parse_errors += 1 + continue + batch_name = manifest_path.parent.name + warm_batch = warm_root / batch_name + for rel in _manifest_relpaths(payload, splits=requested_splits): + total_refs += 1 + local_file = manifest_path.parent / rel + key = str(local_file.absolute()) + if key in inspected: + continue + warm_file = warm_batch / rel + local_ok = local_file.exists() and not local_file.is_symlink() + if local_ok: + hot_ready += 1 + inspected[key] = {"status": "hot_ready", "warm": str(warm_file)} + continue + if max_files > 0 and len(to_hydrate) >= max_files: + skipped_cap += 1 + inspected[key] = {"status": "skipped_cap", "warm": str(warm_file)} + continue + inspected[key] = {"status": "queued", "warm": str(warm_file)} + to_hydrate.append((local_file, warm_file)) + + workers = max(1, int(args.workers)) + if args.dry_run: + for local_file, warm_file in to_hydrate: + key = str(local_file.absolute()) + if warm_file.exists(): + inspected[key]["status"] = "would_hydrate" + else: + inspected[key]["status"] = "missing_warm_file" + missing_warm += 1 + else: + with ThreadPoolExecutor(max_workers=workers) as executor: + futures = { + executor.submit(_copy_warm_to_hot, warm_file=warm_file, local_file=local_file): ( + local_file, + warm_file, + ) + for local_file, warm_file in to_hydrate + } + for future in as_completed(futures): + local_file, _warm_file = futures[future] + key = str(local_file.absolute()) + try: + ok, bytes_copied, detail = future.result() + except Exception as exc: # noqa: BLE001 + ok = False + bytes_copied = 0 + detail = f"copy_exception:{exc}" + if ok: + hydrated += 1 + hydrated_bytes += int(bytes_copied) + inspected[key]["status"] = "hydrated" + else: + if detail.startswith("missing_warm"): + missing_warm += 1 + else: + failed += 1 + inspected[key]["status"] = detail + + summary = { + "shards_root": str(shards_root), + "warm_shards_root": str(warm_root), + "dry_run": bool(args.dry_run), + "workers": workers, + "splits": sorted(requested_splits), + "manifest_count": manifest_total, + "manifest_parse_errors": parse_errors, + "shard_refs_total": total_refs, + "unique_shard_files": len(inspected), + "hot_ready_files": hot_ready, + "queued_for_hydration": len(to_hydrate), + "hydrated_files": hydrated, + "hydrated_bytes": hydrated_bytes, + "missing_warm_files": missing_warm, + "failed_files": failed, + "skipped_due_to_max_files_cap": skipped_cap, + "max_files": max_files, + } + report_payload = { + "summary": summary, + "inspected_files": inspected, + } + report_path.write_text(json.dumps(report_payload, indent=2), encoding="utf-8") + + print( + "hot_shard_warmup_done", + f"manifests={manifest_total}", + f"unique_files={summary['unique_shard_files']}", + f"hot_ready={hot_ready}", + f"queued={len(to_hydrate)}", + f"hydrated={hydrated}", + f"missing_warm={missing_warm}", + f"failed={failed}", + f"report={report_path}", + ) + + if failed > 0: + return 1 + if missing_warm > 0 and not args.allow_missing_warm: + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/install_systemd_services.sh b/scripts/install_systemd_services.sh index 24374a2..3d3be5b 100755 --- a/scripts/install_systemd_services.sh +++ b/scripts/install_systemd_services.sh @@ -8,7 +8,6 @@ ENV_TARGET="/etc/llm/llm.env" ENABLE=1 START=1 INSTALL_WATCHDOG=0 -INSTALL_PREFETCH=0 INSTALL_MAINTENANCE=1 usage() { @@ -17,25 +16,24 @@ Usage: bash scripts/install_systemd_services.sh [options] Install and optionally enable/start systemd service units for long-running -LLM pipeline workers (supervisor + stage/shard watchdog, optional prefetch/HF watchdog). +LLM pipeline workers (supervisor + stage/shard watchdog, optional HF watchdog). Also installs maintenance units (checkpoint offload/prune timer + bad-parquet -revalidate timer + shard-offload timer + VM swappiness tune service). +revalidate timer + shard-offload timer + checkpoint step offload timer + VM swappiness tune service). Options: --repo-dir DIR Repository directory baked into unit files --user NAME Service user (default: current user) --systemd-dir DIR Unit install directory (default: /etc/systemd/system) --env-target FILE Environment file path (default: /etc/llm/llm.env) - --install-prefetch Also install/enable prefetch service unit --install-watchdog Also install/enable HF watchdog service unit - --no-maintenance Skip maintenance units (offload/revalidate/shard-offload timers + VM tuning) + --no-maintenance Skip maintenance units (offload/revalidate/shard-offload/step-offload timers + VM tuning) --no-enable Do not run systemctl enable --no-start Do not run systemctl restart/start -h, --help Show help Examples: bash scripts/install_systemd_services.sh - bash scripts/install_systemd_services.sh --install-watchdog --install-prefetch + bash scripts/install_systemd_services.sh --install-watchdog bash scripts/install_systemd_services.sh --no-maintenance USAGE } @@ -62,10 +60,6 @@ while [[ $# -gt 0 ]]; do INSTALL_WATCHDOG=1 shift ;; - --install-prefetch) - INSTALL_PREFETCH=1 - shift - ;; --no-maintenance) INSTALL_MAINTENANCE=0 shift @@ -126,15 +120,14 @@ install_unit() { install_unit "llm-train-supervisor.service" "llm-train-supervisor.service" install_unit "llm-fineweb-stage-shard-loop.service" "llm-fineweb-stage-shard-loop.service" install_unit "llm-fineweb-stage-shard-watchdog.service" "llm-fineweb-stage-shard-watchdog.service" -if [[ "$INSTALL_PREFETCH" -eq 1 ]]; then - install_unit "llm-fineweb-prefetch.service" "llm-fineweb-prefetch.service" -fi if [[ "$INSTALL_WATCHDOG" -eq 1 ]]; then install_unit "llm-hf-download-watchdog.service" "llm-hf-download-watchdog.service" fi if [[ "$INSTALL_MAINTENANCE" -eq 1 ]]; then install_unit "llm-checkpoint-offload-prune.service" "llm-checkpoint-offload-prune.service" install_unit "llm-checkpoint-offload-prune.timer" "llm-checkpoint-offload-prune.timer" + install_unit "llm-checkpoint-step-offload.service" "llm-checkpoint-step-offload.service" + install_unit "llm-checkpoint-step-offload.timer" "llm-checkpoint-step-offload.timer" install_unit "llm-bad-parquet-revalidate.service" "llm-bad-parquet-revalidate.service" install_unit "llm-bad-parquet-revalidate.timer" "llm-bad-parquet-revalidate.timer" install_unit "llm-shard-offload.service" "llm-shard-offload.service" @@ -156,15 +149,12 @@ units=( llm-fineweb-stage-shard-watchdog.service ) timer_units=() -if [[ "$INSTALL_PREFETCH" -eq 1 ]]; then - units+=(llm-fineweb-prefetch.service) -fi if [[ "$INSTALL_WATCHDOG" -eq 1 ]]; then units+=(llm-hf-download-watchdog.service) fi if [[ "$INSTALL_MAINTENANCE" -eq 1 ]]; then units+=(llm-vm-swappiness.service) - timer_units+=(llm-checkpoint-offload-prune.timer llm-bad-parquet-revalidate.timer llm-shard-offload.timer) + timer_units+=(llm-checkpoint-offload-prune.timer llm-checkpoint-step-offload.timer llm-bad-parquet-revalidate.timer llm-shard-offload.timer) fi if [[ "$ENABLE" -eq 1 ]]; then diff --git a/scripts/install_user_systemd_services.sh b/scripts/install_user_systemd_services.sh index 8cf7149..7ba0aee 100755 --- a/scripts/install_user_systemd_services.sh +++ b/scripts/install_user_systemd_services.sh @@ -29,6 +29,8 @@ Installed by default: - llm-train-supervisor.service - llm-fineweb-stage-shard-watchdog.service - llm-shard-offload.service + llm-shard-offload.timer + - llm-checkpoint-offload-prune.service + llm-checkpoint-offload-prune.timer + - llm-checkpoint-step-offload.service + llm-checkpoint-step-offload.timer USAGE } @@ -117,6 +119,10 @@ install_unit "llm-train-supervisor.service" "llm-train-supervisor.service" install_unit "llm-fineweb-stage-shard-watchdog.service" "llm-fineweb-stage-shard-watchdog.service" install_unit "llm-shard-offload.service" "llm-shard-offload.service" install_unit "llm-shard-offload.timer" "llm-shard-offload.timer" +install_unit "llm-checkpoint-offload-prune.service" "llm-checkpoint-offload-prune.service" +install_unit "llm-checkpoint-offload-prune.timer" "llm-checkpoint-offload-prune.timer" +install_unit "llm-checkpoint-step-offload.service" "llm-checkpoint-step-offload.service" +install_unit "llm-checkpoint-step-offload.timer" "llm-checkpoint-step-offload.timer" if [[ "$INSTALL_WATCHDOG" -eq 1 ]]; then install_unit "llm-hf-download-watchdog.service" "llm-hf-download-watchdog.service" fi @@ -136,6 +142,8 @@ units=( ) timer_units=( llm-shard-offload.timer + llm-checkpoint-offload-prune.timer + llm-checkpoint-step-offload.timer ) if [[ "$INSTALL_WATCHDOG" -eq 1 ]]; then units+=(llm-hf-download-watchdog.service) diff --git a/scripts/offload_shard_bins_to_warm.py b/scripts/offload_shard_bins_to_warm.py index 02f5198..dd4e58c 100644 --- a/scripts/offload_shard_bins_to_warm.py +++ b/scripts/offload_shard_bins_to_warm.py @@ -89,6 +89,14 @@ def parse_args() -> argparse.Namespace: "Useful to guarantee a batch was already included in a successful train chunk." ), ) + parser.add_argument( + "--skip-if-trained-file-missing", + action="store_true", + help=( + "When --require-trained-batches-file is set and file is missing, " + "exit 0 with a no-op report instead of raising." + ), + ) parser.add_argument( "--min-active-manifests", type=int, @@ -107,9 +115,31 @@ def parse_args() -> argparse.Namespace: "aggregate train tokens in active manifest.json batches (default: 0)" ), ) + parser.add_argument( + "--min-manifest-unique-input-files", + type=int, + default=0, + help=( + "Require at least this many unique input parquet basenames across " + "manifest.json + manifest.offloaded.json before offloading (default: 0)" + ), + ) return parser.parse_args() +def _resolve_trained_batches_file(raw_value: str) -> Path | None: + value = raw_value.strip() + if not value: + return None + if "," not in value: + return Path(value) + candidates = [Path(part.strip()) for part in value.split(",") if part.strip()] + for candidate in candidates: + if candidate.exists(): + return candidate + return candidates[0] if candidates else None + + def _free_bytes(path: Path) -> int: stats = os.statvfs(path) return int(stats.f_bsize * stats.f_bavail) @@ -120,6 +150,29 @@ def _iter_manifest_dirs(shards_root: Path) -> list[Path]: return [p.parent for p in manifests] +def _iter_coverage_manifests(shards_root: Path) -> list[Path]: + manifests = list(shards_root.rglob("manifest.json")) + manifests.extend(shards_root.rglob("manifest.offloaded.json")) + return sorted(p for p in manifests if p.is_file()) + + +def _manifest_unique_input_files(shards_root: Path) -> int: + unique: set[str] = set() + for manifest_path in _iter_coverage_manifests(shards_root): + try: + payload = json.loads(manifest_path.read_text(encoding="utf-8")) + except Exception: # noqa: BLE001 + continue + raw_files = payload.get("input_files", []) + if not isinstance(raw_files, list): + continue + for raw in raw_files: + name = Path(str(raw)).name + if name: + unique.add(name) + return len(unique) + + def _manifest_shard_relpaths(manifest_path: Path) -> list[Path]: payload = json.loads(manifest_path.read_text(encoding="utf-8")) out: list[Path] = [] @@ -268,6 +321,8 @@ def main() -> int: target_free_bytes = int(args.target_free_gib) * 1024 * 1024 * 1024 min_active_manifests = max(0, int(args.min_active_manifests)) min_active_train_tokens = max(0, int(args.min_active_train_tokens)) + min_manifest_unique_inputs = max(0, int(args.min_manifest_unique_input_files)) + manifest_unique_inputs = _manifest_unique_input_files(shards_root) batch_dirs = _iter_manifest_dirs(shards_root) batch_dirs = sorted(batch_dirs, key=_batch_sort_key, reverse=True) @@ -277,16 +332,65 @@ def main() -> int: if args.max_batches > 0: candidates = candidates[: int(args.max_batches)] + if min_manifest_unique_inputs > 0 and manifest_unique_inputs < min_manifest_unique_inputs: + summary = { + "shards_root": str(shards_root), + "warm_shards_root": str(warm_root), + "dry_run": bool(args.dry_run), + "keep_local_batches": keep_n, + "candidate_batches": len(candidates), + "files_linked": 0, + "bytes_freed_estimate": 0, + "status": "skip_coverage_not_ready", + "manifest_unique_input_files": manifest_unique_inputs, + "min_manifest_unique_input_files": min_manifest_unique_inputs, + "report_output": str(report_path), + "free_bytes_before": free_before, + "free_bytes_after": free_before, + } + payload = {"summary": summary, "results": [], "kept_batches": sorted(keep_set)} + report_path.write_text(json.dumps(payload, indent=2), encoding="utf-8") + print( + "shard_offload_skip", + f"reason=coverage_not_ready have={manifest_unique_inputs} need={min_manifest_unique_inputs}", + f"report={report_path}", + ) + return 0 + + trained_batches_file = _resolve_trained_batches_file(args.require_trained_batches_file) trained_batches: set[str] | None = None if args.require_trained_batches_file: - trained_file = Path(args.require_trained_batches_file) - if not trained_file.exists(): - raise FileNotFoundError( - f"trained batches file not found: {trained_file}" - ) + if trained_batches_file is None: + raise FileNotFoundError("no valid trained batches file candidates provided") + if not trained_batches_file.exists(): + if args.skip_if_trained_file_missing: + summary = { + "shards_root": str(shards_root), + "warm_shards_root": str(warm_root), + "dry_run": bool(args.dry_run), + "keep_local_batches": keep_n, + "candidate_batches": len(candidates), + "files_linked": 0, + "bytes_freed_estimate": 0, + "require_trained_batches_file": args.require_trained_batches_file, + "resolved_trained_batches_file": str(trained_batches_file), + "status": "skip_missing_trained_registry", + "report_output": str(report_path), + "free_bytes_before": free_before, + "free_bytes_after": free_before, + } + payload = {"summary": summary, "results": [], "kept_batches": sorted(keep_set)} + report_path.write_text(json.dumps(payload, indent=2), encoding="utf-8") + print( + "shard_offload_skip", + f"reason=missing_trained_registry file={trained_batches_file}", + f"report={report_path}", + ) + return 0 + raise FileNotFoundError(f"trained batches file not found: {trained_batches_file}") trained_batches = { line.strip() - for line in trained_file.read_text(encoding="utf-8").splitlines() + for line in trained_batches_file.read_text(encoding="utf-8").splitlines() if line.strip() and not line.strip().startswith("#") } @@ -325,7 +429,7 @@ def main() -> int: status="skip_untrained_batch", detail=( "not_in_trained_registry:" - f"{args.require_trained_batches_file}" + f"{trained_batches_file}" ), ) ) @@ -435,7 +539,12 @@ def main() -> int: "files_linked": total_linked, "bytes_freed_estimate": total_freed, "require_trained_batches_file": args.require_trained_batches_file, + "resolved_trained_batches_file": ( + str(trained_batches_file) if trained_batches_file is not None else "" + ), "skip_untrained_batch_count": skipped_untrained, + "manifest_unique_input_files": manifest_unique_inputs, + "min_manifest_unique_input_files": min_manifest_unique_inputs, "min_active_manifests": min_active_manifests, "min_active_train_tokens": min_active_train_tokens, "skip_min_active_manifests_count": skipped_floor_manifests, diff --git a/scripts/parquet_to_corpus.py b/scripts/parquet_to_corpus.py deleted file mode 100755 index f4c385d..0000000 --- a/scripts/parquet_to_corpus.py +++ /dev/null @@ -1,237 +0,0 @@ -#!/usr/bin/env python3 -"""Convert local Parquet datasets into newline text corpora for tokenization/sharding.""" - -from __future__ import annotations - -import argparse -import json -import time -from pathlib import Path -from typing import Any - - -def _normalize_line(text: str) -> str: - return " ".join(text.split()) - - -def _iter_parquet_files(input_dir: Path, pattern: str, max_files: int) -> list[Path]: - files = sorted(p for p in input_dir.rglob(pattern) if p.is_file()) - if max_files > 0: - return files[:max_files] - return files - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--input-dir", required=True, help="Directory containing parquet files") - parser.add_argument("--output-dir", required=True, help="Output directory for .txt corpora") - parser.add_argument("--pattern", default="*.parquet", help="Glob for parquet files") - parser.add_argument("--field", default="text", help="Column name to extract") - parser.add_argument( - "--batch-size", - type=int, - default=8192, - help="Parquet row batch size", - ) - parser.add_argument( - "--min-chars", - type=int, - default=80, - help="Drop extracted rows shorter than this", - ) - parser.add_argument( - "--max-chars", - type=int, - default=0, - help="Truncate rows to this many chars (0 means no truncation)", - ) - parser.add_argument( - "--max-files", - type=int, - default=0, - help="Optional cap on parquet file count (0 means all)", - ) - parser.add_argument( - "--no-skip-existing", - action="store_true", - help="Rebuild output files that already exist", - ) - parser.add_argument( - "--report-output", - default="artifacts/reports/parquet_to_corpus_report.json", - help="JSON report path", - ) - return parser.parse_args() - - -def _process_one_file( - parquet_path: Path, - output_path: Path, - *, - field: str, - batch_size: int, - min_chars: int, - max_chars: int, -) -> dict[str, Any]: - import pyarrow.parquet as pq - - parquet = pq.ParquetFile(parquet_path) - schema_names = set(parquet.schema.names) - if field not in schema_names: - raise ValueError(f"missing field '{field}' in {parquet_path}") - - output_path.parent.mkdir(parents=True, exist_ok=True) - - rows_seen = 0 - rows_written = 0 - rows_filtered_short = 0 - - with output_path.open("w", encoding="utf-8") as dst: - for batch in parquet.iter_batches(columns=[field], batch_size=batch_size): - values = batch.column(0).to_pylist() - for value in values: - rows_seen += 1 - if value is None: - continue - - text = _normalize_line(str(value)) - if len(text) < min_chars: - rows_filtered_short += 1 - continue - - if max_chars > 0: - text = text[:max_chars] - if not text: - continue - - dst.write(text + "\n") - rows_written += 1 - - return { - "input_path": str(parquet_path), - "output_path": str(output_path), - "rows_seen": rows_seen, - "rows_written": rows_written, - "rows_filtered_short": rows_filtered_short, - "status": "ok", - } - - -def main() -> int: - args = parse_args() - - if args.batch_size <= 0: - raise ValueError("batch-size must be > 0") - if args.min_chars < 0: - raise ValueError("min-chars must be >= 0") - if args.max_chars < 0: - raise ValueError("max-chars must be >= 0") - - input_dir = Path(args.input_dir) - output_dir = Path(args.output_dir) - report_path = Path(args.report_output) - - if not input_dir.exists() or not input_dir.is_dir(): - raise FileNotFoundError(f"input-dir not found or not a dir: {input_dir}") - - try: - import pyarrow.parquet as _pq # noqa: F401 - except ImportError as exc: - raise RuntimeError( - "pyarrow is required for parquet conversion. Install training extras or run: " - "python -m pip install pyarrow" - ) from exc - - parquet_files = _iter_parquet_files(input_dir=input_dir, pattern=args.pattern, max_files=args.max_files) - if not parquet_files: - raise ValueError("no parquet files matched") - - started_at = time.time() - output_dir.mkdir(parents=True, exist_ok=True) - - results: list[dict[str, Any]] = [] - totals = { - "files_total": len(parquet_files), - "files_ok": 0, - "files_skipped_existing": 0, - "files_failed": 0, - "rows_seen": 0, - "rows_written": 0, - "rows_filtered_short": 0, - } - - for parquet_path in parquet_files: - rel = parquet_path.relative_to(input_dir) - output_path = (output_dir / rel).with_suffix(".txt") - - if (not args.no_skip_existing) and output_path.exists(): - totals["files_skipped_existing"] += 1 - results.append( - { - "input_path": str(parquet_path), - "output_path": str(output_path), - "status": "skipped_existing", - } - ) - continue - - try: - row = _process_one_file( - parquet_path=parquet_path, - output_path=output_path, - field=args.field, - batch_size=args.batch_size, - min_chars=args.min_chars, - max_chars=args.max_chars, - ) - totals["files_ok"] += 1 - totals["rows_seen"] += int(row["rows_seen"]) - totals["rows_written"] += int(row["rows_written"]) - totals["rows_filtered_short"] += int(row["rows_filtered_short"]) - results.append(row) - except Exception as exc: # pragma: no cover - totals["files_failed"] += 1 - results.append( - { - "input_path": str(parquet_path), - "output_path": str(output_path), - "status": "error", - "error": f"{type(exc).__name__}: {exc}", - } - ) - - elapsed = time.time() - started_at - report: dict[str, Any] = { - "input_dir": str(input_dir), - "output_dir": str(output_dir), - "pattern": args.pattern, - "field": args.field, - "batch_size": args.batch_size, - "min_chars": args.min_chars, - "max_chars": args.max_chars, - "skip_existing": not args.no_skip_existing, - "elapsed_seconds": elapsed, - "totals": totals, - "files": results, - } - - report_path.parent.mkdir(parents=True, exist_ok=True) - report_path.write_text(json.dumps(report, indent=2), encoding="utf-8") - - print(f"input_dir={input_dir}") - print(f"output_dir={output_dir}") - print(f"report={report_path}") - print(f"files_total={totals['files_total']}") - print(f"files_ok={totals['files_ok']}") - print(f"files_skipped_existing={totals['files_skipped_existing']}") - print(f"files_failed={totals['files_failed']}") - print(f"rows_seen={totals['rows_seen']}") - print(f"rows_written={totals['rows_written']}") - print(f"rows_filtered_short={totals['rows_filtered_short']}") - print(f"elapsed_seconds={elapsed:.2f}") - - return 0 if totals["files_failed"] == 0 else 1 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/pipeline_eta_report.py b/scripts/pipeline_eta_report.py index a874a99..0cbedd3 100644 --- a/scripts/pipeline_eta_report.py +++ b/scripts/pipeline_eta_report.py @@ -12,6 +12,11 @@ from pathlib import Path from typing import Any +SUPERVISOR_STATE_CANDIDATES = ( + "artifacts/reports/train_supervisor_phase1_talk", + "artifacts/reports/train_supervisor_350bt", +) + def _run(cmd: list[str]) -> str: out = subprocess.check_output(cmd, text=True) @@ -52,6 +57,44 @@ def _count_nonempty_lines(path: Path) -> int: return len(unique) +def _file_mtime(path: Path) -> float | None: + try: + return path.stat().st_mtime + except OSError: + return None + + +def _dir_latest_mtime(path: Path) -> float: + latest = _file_mtime(path) or 0.0 + for pattern in ( + "supervisor_*.log", + "train_*.log", + "generation_trend.tsv", + "holdout_trend.tsv", + "eval_trend.tsv", + ): + for candidate in path.glob(pattern): + mtime = _file_mtime(candidate) + if mtime is not None and mtime > latest: + latest = mtime + return latest + + +def _resolve_supervisor_state_dir(raw_value: str) -> Path: + requested = raw_value.strip() + if requested: + requested_path = Path(requested) + if requested_path.exists(): + return requested_path + + existing_candidates = [Path(p) for p in SUPERVISOR_STATE_CANDIDATES if Path(p).exists()] + if not existing_candidates: + if requested: + return Path(requested) + return Path(SUPERVISOR_STATE_CANDIDATES[-1]) + return max(existing_candidates, key=_dir_latest_mtime) + + def _pgrep_root_count(pattern: str) -> int: proc = subprocess.run( ["pgrep", "-af", "--", pattern], @@ -257,6 +300,207 @@ def _latest_generation_summary(supervisor_state_dir: Path) -> dict[str, Any]: } +def _parse_float(value: str) -> float | None: + text = value.strip() + if not text: + return None + try: + return float(text) + except ValueError: + return None + + +def _trend_rows(path: Path, min_cols: int) -> list[list[str]]: + if not path.exists(): + return [] + rows: list[list[str]] = [] + try: + with path.open("r", encoding="utf-8", errors="replace") as handle: + for line in handle: + row = line.strip() + if not row or row.startswith("run_tag\t"): + continue + parts = row.split("\t") + if len(parts) >= min_cols: + rows.append(parts) + except OSError: + return [] + return rows + + +def _trend_metric_state( + *, + latest_rc: str, + latest_regression_pass: str | None, + latest_pass: float | None, + latest_check: float | None, + latest_score: float | None, + prev_pass: float | None, + prev_check: float | None, + prev_score: float | None, + pass_eps: float, + check_eps: float, + score_eps: float, +) -> str: + if latest_rc not in {"0", "NA", ""}: + return "regressed" + if latest_regression_pass in {"False", "0"}: + return "regressed" + if latest_pass is None or latest_check is None or latest_score is None: + return "warming" + if prev_pass is None or prev_check is None or prev_score is None: + return "warming" + d_pass = latest_pass - prev_pass + d_check = latest_check - prev_check + d_score = latest_score - prev_score + if d_pass < -pass_eps or d_check < -check_eps or d_score < -score_eps: + return "regressed" + if d_pass > pass_eps or d_check > check_eps or d_score > score_eps: + return "improving" + return "flat" + + +def _quality_heartbeat(supervisor_state_dir: Path) -> dict[str, str]: + eval_rows = _trend_rows(supervisor_state_dir / "eval_trend.tsv", min_cols=8) + gen_rows = _trend_rows(supervisor_state_dir / "generation_trend.tsv", min_cols=9) + holdout_rows = _trend_rows(supervisor_state_dir / "holdout_trend.tsv", min_cols=9) + + eval_state = "unknown" + if eval_rows: + latest = eval_rows[-1] + prev = eval_rows[-2] if len(eval_rows) > 1 else None + eval_regression = latest[8].strip() if len(latest) > 8 else "" + if eval_regression not in {"True", "False", "1", "0"}: + eval_regression = "" + eval_state = _trend_metric_state( + latest_rc=latest[2].strip(), + latest_regression_pass=(eval_regression or None), + latest_pass=_parse_float(latest[3]), + latest_check=_parse_float(latest[4]), + latest_score=_parse_float(latest[5]), + prev_pass=(_parse_float(prev[3]) if prev is not None else None), + prev_check=(_parse_float(prev[4]) if prev is not None else None), + prev_score=(_parse_float(prev[5]) if prev is not None else None), + pass_eps=0.005, + check_eps=0.005, + score_eps=0.002, + ) + + gen_state = "unknown" + if gen_rows: + latest = gen_rows[-1] + prev = gen_rows[-2] if len(gen_rows) > 1 else None + gen_regression = latest[8].strip() if len(latest) > 8 else "" + if gen_regression not in {"True", "False", "1", "0"}: + gen_regression = "" + gen_state = _trend_metric_state( + latest_rc=latest[2].strip(), + latest_regression_pass=(gen_regression or None), + latest_pass=_parse_float(latest[3]), + latest_check=_parse_float(latest[4]), + latest_score=_parse_float(latest[5]), + prev_pass=(_parse_float(prev[3]) if prev is not None else None), + prev_check=(_parse_float(prev[4]) if prev is not None else None), + prev_score=(_parse_float(prev[5]) if prev is not None else None), + pass_eps=0.005, + check_eps=0.005, + score_eps=0.002, + ) + + holdout_state = "unknown" + if holdout_rows: + latest = holdout_rows[-1] + prev = holdout_rows[-2] if len(holdout_rows) > 1 else None + holdout_regression = latest[8].strip() if len(latest) > 8 else "" + if holdout_regression not in {"True", "False", "1", "0"}: + holdout_regression = "" + holdout_state = _trend_metric_state( + latest_rc=latest[2].strip(), + latest_regression_pass=(holdout_regression or None), + latest_pass=_parse_float(latest[3]), + latest_check=_parse_float(latest[4]), + latest_score=_parse_float(latest[5]), + prev_pass=(_parse_float(prev[3]) if prev is not None else None), + prev_check=(_parse_float(prev[4]) if prev is not None else None), + prev_score=(_parse_float(prev[5]) if prev is not None else None), + pass_eps=0.005, + check_eps=0.005, + score_eps=0.002, + ) + + states = {eval_state, gen_state, holdout_state} + if "regressed" in states: + overall = "regressed" + elif "improving" in states: + overall = "improving" + elif states <= {"unknown"}: + overall = "unknown" + elif "warming" in states: + overall = "warming" + else: + overall = "flat" + return { + "overall": overall, + "eval_state": eval_state, + "gen_state": gen_state, + "holdout_state": holdout_state, + } + + +def _confidence_score(level: str) -> float: + if level == "high": + return 1.0 + if level == "medium": + return 0.65 + return 0.30 + + +def _status_confidence( + *, + manifest_overlap_inputs: int, + manifest_unique_inputs: int, + expected_parquet_files: int, + coverage_rate: float | None, + train_rate: float | None, + trainer_active: bool, + quality_overall: str, +) -> dict[str, Any]: + if manifest_unique_inputs <= 0: + coverage_level = "low" + elif manifest_overlap_inputs == 0 and coverage_rate is not None and coverage_rate > 0: + coverage_level = "high" + elif manifest_overlap_inputs <= max(2, expected_parquet_files // 200): + coverage_level = "medium" + else: + coverage_level = "low" + + if train_rate is not None and train_rate > 0: + train_level = "high" + elif trainer_active: + train_level = "medium" + else: + train_level = "low" + + if quality_overall in {"improving", "flat"}: + quality_level = "high" + elif quality_overall in {"warming", "unknown"}: + quality_level = "medium" + else: + quality_level = "low" + + overall = ( + _confidence_score(coverage_level) + + _confidence_score(train_level) + + _confidence_score(quality_level) + ) / 3.0 + return { + "coverage": coverage_level, + "train_eta": train_level, + "quality": quality_level, + "overall_score": round(overall, 4), + } + + def _manifest_input_coverage(shards_root: Path) -> dict[str, int]: if not shards_root.exists(): return { @@ -267,7 +511,9 @@ def _manifest_input_coverage(shards_root: Path) -> dict[str, int]: "manifest_parse_errors": 0, } - manifests = sorted(shards_root.rglob("manifest.json")) + manifests = list(shards_root.rglob("manifest.json")) + manifests.extend(shards_root.rglob("manifest.offloaded.json")) + manifests = sorted(manifests) file_counts: dict[str, int] = {} per_manifest_files: list[set[str]] = [] parse_errors = 0 @@ -386,6 +632,24 @@ def _offload_eligibility( } +def _default_offload_trained_file( + supervisor_state_dir: Path, + configured_path: str, +) -> Path: + if configured_path: + return Path(configured_path) + primary = supervisor_state_dir / "trained_batch_names.txt" + if primary.exists(): + return primary + phase1 = Path("artifacts/reports/train_supervisor_phase1_talk/trained_batch_names.txt") + if phase1.exists(): + return phase1 + standard = Path("artifacts/reports/train_supervisor_350bt/trained_batch_names.txt") + if standard.exists(): + return standard + return primary + + def _latest_supervisor_gate(supervisor_state_dir: Path) -> str: if not supervisor_state_dir.exists(): return "unknown" @@ -442,12 +706,6 @@ def _task_stop_reason( if active.get("hf_watchdog", 0) > 0: return "watchdog-managed; worker idle/restarting" return "not started" - if task_name == "prefetch_worker": - if coverage_complete: - return "coverage complete" - if active.get("stage_loop", 0) > 0: - return "staging handled by stage-loop" - return "not started" if task_name == "stage_watchdog": if coverage_complete: return "coverage complete" @@ -522,7 +780,13 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--shards-root", default="data/shards_global/fineweb-global-bpe-v1") parser.add_argument("--stage-state-dir", default="artifacts/reports/fineweb_stage_shard_loop") parser.add_argument( - "--supervisor-state-dir", default="artifacts/reports/train_supervisor_350bt" + "--supervisor-state-dir", + default="", + help=( + "Supervisor state dir. Default: auto-detect newest existing path from " + "artifacts/reports/train_supervisor_phase1_talk and " + "artifacts/reports/train_supervisor_350bt." + ), ) parser.add_argument("--expected-parquet-files", type=int, default=510) parser.add_argument("--expected-bytes", type=int, default=1061360917731) @@ -575,7 +839,7 @@ def collect_status(args: argparse.Namespace) -> dict[str, Any]: warm_dir = Path(args.warm_dir) shards_root = Path(args.shards_root) stage_state_dir = Path(args.stage_state_dir) - sup_dir = Path(args.supervisor_state_dir) + sup_dir = _resolve_supervisor_state_dir(args.supervisor_state_dir) now = time.time() warm_parquet = _count_find(warm_dir, "*.parquet") @@ -589,10 +853,9 @@ def collect_status(args: argparse.Namespace) -> dict[str, Any]: manifest_overlap_manifests = int(manifest_coverage["overlap_manifests"]) sharded_parquet = _count_nonempty_lines(stage_state_dir / "processed_parquet_files.txt") trained_batch_count = _count_nonempty_lines(sup_dir / "trained_batch_names.txt") - offload_trained_file = ( - Path(args.offload_trained_batches_file) - if args.offload_trained_batches_file - else (sup_dir / "trained_batch_names.txt") + offload_trained_file = _default_offload_trained_file( + sup_dir, + args.offload_trained_batches_file, ) offload_eligibility = _offload_eligibility( shards_root, @@ -604,11 +867,11 @@ def collect_status(args: argparse.Namespace) -> dict[str, Any]: train_target_step = _effective_train_target_step(args.train_target_step, sup_dir, train_step) supervisor_gate = _latest_supervisor_gate(sup_dir) generation_gate_latest = _latest_generation_summary(sup_dir) + quality_heartbeat = _quality_heartbeat(sup_dir) active = { "hf_watchdog": _pgrep_root_count(r"hf_download_watchdog\.sh"), "download_worker": _pgrep_root_count(r"hf_download_resumable\.sh"), - "prefetch_worker": _pgrep_root_count(r"fineweb_prefetch_hot_queue\.sh"), "stage_watchdog": _pgrep_root_count(r"fineweb_stage_shard_watchdog\.sh"), "stage_loop": _pgrep_root_count(r"fineweb_stage_shard_loop\.sh"), "shard_builder": _pgrep_root_count(r"scripts/fineweb_parquet_to_shards\.py"), @@ -616,7 +879,7 @@ def collect_status(args: argparse.Namespace) -> dict[str, Any]: "trainer": _pgrep_root_count(r"llm\.cli train"), "eval_runner": _pgrep_root_count(r"scripts/eval_checkpoint_prompts\.py"), "generation_gate_runner": _pgrep_root_count( - r"scripts/eval_checkpoint_prompts\.py .*generation_smoke_suite_v1\.json" + r"scripts/eval_checkpoint_prompts\.py .*configs/eval/generation_[^ ]+\.json" ), "shard_offload": _pgrep_root_count(r"scripts/offload_shard_bins_to_warm\.py"), } @@ -711,7 +974,6 @@ def collect_status(args: argparse.Namespace) -> dict[str, Any]: task_order = [ "hf_watchdog", "download_worker", - "prefetch_worker", "stage_watchdog", "stage_loop", "shard_builder", @@ -826,6 +1088,16 @@ def collect_status(args: argparse.Namespace) -> dict[str, Any]: "task_status": task_status, "supervisor_gate": supervisor_gate, "generation_gate_latest": generation_gate_latest, + "quality_heartbeat": quality_heartbeat, + "status_confidence": _status_confidence( + manifest_overlap_inputs=manifest_overlap_inputs, + manifest_unique_inputs=manifest_unique_inputs, + expected_parquet_files=int(args.expected_parquet_files), + coverage_rate=coverage_rate, + train_rate=step_rate, + trainer_active=(active.get("trainer", 0) > 0), + quality_overall=str(quality_heartbeat.get("overall", "unknown")), + ), "system_commands": system_commands, } return status @@ -841,12 +1113,13 @@ def write_reports(status: dict[str, Any], output_json: Path, output_text: Path) p = status["active_processes"] c = status["system_commands"] g = status.get("generation_gate_latest", {}) + q = status.get("quality_heartbeat", {}) + confidence = status.get("status_confidence", {}) supervisor_gate = status.get("supervisor_gate", "unknown") task_status = status.get("task_status", {}) task_order = [ "hf_watchdog", "download_worker", - "prefetch_worker", "stage_watchdog", "stage_loop", "shard_builder", @@ -890,7 +1163,7 @@ def write_reports(status: dict[str, Any], output_json: Path, output_text: Path) f" offload_max_offloadable_batches={m['offload_max_offloadable_batches']}", f"trainer_stall_seconds={m['trainer_stall_seconds']}", "active:" - f" hf_watchdog={p['hf_watchdog']} download_worker={p['download_worker']} prefetch_worker={p['prefetch_worker']} stage_watchdog={p['stage_watchdog']} stage_loop={p['stage_loop']}" + f" hf_watchdog={p['hf_watchdog']} download_worker={p['download_worker']} stage_watchdog={p['stage_watchdog']} stage_loop={p['stage_loop']}" f" shard_builder={p['shard_builder']} train_supervisor={p['train_supervisor']}" f" trainer={p['trainer']} eval_runner={p['eval_runner']} generation_gate_runner={p['generation_gate_runner']}" f" shard_offload={p['shard_offload']}", @@ -899,6 +1172,10 @@ def write_reports(status: dict[str, Any], output_json: Path, output_text: Path) *task_lines, "generation_gate_latest:" f" step={g.get('step')} rc={g.get('generation_rc')} pass_rate={g.get('pass_rate')} regression_pass={g.get('regression_pass')}", + "quality_heartbeat:" + f" overall={q.get('overall')} eval_state={q.get('eval_state')} gen_state={q.get('gen_state')} holdout_state={q.get('holdout_state')}", + "status_confidence:" + f" coverage={confidence.get('coverage')} train_eta={confidence.get('train_eta')} quality={confidence.get('quality')} overall_score={confidence.get('overall_score')}", "", "--- top -b -n 1 ---", c["top"]["output"], diff --git a/scripts/pipeline_live_view.py b/scripts/pipeline_live_view.py index 6a2e85b..9cc9dfe 100644 --- a/scripts/pipeline_live_view.py +++ b/scripts/pipeline_live_view.py @@ -43,6 +43,10 @@ class SampleState: WAIT_TRAIN_TOKENS_RE = re.compile(r"waiting_for_train_tokens have_tokens=(\d+) need_tokens=(\d+)") BATCH_START_RE = re.compile(r"^\[([^\]]+)\]\s+batch_start id=(\S+)\s+files=(\d+)\b") BATCH_DONE_RE = re.compile(r"^\[([^\]]+)\]\s+batch_done id=(\S+)\b") +SUPERVISOR_STATE_CANDIDATES = ( + "artifacts/reports/train_supervisor_phase1_talk", + "artifacts/reports/train_supervisor_350bt", +) def parse_args() -> argparse.Namespace: @@ -53,7 +57,13 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--shards-root", default="data/shards_global/fineweb-global-bpe-v1") parser.add_argument("--stage-state-dir", default="artifacts/reports/fineweb_stage_shard_loop") parser.add_argument( - "--supervisor-state-dir", default="artifacts/reports/train_supervisor_350bt" + "--supervisor-state-dir", + default="", + help=( + "Supervisor state dir. Default: auto-detect newest existing path from " + "artifacts/reports/train_supervisor_phase1_talk and " + "artifacts/reports/train_supervisor_350bt." + ), ) parser.add_argument("--expected-parquet-files", type=int, default=510) parser.add_argument("--expected-bytes", type=int, default=1061360917731) @@ -209,7 +219,9 @@ def _latest_manifest_mtime(shards_root: Path) -> float | None: if not shards_root.exists(): return None latest: float | None = None - for manifest_path in shards_root.rglob("manifest.json"): + manifests = list(shards_root.rglob("manifest.json")) + manifests.extend(shards_root.rglob("manifest.offloaded.json")) + for manifest_path in manifests: try: mtime = manifest_path.stat().st_mtime except OSError: @@ -292,6 +304,37 @@ def _file_mtime(path: Path) -> float | None: return None +def _dir_latest_mtime(path: Path) -> float: + latest = _file_mtime(path) or 0.0 + for pattern in ( + "supervisor_*.log", + "train_*.log", + "generation_trend.tsv", + "holdout_trend.tsv", + "eval_trend.tsv", + ): + for candidate in path.glob(pattern): + mtime = _file_mtime(candidate) + if mtime is not None and mtime > latest: + latest = mtime + return latest + + +def _resolve_supervisor_state_dir(raw_value: str) -> Path: + requested = raw_value.strip() + if requested: + requested_path = Path(requested) + if requested_path.exists(): + return requested_path + + existing_candidates = [Path(p) for p in SUPERVISOR_STATE_CANDIDATES if Path(p).exists()] + if not existing_candidates: + if requested: + return Path(requested) + return Path(SUPERVISOR_STATE_CANDIDATES[-1]) + return max(existing_candidates, key=_dir_latest_mtime) + + def _latest_train_step(supervisor_state_dir: Path) -> int: if not supervisor_state_dir.exists(): return 0 @@ -381,6 +424,202 @@ def _latest_generation_summary(supervisor_state_dir: Path) -> tuple[str, str, st return (parts[2], parts[3], parts[8]) +def _parse_float(value: str) -> float | None: + text = value.strip() + if not text: + return None + try: + return float(text) + except ValueError: + return None + + +def _trend_rows(path: Path, min_cols: int) -> list[list[str]]: + if not path.exists(): + return [] + rows: list[list[str]] = [] + try: + with path.open("r", encoding="utf-8", errors="replace") as handle: + for line in handle: + row = line.strip() + if not row or row.startswith("run_tag\t"): + continue + parts = row.split("\t") + if len(parts) >= min_cols: + rows.append(parts) + except OSError: + return [] + return rows + + +def _trend_metric_state( + *, + latest_rc: str, + latest_regression_pass: str | None, + latest_pass: float | None, + latest_check: float | None, + latest_score: float | None, + prev_pass: float | None, + prev_check: float | None, + prev_score: float | None, + pass_eps: float, + check_eps: float, + score_eps: float, +) -> str: + if latest_rc not in {"0", "NA", ""}: + return "regressed" + if latest_regression_pass in {"False", "0"}: + return "regressed" + if latest_pass is None or latest_check is None or latest_score is None: + return "warming" + if prev_pass is None or prev_check is None or prev_score is None: + return "warming" + d_pass = latest_pass - prev_pass + d_check = latest_check - prev_check + d_score = latest_score - prev_score + if d_pass < -pass_eps or d_check < -check_eps or d_score < -score_eps: + return "regressed" + if d_pass > pass_eps or d_check > check_eps or d_score > score_eps: + return "improving" + return "flat" + + +def _quality_heartbeat( + supervisor_state_dir: Path, +) -> tuple[str, str, str, str, str, str, str]: + eval_rows = _trend_rows(supervisor_state_dir / "eval_trend.tsv", min_cols=8) + gen_rows = _trend_rows(supervisor_state_dir / "generation_trend.tsv", min_cols=9) + holdout_rows = _trend_rows(supervisor_state_dir / "holdout_trend.tsv", min_cols=9) + + eval_state = "unknown" + eval_note = "no eval trend data" + if eval_rows: + latest = eval_rows[-1] + prev = eval_rows[-2] if len(eval_rows) > 1 else None + latest_rc = latest[2].strip() + latest_pass = _parse_float(latest[3]) + latest_check = _parse_float(latest[4]) + latest_score = _parse_float(latest[5]) + prev_pass = _parse_float(prev[3]) if prev is not None else None + prev_check = _parse_float(prev[4]) if prev is not None else None + prev_score = _parse_float(prev[5]) if prev is not None else None + regression_pass = latest[8].strip() if len(latest) > 8 else "NA" + if regression_pass not in {"True", "False", "1", "0"}: + regression_pass = "NA" + promotion_pass = latest[9].strip() if len(latest) > 9 else "NA" + if promotion_pass not in {"True", "False", "1", "0"}: + promotion_pass = "NA" + + eval_state = _trend_metric_state( + latest_rc=latest_rc, + latest_regression_pass=(None if regression_pass == "NA" else regression_pass), + latest_pass=latest_pass, + latest_check=latest_check, + latest_score=latest_score, + prev_pass=prev_pass, + prev_check=prev_check, + prev_score=prev_score, + pass_eps=0.005, + check_eps=0.005, + score_eps=0.002, + ) + eval_note = ( + f"eval={eval_state} rc={latest_rc} " + f"pass={latest_pass:.3f}" if latest_pass is not None else f"eval={eval_state} rc={latest_rc}" + ) + if latest_check is not None: + eval_note += f" check={latest_check:.3f}" + if latest_score is not None: + eval_note += f" score={latest_score:.3f}" + if regression_pass != "NA": + eval_note += f" regression_pass={regression_pass}" + if promotion_pass != "NA": + eval_note += f" promo_pass={promotion_pass}" + + gen_state = "unknown" + gen_note = "gen=no generation trend data" + if gen_rows: + latest = gen_rows[-1] + prev = gen_rows[-2] if len(gen_rows) > 1 else None + latest_rc = latest[2].strip() + latest_pass = _parse_float(latest[3]) + latest_check = _parse_float(latest[4]) + latest_score = _parse_float(latest[5]) + latest_regression_pass = latest[8].strip() + prev_pass = _parse_float(prev[3]) if prev is not None else None + prev_check = _parse_float(prev[4]) if prev is not None else None + prev_score = _parse_float(prev[5]) if prev is not None else None + + gen_state = _trend_metric_state( + latest_rc=latest_rc, + latest_regression_pass=latest_regression_pass, + latest_pass=latest_pass, + latest_check=latest_check, + latest_score=latest_score, + prev_pass=prev_pass, + prev_check=prev_check, + prev_score=prev_score, + pass_eps=0.005, + check_eps=0.005, + score_eps=0.002, + ) + gen_note = ( + f"gen={gen_state} rc={latest_rc} " + f"pass={latest_pass:.3f}" if latest_pass is not None else f"gen={gen_state} rc={latest_rc}" + ) + if latest_regression_pass: + gen_note += f" regression_pass={latest_regression_pass}" + + holdout_state = "unknown" + holdout_note = "holdout=no holdout trend data" + if holdout_rows: + latest = holdout_rows[-1] + prev = holdout_rows[-2] if len(holdout_rows) > 1 else None + latest_rc = latest[2].strip() + latest_pass = _parse_float(latest[3]) + latest_check = _parse_float(latest[4]) + latest_score = _parse_float(latest[5]) + latest_regression_pass = latest[8].strip() + prev_pass = _parse_float(prev[3]) if prev is not None else None + prev_check = _parse_float(prev[4]) if prev is not None else None + prev_score = _parse_float(prev[5]) if prev is not None else None + + holdout_state = _trend_metric_state( + latest_rc=latest_rc, + latest_regression_pass=latest_regression_pass, + latest_pass=latest_pass, + latest_check=latest_check, + latest_score=latest_score, + prev_pass=prev_pass, + prev_check=prev_check, + prev_score=prev_score, + pass_eps=0.005, + check_eps=0.005, + score_eps=0.002, + ) + holdout_note = ( + f"holdout={holdout_state} rc={latest_rc} pass={latest_pass:.3f}" + if latest_pass is not None + else f"holdout={holdout_state} rc={latest_rc}" + ) + if latest_regression_pass: + holdout_note += f" regression_pass={latest_regression_pass}" + + states = {eval_state, gen_state, holdout_state} + if "regressed" in states: + overall = "regressed" + elif "improving" in states: + overall = "improving" + elif states <= {"unknown"}: + overall = "unknown" + elif "warming" in states: + overall = "warming" + else: + overall = "flat" + + return (overall, eval_state, gen_state, holdout_state, eval_note, gen_note, holdout_note) + + def _eta_status_train_rate(status_path: Path, max_age_seconds: int, now_ts: float) -> float | None: if max_age_seconds <= 0 or not status_path.exists(): return None @@ -443,7 +682,9 @@ def _latest_supervisor_gate(supervisor_state_dir: Path) -> str: def _manifest_input_coverage(shards_root: Path) -> tuple[int, int, int]: if not shards_root.exists(): return (0, 0, 0) - manifests = sorted(shards_root.rglob("manifest.json")) + manifests = list(shards_root.rglob("manifest.json")) + manifests.extend(shards_root.rglob("manifest.offloaded.json")) + manifests = sorted(manifests) file_counts: dict[str, int] = {} per_manifest: list[set[str]] = [] for manifest_path in manifests: @@ -534,6 +775,24 @@ def _offload_eligibility( return (effective_eligible, raw_eligible, max_offloadable, trained_registry_present) +def _default_offload_trained_file( + supervisor_state_dir: Path, + configured_path: str, +) -> Path: + if configured_path: + return Path(configured_path) + primary = supervisor_state_dir / "trained_batch_names.txt" + if primary.exists(): + return primary + phase1 = Path("artifacts/reports/train_supervisor_phase1_talk/trained_batch_names.txt") + if phase1.exists(): + return phase1 + standard = Path("artifacts/reports/train_supervisor_350bt/trained_batch_names.txt") + if standard.exists(): + return standard + return primary + + def _cpu_snapshot() -> tuple[int, int]: with open("/proc/stat", "r", encoding="utf-8") as handle: line = handle.readline().strip() @@ -693,6 +952,14 @@ def _task_status(pattern: str) -> tuple[int, list[str]]: return len(root_pids), rows +def _stage_loop_uses_stage_copy() -> bool: + rc, text = _run_capture(["pgrep", "-af", r"fineweb_stage_shard_loop\.sh"], timeout=5) + if rc != 0 or not text: + return True + # New direct Ceph mode is default unless --enable-stage-copy is present. + return any("--enable-stage-copy" in line for line in text.splitlines()) + + def _stop_reason( task_name: str, *, @@ -707,6 +974,7 @@ def _stop_reason( task_counts: dict[str, int], offload_eligible_batches: int, trained_registry_present: bool, + stage_copy_enabled: bool, ) -> str: if task_name == "hf-watchdog": if warm_parquet >= expected_parquet_files: @@ -724,12 +992,6 @@ def _stop_reason( if task_counts.get("download-worker", 0) > 0 or task_counts.get("hf-watchdog", 0) > 0: return "managed by resumable worker/watchdog" return "not started" - if task_name == "prefetch-worker": - if coverage_complete: - return "coverage complete" - if task_counts.get("stage-loop", 0) > 0: - return "staging handled by stage-loop" - return "not started" if task_name == "stage-watchdog": if coverage_complete: return "coverage complete" @@ -746,10 +1008,12 @@ def _stop_reason( if coverage_complete: return "all expected parquet processed" if task_counts.get("stage-loop", 0) > 0: - if hot_parquet <= 0: + if stage_copy_enabled and hot_parquet <= 0: if hot_incomplete > 0: return "finalizing staged parquet copies" return "waiting for staged hot parquet" + if not stage_copy_enabled: + return "idle between direct-source shard batches" return "idle between shard batches" return "not started" if task_name == "shard-verify": @@ -823,6 +1087,55 @@ def _eta(remaining: float | None, rate_per_sec: float | None) -> str: return f"{mins}m" +def _confidence_score(level: str) -> float: + if level == "high": + return 1.0 + if level == "medium": + return 0.65 + return 0.30 + + +def _status_confidence( + *, + manifest_overlap_inputs: int, + manifest_unique_inputs: int, + expected_parquet_files: int, + coverage_pps: float | None, + train_sps: float | None, + trainer_running: bool, + quality_state: str, +) -> tuple[str, str, str, float]: + if manifest_unique_inputs <= 0: + coverage_level = "low" + elif manifest_overlap_inputs == 0 and coverage_pps is not None and coverage_pps > 0: + coverage_level = "high" + elif manifest_overlap_inputs <= max(2, expected_parquet_files // 200): + coverage_level = "medium" + else: + coverage_level = "low" + + if train_sps is not None and train_sps > 0: + train_level = "high" + elif trainer_running: + train_level = "medium" + else: + train_level = "low" + + if quality_state in {"improving", "flat"}: + quality_level = "high" + elif quality_state in {"warming", "unknown"}: + quality_level = "medium" + else: + quality_level = "low" + + overall = ( + _confidence_score(coverage_level) + + _confidence_score(train_level) + + _confidence_score(quality_level) + ) / 3.0 + return (coverage_level, train_level, quality_level, overall) + + def _render( args: argparse.Namespace, state: SampleState, @@ -837,7 +1150,7 @@ def _render( shards_root = Path(args.shards_root) stage_state_dir = Path(args.stage_state_dir) stage_state = stage_state_dir / "processed_parquet_files.txt" - sup_dir = Path(args.supervisor_state_dir) + sup_dir = _resolve_supervisor_state_dir(args.supervisor_state_dir) warm_parquet = _count_find(warm_dir, "*.parquet") warm_incomplete = _count_find(warm_dir, "*.incomplete") @@ -854,11 +1167,7 @@ def _render( ) processed_parquet = _count_nonempty_lines(stage_state) trained_batch_count = _count_nonempty_lines(sup_dir / "trained_batch_names.txt") - offload_trained_file = ( - Path(args.offload_trained_batches_file) - if args.offload_trained_batches_file - else (sup_dir / "trained_batch_names.txt") - ) + offload_trained_file = _default_offload_trained_file(sup_dir, args.offload_trained_batches_file) ( offload_eligible_batches, offload_eligible_raw, @@ -874,6 +1183,15 @@ def _render( train_target_step = _effective_train_target_step(args.train_target_step, sup_dir, train_step) supervisor_gate = _latest_supervisor_gate(sup_dir) gen_rc, gen_pass_rate, gen_regression_pass = _latest_generation_summary(sup_dir) + ( + quality_state, + eval_quality_state, + gen_quality_state, + holdout_quality_state, + eval_quality_note, + gen_quality_note, + holdout_quality_note, + ) = _quality_heartbeat(sup_dir) dt = (now - state.ts) if state.ts is not None else None download_bps = _rate(warm_bytes, state.warm_bytes, dt) @@ -974,7 +1292,6 @@ def _render( tasks = [ ("hf-watchdog", r"hf_download_watchdog\.sh"), ("download-worker", r"hf_download_resumable\.sh"), - ("prefetch-worker", r"fineweb_prefetch_hot_queue\.sh"), ("stage-watchdog", r"fineweb_stage_shard_watchdog\.sh"), ("hf-download", r"\.venv/bin/hf download HuggingFaceFW/fineweb"), ("stage-loop", r"fineweb_stage_shard_loop\.sh"), @@ -983,7 +1300,7 @@ def _render( ("train-supervisor", r"bash scripts/train_supervisor_rtx5070_350bt\.sh"), ("trainer", r"llm\.cli train"), ("eval-runner", r"eval_checkpoint_prompts\.py"), - ("generation-gate", r"eval_checkpoint_prompts\.py .*generation_smoke_suite_v1\.json"), + ("generation-gate", r"eval_checkpoint_prompts\.py .*configs/eval/generation_[^ ]+\.json"), ("shard-offload", r"scripts/offload_shard_bins_to_warm\.py"), ("zim-offload", r"zim_offload_worker\.sh"), ] @@ -997,6 +1314,7 @@ def _render( task_lines: list[str] = [] task_counts: dict[str, int] = {} task_rows: dict[str, list[str]] = {} + stage_copy_enabled = _stage_loop_uses_stage_copy() for name, pattern in tasks: count, rows = _task_status(pattern) task_counts[name] = count @@ -1017,6 +1335,7 @@ def _render( task_counts=task_counts, offload_eligible_batches=offload_eligible_batches, trained_registry_present=offload_registry_present, + stage_copy_enabled=stage_copy_enabled, ) task_lines.append(f"{name:16} STOP | {reason}") continue @@ -1024,6 +1343,21 @@ def _render( summary = rows[0] if rows else "running" task_lines.append(f"{name:16} RUN x{count} | {summary}") + ( + coverage_confidence, + train_confidence, + quality_confidence, + overall_confidence, + ) = _status_confidence( + manifest_overlap_inputs=manifest_overlap_inputs, + manifest_unique_inputs=manifest_unique_inputs, + expected_parquet_files=int(args.expected_parquet_files), + coverage_pps=coverage_pps, + train_sps=train_sps, + trainer_running=(task_counts.get("trainer", 0) > 0), + quality_state=quality_state, + ) + rem_bytes = max(0, int(args.expected_bytes) - warm_bytes) rem_files = max(0, int(args.expected_parquet_files) - warm_parquet) rem_manifest_unique = max(0, int(args.expected_parquet_files) - manifest_unique_inputs) @@ -1068,6 +1402,12 @@ def _render( f"trainer step stalled for {trainer_stall_seconds}s " f"(threshold={args.trainer_stall_alert_seconds}s)" ) + if quality_state == "regressed": + alerts.append("quality heartbeat regressed; inspect eval/generation trend deltas") + if overall_confidence < 0.55: + alerts.append( + "status confidence is low; verify coverage overlap, train step rate, and quality gates" + ) if active_symlink_manifests > 0: alerts.append( f"active manifests include symlink bins ({active_symlink_manifests}); " @@ -1170,6 +1510,16 @@ def _render( f" GenGate: latest_rc={gen_rc} latest_pass_rate={gen_pass_rate} " f"latest_regression_pass={gen_regression_pass}" ) + lines.append( + f" Quality: heartbeat={quality_state} eval={eval_quality_state} gen={gen_quality_state} holdout={holdout_quality_state}" + ) + lines.append( + f" QualityDetail: {eval_quality_note} | {gen_quality_note} | {holdout_quality_note}" + ) + lines.append( + f" Confidence: coverage={coverage_confidence} train_eta={train_confidence} " + f"quality={quality_confidence} overall={overall_confidence:.2f}" + ) lines.append("") lines.append("Alerts") diff --git a/scripts/reconcile_offloaded_manifests.py b/scripts/reconcile_offloaded_manifests.py new file mode 100755 index 0000000..8f35388 --- /dev/null +++ b/scripts/reconcile_offloaded_manifests.py @@ -0,0 +1,435 @@ +#!/usr/bin/env python3 +"""Reconcile offloaded manifests against training registry and coverage targets. + +This script can restore `manifest.offloaded.json` back to `manifest.json` when: +- the batch is not present in the trained-batches registry, and/or +- active manifest unique-input coverage is below a configured threshold. +""" + +from __future__ import annotations + +import argparse +import json +import os +import time +from pathlib import Path +from typing import Any + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--shards-root", + default="data/shards_global/fineweb-global-bpe-v1", + help="Root containing shard batch directories", + ) + parser.add_argument( + "--trained-batches-file", + default="artifacts/reports/train_supervisor_phase1_talk/trained_batch_names.txt,artifacts/reports/train_supervisor_350bt/trained_batch_names.txt", + help=( + "Path or comma-separated fallback list of trained-batch registry files " + "(one batch name per line)" + ), + ) + parser.add_argument( + "--skip-if-trained-file-missing", + action="store_true", + help="Proceed without trained-registry gating if no trained-batches file exists", + ) + parser.add_argument( + "--manifest-disabled-suffix", + default=".offloaded.json", + help="Suffix used for offloaded manifests (default: .offloaded.json)", + ) + parser.add_argument( + "--min-active-unique-input-files", + type=int, + default=0, + help=( + "Restore offloaded manifests until active manifest unique input files " + "reach this threshold (default: 0 disables)" + ), + ) + parser.add_argument( + "--max-restore", + type=int, + default=0, + help="Maximum manifests to restore this run (0 = unlimited)", + ) + parser.add_argument( + "--warm-shards-root", + default="/mnt/ceph/llm/data/shards_global/fineweb-global-bpe-v1", + help="Warm shards root mirroring local batch directories", + ) + parser.add_argument( + "--rehydrate-restored-bins", + action="store_true", + help="Copy shard bins back to local hot storage for manifests restored this run", + ) + parser.add_argument( + "--rehydrate-active-symlink-bins", + action="store_true", + help="Copy shard bins back to local hot storage for all active manifests with symlink bins", + ) + parser.add_argument( + "--report-output", + default="", + help="Optional JSON report output path", + ) + parser.add_argument("--dry-run", action="store_true", help="Preview changes only") + return parser.parse_args() + + +def _resolve_fallback_path(raw: str) -> Path | None: + value = raw.strip() + if not value: + return None + if "," not in value: + return Path(value) + candidates = [Path(part.strip()) for part in value.split(",") if part.strip()] + for candidate in candidates: + if candidate.exists(): + return candidate + return candidates[0] if candidates else None + + +def _load_manifest(path: Path) -> dict[str, Any] | None: + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + return None + if not isinstance(payload, dict): + return None + return payload + + +def _manifest_input_files(payload: dict[str, Any]) -> set[str]: + out: set[str] = set() + raw_files = payload.get("input_files", []) + if not isinstance(raw_files, list): + return out + for raw in raw_files: + name = Path(str(raw)).name + if name: + out.add(name) + return out + + +def _manifest_shard_relpaths(payload: dict[str, Any]) -> list[Path]: + out: list[Path] = [] + for split in ("train", "val"): + split_meta = payload.get(split, {}) + if not isinstance(split_meta, dict): + continue + shards = split_meta.get("shards", []) + if not isinstance(shards, list): + continue + for row in shards: + if not isinstance(row, dict): + continue + rel = row.get("path") + if isinstance(rel, str) and rel.strip(): + out.append(Path(rel)) + # preserve order / dedupe + seen: set[Path] = set() + uniq: list[Path] = [] + for rel in out: + if rel in seen: + continue + seen.add(rel) + uniq.append(rel) + return uniq + + +def _active_manifest_paths(shards_root: Path) -> list[Path]: + return sorted(p for p in shards_root.rglob("manifest.json") if p.is_file()) + + +def _offloaded_manifest_paths(shards_root: Path, suffix: str) -> list[Path]: + filename = f"manifest{suffix}" + return sorted( + (p for p in shards_root.rglob(filename) if p.is_file()), + key=lambda p: p.stat().st_mtime, + reverse=True, + ) + + +def _rehydrate_manifest_bins( + *, + manifest_path: Path, + payload: dict[str, Any], + warm_root: Path, + dry_run: bool, +) -> tuple[int, int, int]: + batch_name = manifest_path.parent.name + warm_batch = warm_root / batch_name + if not warm_batch.exists(): + return (0, 0, 0) + copied = 0 + bytes_copied = 0 + missing_warm = 0 + for rel in _manifest_shard_relpaths(payload): + local_file = manifest_path.parent / rel + warm_file = warm_batch / rel + local_is_symlink = local_file.is_symlink() + needs_copy = local_is_symlink or not local_file.exists() + if not needs_copy: + continue + if not warm_file.exists(): + missing_warm += 1 + continue + try: + warm_size = warm_file.stat().st_size + except OSError: + missing_warm += 1 + continue + copied += 1 + bytes_copied += int(warm_size) + if dry_run: + continue + local_file.parent.mkdir(parents=True, exist_ok=True) + tmp_copy = local_file.with_name(local_file.name + ".rehydrate_tmp") + if tmp_copy.exists() or tmp_copy.is_symlink(): + tmp_copy.unlink() + with warm_file.open("rb") as src, tmp_copy.open("wb") as dst: + while True: + chunk = src.read(4 * 1024 * 1024) + if not chunk: + break + dst.write(chunk) + dst.flush() + os.fsync(dst.fileno()) + os.replace(tmp_copy, local_file) + return (copied, bytes_copied, missing_warm) + + +def _load_trained_batches(path: Path) -> set[str]: + values: set[str] = set() + for raw in path.read_text(encoding="utf-8", errors="replace").splitlines(): + value = raw.strip() + if value and not value.startswith("#"): + values.add(value) + return values + + +def main() -> int: + args = parse_args() + shards_root = Path(args.shards_root) + warm_root = Path(args.warm_shards_root) + if not shards_root.exists(): + raise FileNotFoundError(f"shards-root not found: {shards_root}") + + ts = time.strftime("%Y%m%d_%H%M%S") + report_path = ( + Path(args.report_output) + if args.report_output + else Path(f"artifacts/reports/offload_reconcile_{ts}.json") + ) + report_path.parent.mkdir(parents=True, exist_ok=True) + + trained_file = _resolve_fallback_path(args.trained_batches_file) + trained_batches: set[str] | None = None + trained_registry_present = False + if args.trained_batches_file: + if trained_file is None: + if not args.skip_if_trained_file_missing: + raise FileNotFoundError("no valid trained-batches file candidates provided") + elif trained_file.exists(): + trained_batches = _load_trained_batches(trained_file) + trained_registry_present = True + elif not args.skip_if_trained_file_missing: + raise FileNotFoundError(f"trained batches file not found: {trained_file}") + + active_paths = _active_manifest_paths(shards_root) + active_payloads: dict[Path, dict[str, Any]] = {} + active_unique_inputs: set[str] = set() + active_parse_errors = 0 + for path in active_paths: + payload = _load_manifest(path) + if payload is None: + active_parse_errors += 1 + continue + active_payloads[path] = payload + active_unique_inputs.update(_manifest_input_files(payload)) + + offloaded_paths = _offloaded_manifest_paths(shards_root, args.manifest_disabled_suffix) + restored = 0 + restored_untrained = 0 + restored_coverage = 0 + skipped_active_exists = 0 + skipped_parse_error = 0 + skipped_trained = 0 + rehydrated_files = 0 + rehydrated_bytes = 0 + rehydrate_missing_warm_files = 0 + changes: list[dict[str, Any]] = [] + + target_active_unique = max(0, int(args.min_active_unique_input_files)) + max_restore = max(0, int(args.max_restore)) + + for offloaded_path in offloaded_paths: + if max_restore > 0 and restored >= max_restore: + break + active_path = offloaded_path.with_name("manifest.json") + if active_path.exists(): + skipped_active_exists += 1 + changes.append( + { + "batch": offloaded_path.parent.name, + "action": "skip_active_exists", + "offloaded_manifest": str(offloaded_path), + "active_manifest": str(active_path), + } + ) + continue + + payload = _load_manifest(offloaded_path) + if payload is None: + skipped_parse_error += 1 + changes.append( + { + "batch": offloaded_path.parent.name, + "action": "skip_parse_error", + "offloaded_manifest": str(offloaded_path), + } + ) + continue + + batch_name = offloaded_path.parent.name + inputs = _manifest_input_files(payload) + + restore_reason = "" + if trained_batches is not None and batch_name not in trained_batches: + restore_reason = "untrained_batch" + elif target_active_unique > 0 and len(active_unique_inputs) < target_active_unique: + projected = active_unique_inputs | inputs + if len(projected) > len(active_unique_inputs): + restore_reason = "active_coverage_floor" + else: + skipped_trained += 1 + changes.append( + { + "batch": batch_name, + "action": "skip_trained_or_not_needed", + "offloaded_manifest": str(offloaded_path), + } + ) + continue + + changes.append( + { + "batch": batch_name, + "action": "restore_manifest", + "reason": restore_reason, + "offloaded_manifest": str(offloaded_path), + "active_manifest": str(active_path), + } + ) + if not args.dry_run: + offloaded_path.rename(active_path) + + restored += 1 + if restore_reason == "untrained_batch": + restored_untrained += 1 + elif restore_reason == "active_coverage_floor": + restored_coverage += 1 + active_unique_inputs.update(inputs) + + if args.rehydrate_restored_bins: + copied, copied_bytes, missing_warm = _rehydrate_manifest_bins( + manifest_path=active_path, + payload=payload, + warm_root=warm_root, + dry_run=args.dry_run, + ) + rehydrated_files += copied + rehydrated_bytes += copied_bytes + rehydrate_missing_warm_files += missing_warm + if copied > 0 or missing_warm > 0: + changes.append( + { + "batch": batch_name, + "action": "rehydrate_restored_bins", + "copied_files": copied, + "copied_bytes": copied_bytes, + "missing_warm_files": missing_warm, + } + ) + + if args.rehydrate_active_symlink_bins: + for manifest_path in _active_manifest_paths(shards_root): + payload = _load_manifest(manifest_path) + if payload is None: + continue + copied, copied_bytes, missing_warm = _rehydrate_manifest_bins( + manifest_path=manifest_path, + payload=payload, + warm_root=warm_root, + dry_run=args.dry_run, + ) + rehydrated_files += copied + rehydrated_bytes += copied_bytes + rehydrate_missing_warm_files += missing_warm + if copied > 0 or missing_warm > 0: + changes.append( + { + "batch": manifest_path.parent.name, + "action": "rehydrate_active_symlink_bins", + "copied_files": copied, + "copied_bytes": copied_bytes, + "missing_warm_files": missing_warm, + } + ) + + active_after = len(_active_manifest_paths(shards_root)) + offloaded_after = len(_offloaded_manifest_paths(shards_root, args.manifest_disabled_suffix)) + + summary = { + "shards_root": str(shards_root), + "dry_run": bool(args.dry_run), + "trained_batches_file_arg": args.trained_batches_file, + "trained_batches_file_resolved": str(trained_file) if trained_file is not None else "", + "trained_registry_present": trained_registry_present, + "trained_batch_count": len(trained_batches or set()), + "warm_shards_root": str(warm_root), + "active_manifest_count_before": len(active_paths), + "offloaded_manifest_count_before": len(offloaded_paths), + "active_manifest_parse_errors": active_parse_errors, + "active_unique_input_files_after": len(active_unique_inputs), + "min_active_unique_input_files": target_active_unique, + "restored_manifest_count": restored, + "restored_untrained_count": restored_untrained, + "restored_coverage_floor_count": restored_coverage, + "skipped_active_exists_count": skipped_active_exists, + "skipped_parse_error_count": skipped_parse_error, + "skipped_trained_or_not_needed_count": skipped_trained, + "rehydrate_restored_bins": bool(args.rehydrate_restored_bins), + "rehydrate_active_symlink_bins": bool(args.rehydrate_active_symlink_bins), + "rehydrated_files": rehydrated_files, + "rehydrated_bytes": rehydrated_bytes, + "rehydrate_missing_warm_files": rehydrate_missing_warm_files, + "active_manifest_count_after": active_after, + "offloaded_manifest_count_after": offloaded_after, + "max_restore": max_restore, + "report_output": str(report_path), + } + + payload = {"summary": summary, "changes": changes} + report_path.write_text(json.dumps(payload, indent=2), encoding="utf-8") + print( + "offload_reconcile_done", + f"restored={restored}", + f"restored_untrained={restored_untrained}", + f"restored_coverage_floor={restored_coverage}", + f"active_after={active_after}", + f"offloaded_after={offloaded_after}", + f"active_unique_after={len(active_unique_inputs)}", + f"rehydrated_files={rehydrated_files}", + f"rehydrated_bytes={rehydrated_bytes}", + f"report={report_path}", + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/revalidate_bad_parquet.py b/scripts/revalidate_bad_parquet.py index 31bd120..561c7dd 100644 --- a/scripts/revalidate_bad_parquet.py +++ b/scripts/revalidate_bad_parquet.py @@ -4,6 +4,7 @@ from __future__ import annotations import argparse +from concurrent.futures import ThreadPoolExecutor, as_completed import json import shutil import subprocess @@ -31,6 +32,24 @@ def parse_args() -> argparse.Namespace: default="/mnt/ceph/llm/data/fineweb/sample-350BT/sample/350BT", help="Warm parquet directory", ) + parser.add_argument( + "--max-entries", + type=int, + default=0, + help="Max bad-list entries to process per run (0 = all)", + ) + parser.add_argument( + "--workers", + type=int, + default=4, + help="Parallel validation workers (default: 4)", + ) + parser.add_argument( + "--validate-retries", + type=int, + default=1, + help="Retries for parquet read_error validation failures (default: 1)", + ) parser.add_argument("--field", default="text", help="Expected parquet text field") parser.add_argument( "--report-output", @@ -231,6 +250,32 @@ def _prune_quarantine( return summary +def _validate_name( + name: str, + *, + warm_dir: Path, + field: str, + validate_retries: int, +) -> ValidationResult: + warm_path = warm_dir / name + if not warm_path.exists(): + return ValidationResult(name=name, status="missing_warm") + + retries = max(0, int(validate_retries)) + last_detail = "validation_failed" + for attempt in range(retries + 1): + ok, detail = _validate_parquet(warm_path, field) + if ok: + return ValidationResult(name=name, status="valid", detail=detail) + last_detail = detail + if not detail.startswith("read_error:"): + break + if attempt < retries: + time.sleep(0.2 * float(attempt + 1)) + + return ValidationResult(name=name, status="invalid", detail=last_detail) + + def main() -> int: args = parse_args() bad_list = Path(args.bad_list) @@ -254,27 +299,65 @@ def main() -> int: report_output.parent.mkdir(parents=True, exist_ok=True) bad_names = _read_bad_list(bad_list) + limit = int(args.max_entries) + processed_names = bad_names[: limit if limit > 0 else len(bad_names)] + untouched_names = bad_names[len(processed_names) :] results: list[ValidationResult] = [] valid_names: list[str] = [] retained_bad_names: list[str] = [] - for name in bad_names: - warm_path = warm_dir / name - if not warm_path.exists(): - results.append(ValidationResult(name=name, status="missing_warm")) - retained_bad_names.append(name) - continue - ok, detail = _validate_parquet(warm_path, args.field) - if ok: - results.append(ValidationResult(name=name, status="valid", detail=detail)) - valid_names.append(name) - else: - results.append(ValidationResult(name=name, status="invalid", detail=detail)) - retained_bad_names.append(name) + workers = max(1, int(args.workers)) + retries = max(0, int(args.validate_retries)) + if workers == 1: + for name in processed_names: + result = _validate_name( + name, + warm_dir=warm_dir, + field=args.field, + validate_retries=retries, + ) + results.append(result) + if result.status == "valid": + valid_names.append(name) + else: + retained_bad_names.append(name) + else: + by_name: dict[str, ValidationResult] = {} + with ThreadPoolExecutor(max_workers=workers) as executor: + futures = { + executor.submit( + _validate_name, + name, + warm_dir=warm_dir, + field=args.field, + validate_retries=retries, + ): name + for name in processed_names + } + for future in as_completed(futures): + name = futures[future] + try: + by_name[name] = future.result() + except Exception as exc: # noqa: BLE001 + by_name[name] = ValidationResult( + name=name, + status="invalid", + detail=f"worker_error:{exc}", + ) + + for name in processed_names: + result = by_name[name] + results.append(result) + if result.status == "valid": + valid_names.append(name) + else: + retained_bad_names.append(name) + + retained_with_untouched = retained_bad_names + untouched_names rewritten = False if not args.no_rewrite_bad_list: - _write_bad_list(bad_list, retained_bad_names, args.dry_run) + _write_bad_list(bad_list, retained_with_untouched, args.dry_run) rewritten = not args.dry_run restage_attempted = 0 @@ -316,7 +399,7 @@ def main() -> int: if not args.no_prune_quarantine: quarantine_summary = _prune_quarantine( quarantine_dir, - retained_bad=set(retained_bad_names), + retained_bad=set(retained_with_untouched), valid_names=set(valid_names), keep_per_name=int(args.quarantine_keep_per_name), dry_run=bool(args.dry_run), @@ -329,8 +412,12 @@ def main() -> int: "quarantine_dir": str(quarantine_dir), "field": args.field, "input_bad_entries": len(bad_names), + "processed_entries": len(processed_names), + "untouched_entries": len(untouched_names), + "workers": workers, + "validate_retries": retries, "valid_reinstated": len(valid_names), - "retained_bad_entries": len(retained_bad_names), + "retained_bad_entries": len(retained_with_untouched), "rewrote_bad_list": rewritten, "dry_run": bool(args.dry_run), "restage_valid": bool(args.restage_valid), @@ -348,7 +435,8 @@ def main() -> int: "summary": summary, "results": [result.__dict__ for result in results], "valid_names": valid_names, - "retained_bad_names": retained_bad_names, + "retained_bad_names": retained_with_untouched, + "untouched_names": untouched_names, } report_output.write_text(json.dumps(payload, indent=2), encoding="utf-8") @@ -356,7 +444,7 @@ def main() -> int: "revalidate_done", f"input={len(bad_names)}", f"reinstated={len(valid_names)}", - f"retained={len(retained_bad_names)}", + f"retained={len(retained_with_untouched)}", f"restage_ok={restage_ok}", f"report={report_output}", ) diff --git a/scripts/shard_offload_cycle.sh b/scripts/shard_offload_cycle.sh new file mode 100755 index 0000000..2dece7f --- /dev/null +++ b/scripts/shard_offload_cycle.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Safe shard offload cycle: +# 1) Reconcile offloaded manifests (restore untrained / under-coverage) +# 2) Run gated shard offload +# 3) Reconcile again to self-heal mismatches + +OFFLOAD_ARGS_DEFAULT="--shards-root data/shards_global/fineweb-global-bpe-v1 --warm-shards-root /mnt/ceph/llm/data/shards_global/fineweb-global-bpe-v1 --keep-local-batches 24 --target-free-gib 180 --max-batches 24 --disable-offloaded-manifests --require-trained-batches-file artifacts/reports/train_supervisor_phase1_talk/trained_batch_names.txt,artifacts/reports/train_supervisor_350bt/trained_batch_names.txt --skip-if-trained-file-missing --min-active-manifests 48 --min-active-train-tokens 40000000000" +RECONCILE_ARGS_DEFAULT="--shards-root data/shards_global/fineweb-global-bpe-v1 --trained-batches-file artifacts/reports/train_supervisor_phase1_talk/trained_batch_names.txt,artifacts/reports/train_supervisor_350bt/trained_batch_names.txt --skip-if-trained-file-missing --min-active-unique-input-files 240 --max-restore 4 --warm-shards-root /mnt/ceph/llm/data/shards_global/fineweb-global-bpe-v1 --rehydrate-restored-bins --rehydrate-active-symlink-bins" + +OFFLOAD_ARGS_RAW="${LLM_SHARD_OFFLOAD_ARGS:-$OFFLOAD_ARGS_DEFAULT}" +RECONCILE_ARGS_RAW="${LLM_SHARD_OFFLOAD_RECONCILE_ARGS:-$RECONCILE_ARGS_DEFAULT}" + +expand_args() { + local raw="$1" + local -n out_ref="$2" + out_ref=() + eval "set -- $raw" + while [[ $# -gt 0 ]]; do + out_ref+=("$1") + shift + done +} + +run_cycle() { + local phase="$1" + local -a reconcile_args=() + expand_args "$RECONCILE_ARGS_RAW" reconcile_args + local report="artifacts/reports/offload_reconcile_${phase}_$(date +%Y%m%d_%H%M%S).json" + echo "[$(date -Iseconds)] offload_cycle_reconcile phase=$phase args=${RECONCILE_ARGS_RAW} report=$report" + PYTHONPATH=src .venv/bin/python scripts/reconcile_offloaded_manifests.py \ + "${reconcile_args[@]}" \ + --report-output "$report" +} + +run_cycle "pre" + +declare -a offload_args=() +expand_args "$OFFLOAD_ARGS_RAW" offload_args + +shards_root_from_args() { + local default_root="data/shards_global/fineweb-global-bpe-v1" + local idx + for ((idx = 0; idx < ${#offload_args[@]}; idx++)); do + if [[ "${offload_args[$idx]}" == "--shards-root" ]] && ((idx + 1 < ${#offload_args[@]})); then + echo "${offload_args[$((idx + 1))]}" + return 0 + fi + done + echo "$default_root" +} + +echo "[$(date -Iseconds)] offload_cycle_offload args=${OFFLOAD_ARGS_RAW}" +PYTHONPATH=src .venv/bin/python scripts/offload_shard_bins_to_warm.py "${offload_args[@]}" + +run_cycle "post" + +SHARDS_ROOT="$(shards_root_from_args)" +echo "[$(date -Iseconds)] offload_cycle_enforce_hot_only shards_root=$SHARDS_ROOT" +PYTHONPATH=src .venv/bin/python scripts/enforce_hot_only_manifests.py --shards-root "$SHARDS_ROOT" + +echo "[$(date -Iseconds)] offload_cycle_done" diff --git a/scripts/sync_fineweb_edu_full.sh b/scripts/sync_fineweb_edu_full.sh new file mode 100755 index 0000000..2bf8c47 --- /dev/null +++ b/scripts/sync_fineweb_edu_full.sh @@ -0,0 +1,120 @@ +#!/usr/bin/env bash +set -euo pipefail + +usage() { + cat <<'EOF' +Usage: + bash scripts/sync_fineweb_edu_full.sh [local_dir] + +Defaults: + local_dir: /mnt/pve/cephfs/llm/data/fineweb/fineweb-edu-full + +Environment overrides: + MAX_WORKERS=6 + RETRY_DELAY_SECONDS=30 + ATTEMPT_TIMEOUT_SECONDS=10800 + MAX_RETRIES=0 # 0 = retry forever + HF_HUB_ENABLE_HF_TRANSFER=1 # set 0 to disable + HF_TOKEN=hf_xxx # optional for public dataset, recommended for rate limits + +Notes: + - Downloads full HuggingFaceFW/fineweb-edu parquet set (resumable). + - Safe to rerun; it resumes existing partial files. +EOF +} + +if [[ "${1:-}" == "-h" || "${1:-}" == "--help" ]]; then + usage + exit 0 +fi + +require_cmd() { + if ! command -v "$1" >/dev/null 2>&1; then + echo "error: command not found: $1" >&2 + exit 1 + fi +} + +LOCAL_DIR="${1:-/mnt/pve/cephfs/llm/data/fineweb/fineweb-edu-full}" +DATASET="HuggingFaceFW/fineweb-edu" +INCLUDE_PATTERN="data/*/*.parquet" +MAX_WORKERS="${MAX_WORKERS:-6}" +RETRY_DELAY_SECONDS="${RETRY_DELAY_SECONDS:-30}" +ATTEMPT_TIMEOUT_SECONDS="${ATTEMPT_TIMEOUT_SECONDS:-10800}" +MAX_RETRIES="${MAX_RETRIES:-0}" +HF_HUB_ENABLE_HF_TRANSFER="${HF_HUB_ENABLE_HF_TRANSFER:-1}" +LOG_FILE="${LOCAL_DIR}/fineweb_edu_sync.log" + +HF_BIN="hf" +if ! command -v "$HF_BIN" >/dev/null 2>&1; then + HF_BIN="huggingface-cli" +fi + +require_cmd "$HF_BIN" +require_cmd find +require_cmd date +require_cmd awk +require_cmd sleep + +mkdir -p "$LOCAL_DIR" + +LOCK_FILE="${LOCAL_DIR}/.fineweb_edu_sync.lock" +if [[ -f "$LOCK_FILE" ]]; then + prev_pid="$(cat "$LOCK_FILE" 2>/dev/null || true)" + if [[ -n "${prev_pid:-}" ]] && kill -0 "$prev_pid" 2>/dev/null; then + echo "error: sync already running (pid=$prev_pid)" >&2 + exit 3 + fi +fi +echo "$$" > "$LOCK_FILE" +trap 'rm -f "$LOCK_FILE"' EXIT + +log() { + local ts + ts="$(date '+%Y-%m-%d %H:%M:%S')" + printf '[%s] %s\n' "$ts" "$1" | tee -a "$LOG_FILE" +} + +attempt=1 +while true; do + if [[ "$MAX_RETRIES" -gt 0 && "$attempt" -gt "$MAX_RETRIES" ]]; then + log "max retries reached (${MAX_RETRIES}); exiting with failure" + exit 10 + fi + + parquet_count="$(find "$LOCAL_DIR" -type f -name '*.parquet' | wc -l | tr -d ' ')" + incomplete_count="$(find "$LOCAL_DIR" -type f -name '*.incomplete' | wc -l | tr -d ' ')" + log "attempt=${attempt} parquet_files=${parquet_count} incomplete_files=${incomplete_count}" + + cmd=( + "$HF_BIN" download "$DATASET" + --repo-type dataset + --include "$INCLUDE_PATTERN" + --local-dir "$LOCAL_DIR" + --max-workers "$MAX_WORKERS" + ) + + set +e + if command -v timeout >/dev/null 2>&1; then + HF_HUB_DISABLE_XET=1 HF_HUB_ENABLE_HF_TRANSFER="$HF_HUB_ENABLE_HF_TRANSFER" \ + timeout "$ATTEMPT_TIMEOUT_SECONDS" "${cmd[@]}" >> "$LOG_FILE" 2>&1 + rc=$? + else + HF_HUB_DISABLE_XET=1 HF_HUB_ENABLE_HF_TRANSFER="$HF_HUB_ENABLE_HF_TRANSFER" \ + "${cmd[@]}" >> "$LOG_FILE" 2>&1 + rc=$? + fi + set -e + + parquet_count="$(find "$LOCAL_DIR" -type f -name '*.parquet' | wc -l | tr -d ' ')" + incomplete_count="$(find "$LOCAL_DIR" -type f -name '*.incomplete' | wc -l | tr -d ' ')" + + if [[ "$rc" -eq 0 && "$incomplete_count" -eq 0 ]]; then + log "completed successfully parquet_files=${parquet_count}" + exit 0 + fi + + log "attempt=${attempt} rc=${rc} parquet_files=${parquet_count} incomplete_files=${incomplete_count}; retry in ${RETRY_DELAY_SECONDS}s" + sleep "$RETRY_DELAY_SECONDS" + attempt=$((attempt + 1)) +done diff --git a/scripts/train_rtx5070_fineweb_bpe_v1_big.sh b/scripts/train_rtx5070_fineweb_bpe_v1_big.sh deleted file mode 100755 index d5af911..0000000 --- a/scripts/train_rtx5070_fineweb_bpe_v1_big.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -# Tuned for: NVIDIA GeForce RTX 5070 Ti Laptop GPU (12 GB VRAM) -# Profile source: configs/train/rtx5070/fineweb_global_bpe_v1_big.json - -PYTORCH_ALLOC_CONF=expandable_segments:True \ -PYTHONPATH=src \ -.venv/bin/python -m llm.cli train \ - --shards-path data/shards_global/fineweb-global-bpe-v1 \ - --output-dir artifacts/checkpoints/fineweb-global-bpe-v1-big-run1 \ - --device cuda \ - --max-steps 50000 \ - --batch-size 34 \ - --context-length 512 \ - --n-layers 12 \ - --n-heads 12 \ - --d-model 768 \ - --learning-rate 1.5e-4 \ - --lr-schedule cosine \ - --lr-warmup-steps 1000 \ - --lr-min-ratio 0.10 \ - --eval-interval 2000 \ - --eval-steps 5 \ - --fail-on-eval-regression \ - --eval-regression-tolerance 0.20 \ - --log-interval 100 \ - --precision auto diff --git a/scripts/train_supervisor_phase1_english_talk.sh b/scripts/train_supervisor_phase1_english_talk.sh index f8e8e61..2dc7acf 100755 --- a/scripts/train_supervisor_phase1_english_talk.sh +++ b/scripts/train_supervisor_phase1_english_talk.sh @@ -9,6 +9,7 @@ exec bash scripts/train_supervisor_rtx5070_350bt.sh \ --poll-seconds 60 \ --batch-size 12 \ --target-effective-batch 24 \ + --lr-schedule constant \ --min-train-tokens 30000000000 \ --min-batch-size 6 \ --max-batch-size 20 \ @@ -18,10 +19,16 @@ exec bash scripts/train_supervisor_rtx5070_350bt.sh \ --ema-decay 0.999 \ --dedupe-report-keep 240 \ --eval-suite configs/eval/english_talk_suite_v1.json \ - --eval-promotion-policy configs/eval/promotion_policy_talk_v1.json \ - --generation-suite configs/eval/generation_talk_smoke_v1.json \ + --eval-promotion-policy configs/eval/promotion_policy_talk_recovery_v2.json \ + --promotion-min-quality-streak 2 \ + --generation-suite configs/eval/generation_talk_quality_v2.json \ --generation-temperature 0.2 \ --generation-top-k 1 \ + --generation-fail-below-pass-rate 0.35 \ --generation-every-chunks 1 \ - --no-train-fail-on-eval-regression \ + --holdout-suite configs/eval/english_talk_holdout_suite_v1.json \ + --holdout-temperature 0.2 \ + --holdout-top-k 1 \ + --holdout-fail-below-pass-rate 0.35 \ + --holdout-every-chunks 1 \ "$@" diff --git a/scripts/train_supervisor_rtx5070_350bt.sh b/scripts/train_supervisor_rtx5070_350bt.sh index e38e240..d2521e5 100755 --- a/scripts/train_supervisor_rtx5070_350bt.sh +++ b/scripts/train_supervisor_rtx5070_350bt.sh @@ -9,6 +9,7 @@ set -euo pipefail # - automatic post-chunk eval + trend files SHARDS_PATH="data/shards_global/fineweb-global-bpe-v1" +WARM_SHARDS_ROOT="/mnt/ceph/llm/data/shards_global/fineweb-global-bpe-v1" OUTPUT_DIR="artifacts/checkpoints/fineweb-350bt-bpe-v2-run1" STATE_DIR="artifacts/reports/train_supervisor_350bt" @@ -31,6 +32,7 @@ N_LAYERS=12 N_HEADS=12 D_MODEL=768 LEARNING_RATE="3e-4" +LR_SCHEDULE="cosine" LR_WARMUP_STEPS=2000 LR_MIN_RATIO="0.10" EVAL_INTERVAL=1000 @@ -43,6 +45,17 @@ CHECKPOINT_KEEP_EVERY=10000 EMA_DECAY="0.0" EMA_UPDATE_EVERY=1 EMA_START_STEP=0 +SAMPLER_STRATEGY="balanced" +SAMPLER_MIN_FULL_PASSES=1 + +HOT_SHARD_WARMUP=1 +HOT_SHARD_WARMUP_WORKERS=4 +HOT_SHARD_WARMUP_MAX_FILES=0 +HOT_SHARD_WARMUP_ALLOW_MISSING_WARM=0 +HOT_SHARD_WARMUP_BACKGROUND=1 +HOT_SHARD_WARMUP_BACKGROUND_INTERVAL_SECONDS=120 +HOT_SHARD_WARMUP_BACKGROUND_MAX_FILES=64 +HOT_SHARD_WARMUP_BACKGROUND_INCLUDE_OFFLOADED=0 AUTO_TUNE=1 BATCH_STEP=2 @@ -85,6 +98,30 @@ GENERATION_FAIL_BELOW_PASS_RATE="" GENERATION_EVERY_CHUNKS=1 GENERATION_STOP_ON_FAIL=0 +HOLDOUT_GATE=0 +HOLDOUT_SUITE="" +HOLDOUT_MAX_NEW_TOKENS=120 +HOLDOUT_TEMPERATURE="0.2" +HOLDOUT_TOP_K=1 +HOLDOUT_SEED=2718 +HOLDOUT_SEED_STRIDE=53 +HOLDOUT_DEVICE="auto" +HOLDOUT_FAIL_ON_REGRESSION=1 +HOLDOUT_MAX_PASS_RATE_DROP="0.01" +HOLDOUT_MAX_CHECK_PASS_RATE_DROP="0.01" +HOLDOUT_MAX_AVG_CASE_SCORE_DROP="0.01" +HOLDOUT_FAIL_BELOW_PASS_RATE="" +HOLDOUT_EVERY_CHUNKS=1 +HOLDOUT_STOP_ON_FAIL=0 + +PROMOTION_REQUIRE_POLICY_PASS=1 +PROMOTION_REQUIRE_GENERATION_PASS=1 +PROMOTION_REQUIRE_HOLDOUT_PASS=1 +PROMOTION_MIN_QUALITY_STREAK=2 + +QUALITY_ROLLBACK_STREAK=3 +QUALITY_ROLLBACK_COOLDOWN_STEPS=4000 + DEDUPE_OVERLAP_MANIFESTS=1 DEDUPE_KEEP="newest" DEDUPE_DRY_RUN=0 @@ -99,6 +136,8 @@ Core options: --shards-path DIR Root containing shard manifest.json files --output-dir DIR Training output directory (last.pt lives here) --state-dir DIR Supervisor logs/state directory + --warm-shards-root DIR Warm shard root used for hot warmup hydration + (default: /mnt/ceph/llm/data/shards_global/fineweb-global-bpe-v1) --poll-seconds N Sleep between checks/restarts (default: 120) --step-chunk N Steps per training cycle before restart (default: 2000) --min-manifests N Wait until at least N manifests exist (default: 1) @@ -126,6 +165,7 @@ Training shape: --n-heads N Attention head count (default: 12) --d-model N Hidden size (default: 768) --learning-rate X Learning rate (default: 3e-4) + --lr-schedule NAME LR schedule: constant|cosine (default: cosine) --lr-warmup-steps N Warmup steps (default: 2000) --lr-min-ratio X Cosine min ratio (default: 0.10) --eval-interval N Train-loop eval interval (default: 1000) @@ -139,6 +179,25 @@ Training shape: --ema-decay X EMA decay for model weights (default: 0.0 disabled) --ema-update-every N EMA update interval in optimizer steps (default: 1) --ema-start-step N First optimizer step to apply EMA updates (default: 0) + --sampler-strategy MODE Train sampler mode: weighted|balanced (default: balanced) + --sampler-min-full-passes N Require at least N guaranteed full shard passes per chunk + when using balanced sampler (default: 1) + +Hot-set warmup: + --no-hot-shard-warmup Disable pre-train warm->hot shard hydration + --hot-shard-warmup-workers N Parallel warmup copy workers (default: 4) + --hot-shard-warmup-max-files N + Cap warmup copy file count per loop (default: 0 = all) + --hot-shard-warmup-allow-missing-warm + Continue even when warm shard copies are missing + --no-hot-shard-warmup-background + Disable background warmup/prefetch while train chunk runs + --hot-shard-warmup-background-interval-seconds N + Background prefetch interval while train runs (default: 120) + --hot-shard-warmup-background-max-files N + Max files per background warmup pass (default: 64) + --hot-shard-warmup-background-include-offloaded + Also prefetch from manifest.offloaded.json during background passes Auto-tune options: --no-auto-tune Disable automatic batch tuning @@ -193,6 +252,47 @@ Generation gate (scheduled post-chunk prompt generation checks): Fail generation gate if pass_rate drops below X --generation-every-chunks N Run generation gate every N successful chunks (default: 1) --generation-stop-on-fail Stop supervisor when generation gate returns non-zero + +Fixed holdout gate (frozen quality suite): + --no-holdout-gate Disable fixed holdout gate + --holdout-suite FILE Holdout suite JSON path (enables holdout gate) + --holdout-max-new-tokens N Holdout max new tokens per case (default: 120) + --holdout-temperature X Holdout sampling temperature (default: 0.2) + --holdout-top-k N Holdout top-k (default: 1) + --holdout-seed N Holdout base seed (default: 2718) + --holdout-seed-stride N Holdout seed stride (default: 53) + --holdout-device NAME Holdout device (default: auto) + --no-holdout-fail-on-regression + Disable holdout regression fail flag + --holdout-max-pass-rate-drop X + Allowed holdout pass_rate drop vs fixed baseline (default: 0.01) + --holdout-max-check-pass-rate-drop X + Allowed holdout check_pass_rate drop vs fixed baseline (default: 0.01) + --holdout-max-avg-case-score-drop X + Allowed holdout avg_case_score drop vs fixed baseline (default: 0.01) + --holdout-fail-below-pass-rate X + Fail holdout gate if pass_rate drops below X + --holdout-every-chunks N Run holdout gate every N successful chunks (default: 1) + --holdout-stop-on-fail Stop supervisor when holdout gate returns non-zero + +Promotion discipline: + --no-promotion-require-policy-pass + Allow best promotion without eval policy promotion flag + --no-promotion-require-generation-pass + Allow best promotion without passing generation gate + --no-promotion-require-holdout-pass + Allow best promotion without passing holdout gate + --promotion-min-quality-streak N + Require N consecutive quality-passing chunks before promotion + (default: 2) + +Quality rollback: + --quality-rollback-streak N Roll back to best checkpoint after N consecutive failed + quality chunks (eval/gen gate regressions). 0 disables. + (default: 3) + --quality-rollback-cooldown-steps N + Minimum training-step gap between auto-rollbacks + (default: 4000) -h, --help Show help Example: @@ -205,6 +305,7 @@ USAGE while [[ $# -gt 0 ]]; do case "$1" in --shards-path) SHARDS_PATH="$2"; shift 2 ;; + --warm-shards-root) WARM_SHARDS_ROOT="$2"; shift 2 ;; --output-dir) OUTPUT_DIR="$2"; shift 2 ;; --state-dir) STATE_DIR="$2"; shift 2 ;; --poll-seconds) POLL_SECONDS="$2"; shift 2 ;; @@ -229,6 +330,7 @@ while [[ $# -gt 0 ]]; do --n-heads) N_HEADS="$2"; shift 2 ;; --d-model) D_MODEL="$2"; shift 2 ;; --learning-rate) LEARNING_RATE="$2"; shift 2 ;; + --lr-schedule) LR_SCHEDULE="$2"; shift 2 ;; --lr-warmup-steps) LR_WARMUP_STEPS="$2"; shift 2 ;; --lr-min-ratio) LR_MIN_RATIO="$2"; shift 2 ;; --eval-interval) EVAL_INTERVAL="$2"; shift 2 ;; @@ -241,6 +343,16 @@ while [[ $# -gt 0 ]]; do --ema-decay) EMA_DECAY="$2"; shift 2 ;; --ema-update-every) EMA_UPDATE_EVERY="$2"; shift 2 ;; --ema-start-step) EMA_START_STEP="$2"; shift 2 ;; + --sampler-strategy) SAMPLER_STRATEGY="$2"; shift 2 ;; + --sampler-min-full-passes) SAMPLER_MIN_FULL_PASSES="$2"; shift 2 ;; + --no-hot-shard-warmup) HOT_SHARD_WARMUP=0; shift ;; + --hot-shard-warmup-workers) HOT_SHARD_WARMUP_WORKERS="$2"; shift 2 ;; + --hot-shard-warmup-max-files) HOT_SHARD_WARMUP_MAX_FILES="$2"; shift 2 ;; + --hot-shard-warmup-allow-missing-warm) HOT_SHARD_WARMUP_ALLOW_MISSING_WARM=1; shift ;; + --no-hot-shard-warmup-background) HOT_SHARD_WARMUP_BACKGROUND=0; shift ;; + --hot-shard-warmup-background-interval-seconds) HOT_SHARD_WARMUP_BACKGROUND_INTERVAL_SECONDS="$2"; shift 2 ;; + --hot-shard-warmup-background-max-files) HOT_SHARD_WARMUP_BACKGROUND_MAX_FILES="$2"; shift 2 ;; + --hot-shard-warmup-background-include-offloaded) HOT_SHARD_WARMUP_BACKGROUND_INCLUDE_OFFLOADED=1; shift ;; --no-auto-tune) AUTO_TUNE=0; shift ;; --batch-step) BATCH_STEP="$2"; shift 2 ;; --min-batch-size) MIN_BATCH_SIZE="$2"; shift 2 ;; @@ -279,6 +391,27 @@ while [[ $# -gt 0 ]]; do --generation-fail-below-pass-rate) GENERATION_FAIL_BELOW_PASS_RATE="$2"; shift 2 ;; --generation-every-chunks) GENERATION_EVERY_CHUNKS="$2"; shift 2 ;; --generation-stop-on-fail) GENERATION_STOP_ON_FAIL=1; shift ;; + --no-holdout-gate) HOLDOUT_GATE=0; shift ;; + --holdout-suite) HOLDOUT_SUITE="$2"; HOLDOUT_GATE=1; shift 2 ;; + --holdout-max-new-tokens) HOLDOUT_MAX_NEW_TOKENS="$2"; shift 2 ;; + --holdout-temperature) HOLDOUT_TEMPERATURE="$2"; shift 2 ;; + --holdout-top-k) HOLDOUT_TOP_K="$2"; shift 2 ;; + --holdout-seed) HOLDOUT_SEED="$2"; shift 2 ;; + --holdout-seed-stride) HOLDOUT_SEED_STRIDE="$2"; shift 2 ;; + --holdout-device) HOLDOUT_DEVICE="$2"; shift 2 ;; + --no-holdout-fail-on-regression) HOLDOUT_FAIL_ON_REGRESSION=0; shift ;; + --holdout-max-pass-rate-drop) HOLDOUT_MAX_PASS_RATE_DROP="$2"; shift 2 ;; + --holdout-max-check-pass-rate-drop) HOLDOUT_MAX_CHECK_PASS_RATE_DROP="$2"; shift 2 ;; + --holdout-max-avg-case-score-drop) HOLDOUT_MAX_AVG_CASE_SCORE_DROP="$2"; shift 2 ;; + --holdout-fail-below-pass-rate) HOLDOUT_FAIL_BELOW_PASS_RATE="$2"; shift 2 ;; + --holdout-every-chunks) HOLDOUT_EVERY_CHUNKS="$2"; shift 2 ;; + --holdout-stop-on-fail) HOLDOUT_STOP_ON_FAIL=1; shift ;; + --no-promotion-require-policy-pass) PROMOTION_REQUIRE_POLICY_PASS=0; shift ;; + --no-promotion-require-generation-pass) PROMOTION_REQUIRE_GENERATION_PASS=0; shift ;; + --no-promotion-require-holdout-pass) PROMOTION_REQUIRE_HOLDOUT_PASS=0; shift ;; + --promotion-min-quality-streak) PROMOTION_MIN_QUALITY_STREAK="$2"; shift 2 ;; + --quality-rollback-streak) QUALITY_ROLLBACK_STREAK="$2"; shift 2 ;; + --quality-rollback-cooldown-steps) QUALITY_ROLLBACK_COOLDOWN_STEPS="$2"; shift 2 ;; -h|--help) usage; exit 0 ;; *) echo "unknown argument: $1" >&2 @@ -325,6 +458,38 @@ if [[ "$BATCH_SIZE" -le 0 || "$GRAD_ACCUM_STEPS" -le 0 ]]; then echo "error: batch-size and grad-accum-steps must be > 0" >&2 exit 1 fi +if [[ "$LR_SCHEDULE" != "constant" && "$LR_SCHEDULE" != "cosine" ]]; then + echo "error: lr-schedule must be one of: constant, cosine" >&2 + exit 1 +fi +if [[ "$SAMPLER_STRATEGY" != "weighted" && "$SAMPLER_STRATEGY" != "balanced" ]]; then + echo "error: sampler-strategy must be one of: weighted, balanced" >&2 + exit 1 +fi +if ! [[ "$SAMPLER_MIN_FULL_PASSES" =~ ^[0-9]+$ ]]; then + echo "error: sampler-min-full-passes must be an integer >= 0" >&2 + exit 1 +fi +if ! [[ "$HOT_SHARD_WARMUP_WORKERS" =~ ^[0-9]+$ ]] || [[ "$HOT_SHARD_WARMUP_WORKERS" -le 0 ]]; then + echo "error: hot-shard-warmup-workers must be an integer > 0" >&2 + exit 1 +fi +if ! [[ "$HOT_SHARD_WARMUP_MAX_FILES" =~ ^[0-9]+$ ]]; then + echo "error: hot-shard-warmup-max-files must be an integer >= 0" >&2 + exit 1 +fi +if ! [[ "$HOT_SHARD_WARMUP_BACKGROUND_INTERVAL_SECONDS" =~ ^[0-9]+$ ]] || [[ "$HOT_SHARD_WARMUP_BACKGROUND_INTERVAL_SECONDS" -le 0 ]]; then + echo "error: hot-shard-warmup-background-interval-seconds must be an integer > 0" >&2 + exit 1 +fi +if ! [[ "$HOT_SHARD_WARMUP_BACKGROUND_MAX_FILES" =~ ^[0-9]+$ ]]; then + echo "error: hot-shard-warmup-background-max-files must be an integer >= 0" >&2 + exit 1 +fi +if [[ "$HOT_SHARD_WARMUP" -eq 1 && ! -d "$WARM_SHARDS_ROOT" ]]; then + echo "error: warm-shards-root not found: $WARM_SHARDS_ROOT" >&2 + exit 1 +fi if [[ "$CHECKPOINT_KEEP_LAST" -lt 0 || "$CHECKPOINT_KEEP_EVERY" -lt 0 ]]; then echo "error: checkpoint retention values must be >= 0" >&2 exit 1 @@ -333,6 +498,26 @@ if [[ "$GENERATION_EVERY_CHUNKS" -le 0 ]]; then echo "error: generation-every-chunks must be > 0" >&2 exit 1 fi +if [[ "$HOLDOUT_GATE" -eq 1 && -z "$HOLDOUT_SUITE" ]]; then + echo "error: holdout-suite must be set when holdout gate is enabled" >&2 + exit 1 +fi +if [[ "$HOLDOUT_GATE" -eq 1 && ! -f "$HOLDOUT_SUITE" ]]; then + echo "error: holdout-suite not found: $HOLDOUT_SUITE" >&2 + exit 1 +fi +if [[ "$HOLDOUT_EVERY_CHUNKS" -le 0 ]]; then + echo "error: holdout-every-chunks must be > 0" >&2 + exit 1 +fi +if ! [[ "$QUALITY_ROLLBACK_STREAK" =~ ^[0-9]+$ ]] || ! [[ "$QUALITY_ROLLBACK_COOLDOWN_STEPS" =~ ^[0-9]+$ ]]; then + echo "error: quality rollback values must be integers >= 0" >&2 + exit 1 +fi +if ! [[ "$PROMOTION_MIN_QUALITY_STREAK" =~ ^[0-9]+$ ]]; then + echo "error: promotion-min-quality-streak must be an integer >= 0" >&2 + exit 1 +fi if [[ "$DEDUPE_KEEP" != "newest" && "$DEDUPE_KEEP" != "oldest" ]]; then echo "error: dedupe-keep must be one of: newest, oldest" >&2 exit 1 @@ -359,13 +544,16 @@ SUP_LOG="$STATE_DIR/supervisor_$(date +%Y%m%d_%H%M%S).log" TRAIN_TREND_TSV="$STATE_DIR/train_trend.tsv" EVAL_TREND_TSV="$STATE_DIR/eval_trend.tsv" GENERATION_TREND_TSV="$STATE_DIR/generation_trend.tsv" +HOLDOUT_TREND_TSV="$STATE_DIR/holdout_trend.tsv" +HOLDOUT_BASELINE_FILE="$STATE_DIR/holdout_baseline_report.txt" BEST_META_JSON="$STATE_DIR/best_checkpoint.json" TRAINED_BATCHES_FILE="$STATE_DIR/trained_batch_names.txt" +LAST_QUALITY_ROLLBACK_STEP_FILE="$STATE_DIR/last_quality_rollback_step.txt" touch "$SUP_LOG" touch "$TRAINED_BATCHES_FILE" if [[ ! -f "$TRAIN_TREND_TSV" ]]; then - echo -e "run_tag\tstep_start\tstep_target\tstep_end\trc\tmanifests\tbatch_size\tgrad_accum\tbest_val_ppl\tgpu_avg_util\tgpu_max_mem_mib\ttrain_log" > "$TRAIN_TREND_TSV" + echo -e "run_tag\tstep_start\tstep_target\tstep_end\trc\tmanifests\tbatch_size\tgrad_accum\tbest_val_ppl\tgpu_avg_util\tgpu_max_mem_mib\tsampled_batches\tsampled_trace\ttrain_log" > "$TRAIN_TREND_TSV" fi if [[ ! -f "$EVAL_TREND_TSV" ]]; then echo -e "run_tag\tstep\teval_rc\tpass_rate\tcheck_pass_rate\tavg_case_score\tcases_passed\tcases_total\tregression_pass\tpromotion_pass\tfailed_checks\tbaseline_report\treport_json" > "$EVAL_TREND_TSV" @@ -373,6 +561,20 @@ fi if [[ ! -f "$GENERATION_TREND_TSV" ]]; then echo -e "run_tag\tstep\tgeneration_rc\tpass_rate\tcheck_pass_rate\tavg_case_score\tcases_passed\tcases_total\tregression_pass\tbaseline_report\treport_json" > "$GENERATION_TREND_TSV" fi +if [[ ! -f "$HOLDOUT_TREND_TSV" ]]; then + echo -e "run_tag\tstep\tholdout_rc\tpass_rate\tcheck_pass_rate\tavg_case_score\tcases_passed\tcases_total\tregression_pass\tbaseline_report\treport_json" > "$HOLDOUT_TREND_TSV" +fi + +LAST_EVAL_RAN=0 +LAST_EVAL_RC=0 +LAST_EVAL_REPORT="" +LAST_EVAL_REGRESSION_PASS="NA" +LAST_EVAL_PROMOTION_PASS="NA" +LAST_GENERATION_GATE_RAN=0 +LAST_GENERATION_GATE_RC=0 +LAST_HOLDOUT_GATE_RAN=0 +LAST_HOLDOUT_GATE_RC=0 +QUALITY_PASS_STREAK=0 log() { echo "[$(date -Iseconds)] $*" | tee -a "$SUP_LOG" @@ -451,6 +653,35 @@ print("") PY } +is_truthy() { + case "$1" in + 1|true|True|TRUE|yes|Yes|YES) return 0 ;; + *) return 1 ;; + esac +} + +fixed_holdout_baseline_report() { + if [[ ! -f "$HOLDOUT_BASELINE_FILE" ]]; then + echo "" + return 0 + fi + local report + report="$(head -n 1 "$HOLDOUT_BASELINE_FILE" 2>/dev/null || true)" + if [[ -n "$report" && -f "$report" ]]; then + echo "$report" + return 0 + fi + echo "" +} + +set_holdout_baseline_report() { + local report="$1" + if [[ -z "$report" || ! -f "$report" ]]; then + return 0 + fi + printf '%s\n' "$report" > "$HOLDOUT_BASELINE_FILE" +} + find_oldest_supervisor_pid() { ps -eo pid=,ppid=,etimes=,args= | awk -v target_state="$STATE_DIR" -v self_pid="$$" ' { @@ -687,17 +918,61 @@ for name in names: PY } +collect_sampled_batch_names() { + local sampled_trace_json="$1" + "$PYTHON_BIN" - <<'PY' "$sampled_trace_json" +import json +import sys +from pathlib import Path + +trace_path = Path(sys.argv[1]) +if not trace_path.is_file(): + raise SystemExit(0) + +try: + payload = json.loads(trace_path.read_text(encoding="utf-8")) +except Exception: + raise SystemExit(0) + +rows = payload.get("sampled_shards", []) +if not isinstance(rows, list): + raise SystemExit(0) + +batches: set[str] = set() +for row in rows: + if not isinstance(row, dict): + continue + raw = row.get("path") + if not isinstance(raw, str) or not raw.strip(): + continue + batch = Path(raw).resolve().parent.name + if batch: + batches.add(batch) + +for batch in sorted(batches): + print(batch) +PY +} + update_trained_batch_registry() { - local chunk_batches_file="$1" + local sampled_batches_file="$1" local step="$2" - if [[ ! -f "$chunk_batches_file" ]]; then + local chunk_batches_count="$3" + if [[ ! -f "$sampled_batches_file" ]]; then + log "trained_batches_skip step=$step reason=missing_sampled_batches_file file=$sampled_batches_file" + return 0 + fi + local sampled_count + sampled_count="$(wc -l < "$sampled_batches_file" | tr -d ' ')" + if [[ "$sampled_count" -le 0 ]]; then + log "trained_batches_skip step=$step reason=empty_sampled_batches sampled_file=$sampled_batches_file chunk_batches=$chunk_batches_count" return 0 fi - cat "$chunk_batches_file" >> "$TRAINED_BATCHES_FILE" + cat "$sampled_batches_file" >> "$TRAINED_BATCHES_FILE" sort -u -o "$TRAINED_BATCHES_FILE" "$TRAINED_BATCHES_FILE" local trained_count trained_count="$(wc -l < "$TRAINED_BATCHES_FILE" | tr -d ' ')" - log "trained_batches_update step=$step trained_batches=$trained_count registry=$TRAINED_BATCHES_FILE" + log "trained_batches_update step=$step sampled_batches=$sampled_count chunk_batches=$chunk_batches_count trained_batches=$trained_count registry=$TRAINED_BATCHES_FILE sampled_file=$sampled_batches_file" } backfill_trained_batch_registry() { @@ -728,8 +1003,8 @@ if trend_path.exists(): rc = parts[4].strip() if not run_tag or rc != "0": continue - for chunk_file in sorted(state_dir.glob(f"chunk_batches_*_{run_tag}.txt")): - for line in chunk_file.read_text(encoding="utf-8", errors="replace").splitlines(): + for sampled_file in sorted(state_dir.glob(f"sampled_batches_*_{run_tag}.txt")): + for line in sampled_file.read_text(encoding="utf-8", errors="replace").splitlines(): batch = line.strip() if not batch or batch in existing: continue @@ -817,6 +1092,96 @@ run_hot_manifest_guard() { log "$guard_out" } +run_hot_shard_warmup() { + if [[ "$HOT_SHARD_WARMUP" -ne 1 ]]; then + return 0 + fi + local warmup_report="$STATE_DIR/hot_shard_warmup_latest.json" + local -a warmup_args=( + --shards-root "$SHARDS_PATH" + --warm-shards-root "$WARM_SHARDS_ROOT" + --workers "$HOT_SHARD_WARMUP_WORKERS" + --report-output "$warmup_report" + ) + if [[ "$HOT_SHARD_WARMUP_MAX_FILES" -gt 0 ]]; then + warmup_args+=(--max-files "$HOT_SHARD_WARMUP_MAX_FILES") + fi + if [[ "$HOT_SHARD_WARMUP_ALLOW_MISSING_WARM" -eq 1 ]]; then + warmup_args+=(--allow-missing-warm) + fi + local warmup_out + set +e + warmup_out="$( + PYTHONPATH=src \ + "$PYTHON_BIN" scripts/hot_shard_warmup.py \ + "${warmup_args[@]}" \ + 2>&1 + )" + local rc=$? + set -e + if [[ "$rc" -ne 0 ]]; then + log "hot_shard_warmup_failed rc=$rc detail=$(echo "$warmup_out" | tr '\n' ' ')" + return "$rc" + fi + log "$warmup_out" + return 0 +} + +start_hot_shard_warmup_background() { + local train_pid="$1" + HOT_SHARD_WARMUP_BG_PID="" + if [[ "$HOT_SHARD_WARMUP" -ne 1 || "$HOT_SHARD_WARMUP_BACKGROUND" -ne 1 ]]; then + return 0 + fi + ( + while kill -0 "$train_pid" 2>/dev/null; do + local -a warmup_args=( + --shards-root "$SHARDS_PATH" + --warm-shards-root "$WARM_SHARDS_ROOT" + --workers "$HOT_SHARD_WARMUP_WORKERS" + --report-output "$STATE_DIR/hot_shard_warmup_background_latest.json" + ) + if [[ "$HOT_SHARD_WARMUP_BACKGROUND_MAX_FILES" -gt 0 ]]; then + warmup_args+=(--max-files "$HOT_SHARD_WARMUP_BACKGROUND_MAX_FILES") + fi + if [[ "$HOT_SHARD_WARMUP_ALLOW_MISSING_WARM" -eq 1 ]]; then + warmup_args+=(--allow-missing-warm) + fi + if [[ "$HOT_SHARD_WARMUP_BACKGROUND_INCLUDE_OFFLOADED" -eq 1 ]]; then + warmup_args+=(--include-offloaded-manifests) + fi + local warmup_out + set +e + warmup_out="$( + PYTHONPATH=src \ + "$PYTHON_BIN" scripts/hot_shard_warmup.py \ + "${warmup_args[@]}" \ + 2>&1 + )" + local rc=$? + set -e + if [[ "$rc" -eq 0 ]]; then + log "hot_shard_warmup_background_done detail=$(echo "$warmup_out" | tr '\n' ' ')" + else + log "hot_shard_warmup_background_failed rc=$rc detail=$(echo "$warmup_out" | tr '\n' ' ')" + fi + sleep "$HOT_SHARD_WARMUP_BACKGROUND_INTERVAL_SECONDS" + done + ) & + HOT_SHARD_WARMUP_BG_PID="$!" + log "hot_shard_warmup_background_start pid=$HOT_SHARD_WARMUP_BG_PID interval=${HOT_SHARD_WARMUP_BACKGROUND_INTERVAL_SECONDS}s max_files=$HOT_SHARD_WARMUP_BACKGROUND_MAX_FILES include_offloaded=$HOT_SHARD_WARMUP_BACKGROUND_INCLUDE_OFFLOADED" +} + +stop_hot_shard_warmup_background() { + if [[ -z "${HOT_SHARD_WARMUP_BG_PID:-}" ]]; then + return 0 + fi + kill "$HOT_SHARD_WARMUP_BG_PID" 2>/dev/null || true + wait "$HOT_SHARD_WARMUP_BG_PID" 2>/dev/null || true + log "hot_shard_warmup_background_stop pid=$HOT_SHARD_WARMUP_BG_PID" + HOT_SHARD_WARMUP_BG_PID="" +} + run_manifest_dedupe() { if [[ "$DEDUPE_OVERLAP_MANIFESTS" -ne 1 ]]; then return 0 @@ -1087,6 +1452,11 @@ auto_tune_after_chunk() { run_post_chunk_eval() { local run_tag="$1" local step="$2" + LAST_EVAL_RAN=0 + LAST_EVAL_RC=0 + LAST_EVAL_REPORT="" + LAST_EVAL_REGRESSION_PASS="NA" + LAST_EVAL_PROMOTION_PASS="NA" if [[ "$EVAL_AFTER_CHUNK" -ne 1 ]]; then return 0 fi @@ -1169,6 +1539,11 @@ print( PY ) fi + LAST_EVAL_RAN=1 + LAST_EVAL_RC="$eval_rc" + LAST_EVAL_REPORT="$eval_report" + LAST_EVAL_REGRESSION_PASS="$regression_pass" + LAST_EVAL_PROMOTION_PASS="$promotion_pass" echo -e "${run_tag}\t${step}\t${eval_rc}\t${pass_rate}\t${check_pass_rate}\t${avg_case_score}\t${cases_passed}\t${cases_total}\t${regression_pass}\t${promotion_pass}\t${failed_checks}\t${baseline_report:-NA}\t${eval_report}" >> "$EVAL_TREND_TSV" log "eval_done rc=$eval_rc pass_rate=$pass_rate check_pass_rate=$check_pass_rate avg_case_score=$avg_case_score regression_pass=$regression_pass promotion_pass=$promotion_pass baseline=${baseline_report:-none} report=$eval_report" @@ -1183,13 +1558,14 @@ PY log "eval_dashboard_update_failed" fi fi - promote_best_checkpoint_if_needed "$step" "$eval_rc" "$eval_report" } run_generation_gate() { local run_tag="$1" local step="$2" local successful_chunks="$3" + LAST_GENERATION_GATE_RAN=0 + LAST_GENERATION_GATE_RC=0 if [[ "$GENERATION_GATE" -ne 1 ]]; then return 0 fi @@ -1241,6 +1617,8 @@ run_generation_gate() { > "$gen_log" 2>&1 local gen_rc=$? set -e + LAST_GENERATION_GATE_RAN=1 + LAST_GENERATION_GATE_RC="$gen_rc" local pass_rate="NA" local check_pass_rate="NA" @@ -1273,6 +1651,315 @@ PY return "$gen_rc" } +run_holdout_gate() { + local run_tag="$1" + local step="$2" + local successful_chunks="$3" + LAST_HOLDOUT_GATE_RAN=0 + LAST_HOLDOUT_GATE_RC=0 + if [[ "$HOLDOUT_GATE" -ne 1 ]]; then + return 0 + fi + if (( successful_chunks % HOLDOUT_EVERY_CHUNKS != 0 )); then + log "holdout_gate_skip reason=interval step=$step successful_chunks=$successful_chunks interval=$HOLDOUT_EVERY_CHUNKS" + return 0 + fi + if [[ ! -f "$OUTPUT_DIR/last.pt" ]]; then + log "holdout_gate_skip reason=no_last_checkpoint" + return 0 + fi + + local holdout_report="artifacts/reports/evals/holdout_gate_step$(printf '%07d' "$step")_${run_tag}.json" + local holdout_log="$STATE_DIR/holdout_gate_${step}_${run_tag}.log" + local baseline_report="" + baseline_report="$(fixed_holdout_baseline_report)" + if [[ -n "$baseline_report" && ! -f "$baseline_report" ]]; then + baseline_report="" + fi + log "holdout_gate_start step=$step suite=$HOLDOUT_SUITE report=$holdout_report baseline=${baseline_report:-none}" + + local -a holdout_extra_args=() + if [[ -n "$baseline_report" ]]; then + holdout_extra_args+=(--baseline-report "$baseline_report") + holdout_extra_args+=(--max-pass-rate-drop "$HOLDOUT_MAX_PASS_RATE_DROP") + holdout_extra_args+=(--max-check-pass-rate-drop "$HOLDOUT_MAX_CHECK_PASS_RATE_DROP") + holdout_extra_args+=(--max-avg-case-score-drop "$HOLDOUT_MAX_AVG_CASE_SCORE_DROP") + if [[ "$HOLDOUT_FAIL_ON_REGRESSION" -eq 1 ]]; then + holdout_extra_args+=(--fail-on-regression) + fi + fi + if [[ -n "$HOLDOUT_FAIL_BELOW_PASS_RATE" ]]; then + holdout_extra_args+=(--fail-below-pass-rate "$HOLDOUT_FAIL_BELOW_PASS_RATE") + fi + + set +e + PYTHONPATH=src \ + .venv/bin/python scripts/eval_checkpoint_prompts.py \ + --checkpoint "$OUTPUT_DIR/last.pt" \ + --suite "$HOLDOUT_SUITE" \ + --output "$holdout_report" \ + --device "$HOLDOUT_DEVICE" \ + --max-new-tokens "$HOLDOUT_MAX_NEW_TOKENS" \ + --temperature "$HOLDOUT_TEMPERATURE" \ + --top-k "$HOLDOUT_TOP_K" \ + --seed "$HOLDOUT_SEED" \ + --seed-stride "$HOLDOUT_SEED_STRIDE" \ + "${holdout_extra_args[@]}" \ + > "$holdout_log" 2>&1 + local holdout_rc=$? + set -e + LAST_HOLDOUT_GATE_RAN=1 + LAST_HOLDOUT_GATE_RC="$holdout_rc" + + local pass_rate="NA" + local check_pass_rate="NA" + local avg_case_score="NA" + local cases_passed="NA" + local cases_total="NA" + local regression_pass="NA" + if [[ -f "$holdout_report" ]]; then + read -r pass_rate check_pass_rate avg_case_score cases_passed cases_total regression_pass < <( + python3 - <<'PY' "$holdout_report" +import json +import sys +obj = json.load(open(sys.argv[1], "r", encoding="utf-8")) +s = obj.get("summary", {}) +r = obj.get("regression", {}) +print( + s.get("pass_rate", "NA"), + s.get("check_pass_rate", "NA"), + s.get("avg_case_score", "NA"), + s.get("cases_passed", "NA"), + s.get("cases_total", "NA"), + r.get("pass", "NA") if isinstance(r, dict) else "NA", +) +PY + ) + fi + + echo -e "${run_tag}\t${step}\t${holdout_rc}\t${pass_rate}\t${check_pass_rate}\t${avg_case_score}\t${cases_passed}\t${cases_total}\t${regression_pass}\t${baseline_report:-NA}\t${holdout_report}" >> "$HOLDOUT_TREND_TSV" + log "holdout_gate_done rc=$holdout_rc pass_rate=$pass_rate check_pass_rate=$check_pass_rate avg_case_score=$avg_case_score regression_pass=$regression_pass baseline=${baseline_report:-none} report=$holdout_report" + + if [[ -z "$baseline_report" && "$holdout_rc" -eq 0 && -f "$holdout_report" ]]; then + set_holdout_baseline_report "$holdout_report" + log "holdout_baseline_set report=$holdout_report" + fi + return "$holdout_rc" +} + +evaluate_quality_gates_and_maybe_promote() { + local step="$1" + local quality_ok=1 + local -a reasons=() + + if [[ "$LAST_EVAL_RAN" -ne 1 ]]; then + quality_ok=0 + reasons+=("eval_not_run") + elif [[ "$LAST_EVAL_RC" -ne 0 ]]; then + quality_ok=0 + reasons+=("eval_failed") + fi + if [[ "$LAST_EVAL_REGRESSION_PASS" == "False" || "$LAST_EVAL_REGRESSION_PASS" == "0" ]]; then + quality_ok=0 + reasons+=("eval_regressed") + fi + if [[ "$PROMOTION_REQUIRE_POLICY_PASS" -eq 1 ]]; then + if ! is_truthy "$LAST_EVAL_PROMOTION_PASS"; then + quality_ok=0 + reasons+=("policy_not_promoted") + fi + fi + if [[ "$PROMOTION_REQUIRE_GENERATION_PASS" -eq 1 && "$GENERATION_GATE" -eq 1 ]]; then + if [[ "$LAST_GENERATION_GATE_RAN" -ne 1 ]]; then + quality_ok=0 + reasons+=("generation_not_run") + elif [[ "$LAST_GENERATION_GATE_RC" -ne 0 ]]; then + quality_ok=0 + reasons+=("generation_failed") + fi + fi + if [[ "$PROMOTION_REQUIRE_HOLDOUT_PASS" -eq 1 && "$HOLDOUT_GATE" -eq 1 ]]; then + if [[ "$LAST_HOLDOUT_GATE_RAN" -ne 1 ]]; then + quality_ok=0 + reasons+=("holdout_not_run") + elif [[ "$LAST_HOLDOUT_GATE_RC" -ne 0 ]]; then + quality_ok=0 + reasons+=("holdout_failed") + fi + fi + + if [[ "$quality_ok" -eq 1 ]]; then + QUALITY_PASS_STREAK=$((QUALITY_PASS_STREAK + 1)) + else + QUALITY_PASS_STREAK=0 + fi + + local reason_text="none" + if [[ "${#reasons[@]}" -gt 0 ]]; then + reason_text="$(IFS=,; echo "${reasons[*]}")" + fi + + log "quality_gate_result step=$step pass=$quality_ok streak=$QUALITY_PASS_STREAK required_streak=$PROMOTION_MIN_QUALITY_STREAK reasons=$reason_text" + + if [[ "$quality_ok" -ne 1 ]]; then + log "best_skip step=$step reason=quality_gate_failed reasons=$reason_text" + return 0 + fi + if [[ "$PROMOTION_MIN_QUALITY_STREAK" -gt 0 && "$QUALITY_PASS_STREAK" -lt "$PROMOTION_MIN_QUALITY_STREAK" ]]; then + log "best_skip step=$step reason=quality_streak_not_met streak=$QUALITY_PASS_STREAK required=$PROMOTION_MIN_QUALITY_STREAK" + return 0 + fi + + promote_best_checkpoint_if_needed "$step" "$LAST_EVAL_RC" "$LAST_EVAL_REPORT" +} + +quality_failure_tail_summary() { + "$PYTHON_BIN" - <<'PY' "$EVAL_TREND_TSV" "$GENERATION_TREND_TSV" "$HOLDOUT_TREND_TSV" +from pathlib import Path +import sys + +eval_path = Path(sys.argv[1]) +gen_path = Path(sys.argv[2]) +holdout_path = Path(sys.argv[3]) + +step_fail: dict[int, bool] = {} +step_sources: dict[int, set[str]] = {} + +def parse_rows(path: Path, source: str, rc_idx: int, regression_idx: int) -> None: + if not path.exists(): + return + for row in path.read_text(encoding="utf-8", errors="replace").splitlines(): + if not row or row.startswith("run_tag\t"): + continue + parts = row.split("\t") + if len(parts) <= max(rc_idx, regression_idx): + continue + step_raw = parts[1].strip() + if not step_raw.isdigit(): + continue + step = int(step_raw) + rc = parts[rc_idx].strip() + regression = parts[regression_idx].strip() if regression_idx >= 0 else "" + failed = rc != "0" or regression in {"False", "0"} + if not failed: + step_fail.setdefault(step, False) + continue + step_fail[step] = True + sources = step_sources.setdefault(step, set()) + sources.add(source) + +parse_rows(eval_path, "eval", rc_idx=2, regression_idx=8) +parse_rows(gen_path, "generation", rc_idx=2, regression_idx=8) +parse_rows(holdout_path, "holdout", rc_idx=2, regression_idx=8) + +if not step_fail: + print("0\t0\tnone") + raise SystemExit(0) + +ordered_steps = sorted(step_fail.keys()) +streak = 0 +for step in reversed(ordered_steps): + if step_fail.get(step): + streak += 1 + else: + break + +latest = ordered_steps[-1] +sources = ",".join(sorted(step_sources.get(latest, set()))) or "none" +print(f"{streak}\t{latest}\t{sources}") +PY +} + +best_checkpoint_step() { + if [[ -f "$BEST_META_JSON" ]]; then + local best_step + best_step="$("$PYTHON_BIN" - <<'PY' "$BEST_META_JSON" +import json +import sys +from pathlib import Path + +path = Path(sys.argv[1]) +try: + payload = json.loads(path.read_text(encoding="utf-8")) +except Exception: + print("") + raise SystemExit(0) +step = payload.get("best_step") +print(str(step) if isinstance(step, int) else "") +PY +)" + if [[ "$best_step" =~ ^[0-9]+$ ]]; then + echo "$best_step" + return 0 + fi + fi + if [[ -f "$OUTPUT_DIR/best.pt" ]]; then + checkpoint_step "$OUTPUT_DIR/best.pt" || echo "0" + return 0 + fi + echo "0" +} + +maybe_auto_rollback_on_quality_regression() { + local current_step="$1" + if [[ "$QUALITY_ROLLBACK_STREAK" -le 0 ]]; then + return 0 + fi + if [[ ! -f "$OUTPUT_DIR/best.pt" ]]; then + return 0 + fi + + local tail_streak=0 + local latest_step=0 + local latest_sources="none" + read -r tail_streak latest_step latest_sources < <(quality_failure_tail_summary) + if [[ "$tail_streak" -lt "$QUALITY_ROLLBACK_STREAK" ]]; then + return 0 + fi + + local best_step + best_step="$(best_checkpoint_step)" + if [[ -z "$best_step" || "$best_step" -le 0 ]]; then + log "quality_rollback_skip reason=missing_best_step streak=$tail_streak latest_step=$latest_step" + return 0 + fi + if [[ "$best_step" -ge "$current_step" ]]; then + log "quality_rollback_skip reason=best_not_older current_step=$current_step best_step=$best_step streak=$tail_streak" + return 0 + fi + + local last_rollback_step=-1 + if [[ -f "$LAST_QUALITY_ROLLBACK_STEP_FILE" ]]; then + local raw + raw="$(cat "$LAST_QUALITY_ROLLBACK_STEP_FILE" 2>/dev/null || true)" + if [[ "$raw" =~ ^[0-9]+$ ]]; then + last_rollback_step="$raw" + fi + fi + if [[ "$last_rollback_step" -ge 0 ]]; then + local delta=$((current_step - last_rollback_step)) + if [[ "$delta" -lt "$QUALITY_ROLLBACK_COOLDOWN_STEPS" ]]; then + log "quality_rollback_skip reason=cooldown current_step=$current_step last_rollback_step=$last_rollback_step cooldown_steps=$QUALITY_ROLLBACK_COOLDOWN_STEPS streak=$tail_streak" + return 0 + fi + fi + + local ts + ts="$(date +%Y%m%d_%H%M%S)" + if [[ -f "$OUTPUT_DIR/last.pt" ]]; then + cp -f "$OUTPUT_DIR/last.pt" "$OUTPUT_DIR/last_pre_quality_rollback_${ts}.pt" + fi + cp -f "$OUTPUT_DIR/best.pt" "$OUTPUT_DIR/last.pt" + if [[ -f "$OUTPUT_DIR/best.safetensors" ]]; then + cp -f "$OUTPUT_DIR/best.safetensors" "$OUTPUT_DIR/last.safetensors" + fi + if [[ -f "$OUTPUT_DIR/best_ema.safetensors" ]]; then + cp -f "$OUTPUT_DIR/best_ema.safetensors" "$OUTPUT_DIR/last_ema.safetensors" + fi + echo "$current_step" > "$LAST_QUALITY_ROLLBACK_STEP_FILE" + log "quality_rollback_applied current_step=$current_step best_step=$best_step streak=$tail_streak latest_quality_step=$latest_step latest_sources=$latest_sources cooldown_steps=$QUALITY_ROLLBACK_COOLDOWN_STEPS" +} + clamp_batch_size if [[ "$TARGET_EFFECTIVE_BATCH" -gt 0 ]]; then set_grad_accum_from_target @@ -1281,7 +1968,7 @@ fi failure_streak=0 successful_chunks=0 log "supervisor_start shards_path=$SHARDS_PATH output_dir=$OUTPUT_DIR step_chunk=$STEP_CHUNK" -log "tuning_start batch_size=$BATCH_SIZE grad_accum=$GRAD_ACCUM_STEPS auto_tune=$AUTO_TUNE target_effective_batch=$TARGET_EFFECTIVE_BATCH train_fail_on_eval_regression=$TRAIN_FAIL_ON_EVAL_REGRESSION checkpoint_keep_last=$CHECKPOINT_KEEP_LAST checkpoint_keep_every=$CHECKPOINT_KEEP_EVERY allow_context_extension=$ALLOW_CONTEXT_EXTENSION ema_decay=$EMA_DECAY ema_update_every=$EMA_UPDATE_EVERY ema_start_step=$EMA_START_STEP generation_gate=$GENERATION_GATE generation_every_chunks=$GENERATION_EVERY_CHUNKS generation_stop_on_fail=$GENERATION_STOP_ON_FAIL dedupe_overlap_manifests=$DEDUPE_OVERLAP_MANIFESTS dedupe_keep=$DEDUPE_KEEP dedupe_dry_run=$DEDUPE_DRY_RUN min_train_tokens=$MIN_TRAIN_TOKENS train_stall_check_seconds=$TRAIN_STALL_CHECK_SECONDS train_stall_kill_seconds=$TRAIN_STALL_KILL_SECONDS" +log "tuning_start batch_size=$BATCH_SIZE grad_accum=$GRAD_ACCUM_STEPS auto_tune=$AUTO_TUNE target_effective_batch=$TARGET_EFFECTIVE_BATCH train_fail_on_eval_regression=$TRAIN_FAIL_ON_EVAL_REGRESSION checkpoint_keep_last=$CHECKPOINT_KEEP_LAST checkpoint_keep_every=$CHECKPOINT_KEEP_EVERY allow_context_extension=$ALLOW_CONTEXT_EXTENSION learning_rate=$LEARNING_RATE lr_schedule=$LR_SCHEDULE lr_warmup_steps=$LR_WARMUP_STEPS lr_min_ratio=$LR_MIN_RATIO ema_decay=$EMA_DECAY ema_update_every=$EMA_UPDATE_EVERY ema_start_step=$EMA_START_STEP sampler_strategy=$SAMPLER_STRATEGY sampler_min_full_passes=$SAMPLER_MIN_FULL_PASSES hot_shard_warmup=$HOT_SHARD_WARMUP hot_shard_warmup_workers=$HOT_SHARD_WARMUP_WORKERS hot_shard_warmup_max_files=$HOT_SHARD_WARMUP_MAX_FILES hot_shard_warmup_background=$HOT_SHARD_WARMUP_BACKGROUND hot_shard_warmup_background_interval_seconds=$HOT_SHARD_WARMUP_BACKGROUND_INTERVAL_SECONDS hot_shard_warmup_background_max_files=$HOT_SHARD_WARMUP_BACKGROUND_MAX_FILES hot_shard_warmup_background_include_offloaded=$HOT_SHARD_WARMUP_BACKGROUND_INCLUDE_OFFLOADED generation_gate=$GENERATION_GATE generation_every_chunks=$GENERATION_EVERY_CHUNKS generation_stop_on_fail=$GENERATION_STOP_ON_FAIL holdout_gate=$HOLDOUT_GATE holdout_suite=${HOLDOUT_SUITE:-none} holdout_every_chunks=$HOLDOUT_EVERY_CHUNKS holdout_stop_on_fail=$HOLDOUT_STOP_ON_FAIL promotion_require_policy_pass=$PROMOTION_REQUIRE_POLICY_PASS promotion_require_generation_pass=$PROMOTION_REQUIRE_GENERATION_PASS promotion_require_holdout_pass=$PROMOTION_REQUIRE_HOLDOUT_PASS promotion_min_quality_streak=$PROMOTION_MIN_QUALITY_STREAK quality_rollback_streak=$QUALITY_ROLLBACK_STREAK quality_rollback_cooldown_steps=$QUALITY_ROLLBACK_COOLDOWN_STEPS dedupe_overlap_manifests=$DEDUPE_OVERLAP_MANIFESTS dedupe_keep=$DEDUPE_KEEP dedupe_dry_run=$DEDUPE_DRY_RUN min_train_tokens=$MIN_TRAIN_TOKENS train_stall_check_seconds=$TRAIN_STALL_CHECK_SECONDS train_stall_kill_seconds=$TRAIN_STALL_KILL_SECONDS" backfill_trained_batch_registry ensure_single_supervisor_process @@ -1307,6 +1994,11 @@ while true; do sleep "$POLL_SECONDS" continue fi + if ! run_hot_shard_warmup; then + log "waiting_for_hot_shard_warmup sleep=${POLL_SECONDS}s" + sleep "$POLL_SECONDS" + continue + fi resume_ckpt="$(select_resume_checkpoint)" if [[ -n "$resume_ckpt" ]]; then @@ -1319,6 +2011,8 @@ while true; do run_log="$STATE_DIR/train_${step_now}_to_${target_step}_${run_tag}.log" gpu_log="$STATE_DIR/gpu_${step_now}_to_${target_step}_${run_tag}.csv" chunk_batches_file="$STATE_DIR/chunk_batches_${step_now}_to_${target_step}_${run_tag}.txt" + sampled_shards_trace="$STATE_DIR/sampled_shards_${step_now}_to_${target_step}_${run_tag}.json" + sampled_batches_file="$STATE_DIR/sampled_batches_${step_now}_to_${target_step}_${run_tag}.txt" collect_manifest_batch_names > "$chunk_batches_file" chunk_batches_count="$(wc -l < "$chunk_batches_file" | tr -d ' ')" resume_args=() @@ -1326,7 +2020,7 @@ while true; do resume_args=(--resume-from "$resume_ckpt") fi - log "train_launch manifests=$mcount unique_inputs=$unique_inputs train_tokens=$train_tokens overlap_inputs=$overlap_inputs overlap_manifests=$overlap_manifests step_now=$step_now target_step=$target_step batch_size=$BATCH_SIZE grad_accum=$GRAD_ACCUM_STEPS resume=${resume_ckpt:-none} chunk_batches=$chunk_batches_count chunk_batches_file=$chunk_batches_file run_log=$run_log" + log "train_launch manifests=$mcount unique_inputs=$unique_inputs train_tokens=$train_tokens overlap_inputs=$overlap_inputs overlap_manifests=$overlap_manifests step_now=$step_now target_step=$target_step batch_size=$BATCH_SIZE grad_accum=$GRAD_ACCUM_STEPS sampler_strategy=$SAMPLER_STRATEGY sampler_min_full_passes=$SAMPLER_MIN_FULL_PASSES resume=${resume_ckpt:-none} chunk_batches=$chunk_batches_count chunk_batches_file=$chunk_batches_file sampled_trace=$sampled_shards_trace run_log=$run_log" train_gate_args=() if [[ "$TRAIN_FAIL_ON_EVAL_REGRESSION" -eq 1 ]]; then @@ -1354,7 +2048,7 @@ while true; do --n-heads "$N_HEADS" \ --d-model "$D_MODEL" \ --learning-rate "$LEARNING_RATE" \ - --lr-schedule cosine \ + --lr-schedule "$LR_SCHEDULE" \ --lr-warmup-steps "$LR_WARMUP_STEPS" \ --lr-min-ratio "$LR_MIN_RATIO" \ --eval-interval "$EVAL_INTERVAL" \ @@ -1364,6 +2058,10 @@ while true; do --precision "$PRECISION" \ --checkpoint-keep-last "$CHECKPOINT_KEEP_LAST" \ --checkpoint-keep-every "$CHECKPOINT_KEEP_EVERY" \ + --sampler-strategy "$SAMPLER_STRATEGY" \ + --sampler-min-full-passes "$SAMPLER_MIN_FULL_PASSES" \ + --sampled-shards-trace "$sampled_shards_trace" \ + --sampled-shards-trace-min-rows 1 \ --ema-decay "$EMA_DECAY" \ --ema-update-every "$EMA_UPDATE_EVERY" \ --ema-start-step "$EMA_START_STEP" \ @@ -1373,6 +2071,7 @@ while true; do train_pid=$! start_gpu_monitor "$train_pid" "$gpu_log" monitor_pid="${GPU_MONITOR_PID:-}" + start_hot_shard_warmup_background "$train_pid" wait_for_train_with_stall_guard "$train_pid" "$run_log" "$step_now" "$stall_flag_file" rc=$? stalled_chunk=0 @@ -1384,6 +2083,7 @@ while true; do kill "$monitor_pid" 2>/dev/null || true wait "$monitor_pid" 2>/dev/null || true fi + stop_hot_shard_warmup_background set -e new_resume_ckpt="$(select_resume_checkpoint)" @@ -1394,13 +2094,15 @@ while true; do fi best_val_ppl="$(best_val_ppl_from_log "$run_log")" read -r gpu_avg_util gpu_max_mem < <(gpu_summary "$gpu_log") - echo -e "${run_tag}\t${step_now}\t${target_step}\t${new_step}\t${rc}\t${mcount}\t${BATCH_SIZE}\t${GRAD_ACCUM_STEPS}\t${best_val_ppl}\t${gpu_avg_util}\t${gpu_max_mem}\t${run_log}" >> "$TRAIN_TREND_TSV" + sampled_batches_count="NA" if [[ "$rc" -eq 0 ]]; then failure_streak=0 successful_chunks=$((successful_chunks + 1)) - log "train_done rc=0 step_now=$new_step best_val_ppl=$best_val_ppl gpu_avg_util=$gpu_avg_util gpu_max_mem=$gpu_max_mem" - update_trained_batch_registry "$chunk_batches_file" "$new_step" + collect_sampled_batch_names "$sampled_shards_trace" > "$sampled_batches_file" + sampled_batches_count="$(wc -l < "$sampled_batches_file" | tr -d ' ')" + log "train_done rc=0 step_now=$new_step best_val_ppl=$best_val_ppl gpu_avg_util=$gpu_avg_util gpu_max_mem=$gpu_max_mem sampled_batches=$sampled_batches_count sampled_batches_file=$sampled_batches_file" + update_trained_batch_registry "$sampled_batches_file" "$new_step" "$chunk_batches_count" run_post_chunk_eval "$run_tag" "$new_step" if ! run_generation_gate "$run_tag" "$new_step" "$successful_chunks"; then log "generation_gate_failed step=$new_step successful_chunks=$successful_chunks" @@ -1409,10 +2111,20 @@ while true; do exit 11 fi fi + if ! run_holdout_gate "$run_tag" "$new_step" "$successful_chunks"; then + log "holdout_gate_failed step=$new_step successful_chunks=$successful_chunks" + if [[ "$HOLDOUT_STOP_ON_FAIL" -eq 1 ]]; then + log "supervisor_stop reason=holdout_gate_failed" + exit 12 + fi + fi + evaluate_quality_gates_and_maybe_promote "$new_step" + maybe_auto_rollback_on_quality_regression "$new_step" else if [[ -n "$resume_ckpt" ]] && log_has_resume_checkpoint_error "$run_log"; then quarantine_bad_checkpoint "$resume_ckpt" "resume_failure" fi + QUALITY_PASS_STREAK=0 failure_streak=$((failure_streak + 1)) if [[ "$stalled_chunk" -eq 1 ]]; then log "train_failed rc=$rc failure_streak=$failure_streak reason=stall_killed best_val_ppl=$best_val_ppl gpu_avg_util=$gpu_avg_util gpu_max_mem=$gpu_max_mem" @@ -1423,8 +2135,10 @@ while true; do log "supervisor_stop reason=max_failure_streak_reached" exit 10 fi + maybe_auto_rollback_on_quality_regression "$new_step" fi auto_tune_after_chunk "$rc" "$run_log" "$gpu_avg_util" "$gpu_max_mem" + echo -e "${run_tag}\t${step_now}\t${target_step}\t${new_step}\t${rc}\t${mcount}\t${BATCH_SIZE}\t${GRAD_ACCUM_STEPS}\t${best_val_ppl}\t${gpu_avg_util}\t${gpu_max_mem}\t${sampled_batches_count}\t${sampled_shards_trace}\t${run_log}" >> "$TRAIN_TREND_TSV" sleep "$POLL_SECONDS" done diff --git a/src/llm/cli.py b/src/llm/cli.py index 71813cf..3578843 100644 --- a/src/llm/cli.py +++ b/src/llm/cli.py @@ -569,6 +569,12 @@ def cmd_train( tf32: bool, compile_model: bool, compile_mode: str, + compile_strict: bool, + sampler_max_open_shards: int, + sampler_strategy: str, + sampler_min_full_passes: int, + sampled_shards_trace: str | None, + sampled_shards_trace_min_rows: int, export_safetensors: bool, safetensors_every_checkpoint: bool, checkpoint_keep_last: int, @@ -615,6 +621,12 @@ def cmd_train( tf32=tf32, compile_model=compile_model, compile_mode=compile_mode, + compile_strict=compile_strict, + sampler_max_open_shards=sampler_max_open_shards, + sampler_strategy=sampler_strategy, + sampler_min_full_passes=sampler_min_full_passes, + sampled_shards_trace=(Path(sampled_shards_trace) if sampled_shards_trace else None), + sampled_shards_trace_min_rows=sampled_shards_trace_min_rows, export_safetensors=export_safetensors, safetensors_every_checkpoint=safetensors_every_checkpoint, checkpoint_keep_last=checkpoint_keep_last, @@ -1341,6 +1353,46 @@ def parse_args() -> argparse.Namespace: default="reduce-overhead", help="torch.compile mode (used only with --compile-model)", ) + train_parser.add_argument( + "--compile-strict", + action="store_true", + help="Fail immediately if torch.compile init/warmup fails (default falls back to eager)", + ) + train_parser.add_argument( + "--sampler-max-open-shards", + type=int, + default=256, + help="Max open shard memmaps per train/val sampler to limit file descriptors", + ) + train_parser.add_argument( + "--sampler-strategy", + choices=["weighted", "balanced"], + default="weighted", + help=( + "Shard sampling strategy: weighted (token-proportional random) or " + "balanced (shuffled full-cycle over shards)" + ), + ) + train_parser.add_argument( + "--sampler-min-full-passes", + type=int, + default=0, + help=( + "When > 0 with --sampler-strategy balanced, require this many guaranteed full " + "passes over all eligible train shards in the configured run window" + ), + ) + train_parser.add_argument( + "--sampled-shards-trace", + default=None, + help="Optional JSON path to write sampled shard coverage trace for this run", + ) + train_parser.add_argument( + "--sampled-shards-trace-min-rows", + type=int, + default=1, + help="Minimum sampled rows per shard to include in sampled shard trace", + ) train_parser.add_argument( "--resume-from", default=None, @@ -1635,6 +1687,12 @@ def main() -> int: tf32=not args.no_tf32, compile_model=args.compile_model, compile_mode=args.compile_mode, + compile_strict=args.compile_strict, + sampler_max_open_shards=args.sampler_max_open_shards, + sampler_strategy=args.sampler_strategy, + sampler_min_full_passes=args.sampler_min_full_passes, + sampled_shards_trace=args.sampled_shards_trace, + sampled_shards_trace_min_rows=args.sampled_shards_trace_min_rows, export_safetensors=args.export_safetensors, safetensors_every_checkpoint=args.safetensors_every_checkpoint, checkpoint_keep_last=args.checkpoint_keep_last, diff --git a/src/llm/train.py b/src/llm/train.py index 6f3fb70..475f328 100644 --- a/src/llm/train.py +++ b/src/llm/train.py @@ -6,6 +6,7 @@ import math import random import time +from collections import OrderedDict from dataclasses import asdict, dataclass from pathlib import Path from typing import Any, cast @@ -56,6 +57,12 @@ class TrainConfig: tf32: bool = True compile_model: bool = False compile_mode: str = "reduce-overhead" + compile_strict: bool = False + sampler_max_open_shards: int = 256 + sampler_strategy: str = "weighted" + sampler_min_full_passes: int = 0 + sampled_shards_trace: Path | None = None + sampled_shards_trace_min_rows: int = 1 export_safetensors: bool = False safetensors_every_checkpoint: bool = False checkpoint_keep_last: int = 0 @@ -69,6 +76,9 @@ def to_dict(self) -> dict[str, Any]: payload["shards_path"] = str(self.shards_path) payload["output_dir"] = str(self.output_dir) payload["resume_from"] = str(self.resume_from) if self.resume_from else None + payload["sampled_shards_trace"] = ( + str(self.sampled_shards_trace) if self.sampled_shards_trace else None + ) return payload @@ -145,10 +155,7 @@ def _build_optimizer( continue lowered = name.lower() no_decay = ( - param.ndim < 2 - or lowered.endswith("bias") - or "norm" in lowered - or "embed" in lowered + param.ndim < 2 or lowered.endswith("bias") or "norm" in lowered or "embed" in lowered ) if no_decay: no_decay_params.append(param) @@ -297,43 +304,135 @@ def __init__( context_length: int, seed: int, device: torch.device, + max_open_shards: int = 256, + sampling_strategy: str = "weighted", ) -> None: if context_length <= 0: raise ValueError("context_length must be > 0") + if max_open_shards <= 0: + raise ValueError("max_open_shards must be > 0") self.context_length = context_length self.np_rng = np.random.default_rng(seed) self.device = device - dtype = _np_dtype(token_dtype) - self.arrays = [np.memmap(path, mode="r", dtype=dtype) for path in shard_paths] + self.dtype = _np_dtype(token_dtype) + self.max_open_shards = max_open_shards + normalized_strategy = sampling_strategy.strip().lower() + if normalized_strategy not in {"weighted", "balanced"}: + raise ValueError("sampling_strategy must be one of: weighted, balanced") + self.sampling_strategy = normalized_strategy + self.shard_paths = [path.resolve() for path in shard_paths] + self._array_cache: OrderedDict[int, np.memmap[Any, Any]] = OrderedDict() + self._sampled_rows: dict[int, int] = {} + self._elem_size = self.dtype.itemsize + self.shard_sizes: list[int] = [] + for path in self.shard_paths: + size_bytes = path.stat().st_size + if size_bytes % self._elem_size != 0: + raise ValueError( + f"shard byte size is not aligned to dtype element size: {path} " + f"(size={size_bytes}, itemsize={self._elem_size})" + ) + self.shard_sizes.append(size_bytes // self._elem_size) self.eligible: list[int] = [] self.weights: list[int] = [] - for idx, arr in enumerate(self.arrays): - if int(arr.size) > context_length: + for idx, size in enumerate(self.shard_sizes): + if int(size) > context_length: self.eligible.append(idx) - self.weights.append(int(arr.size) - context_length) + self.weights.append(int(size) - context_length) if not self.eligible: raise ValueError("no shards have enough tokens for context_length") self._eligible_np = np.asarray(self.eligible, dtype=np.int32) probs = np.asarray(self.weights, dtype=np.float64) self._weight_probs = probs / probs.sum() self._offsets = np.arange(self.context_length, dtype=np.int64) + self._cycle_order = np.empty(0, dtype=np.int32) + self._cycle_pos = 0 + + def _get_array(self, idx: int) -> np.memmap[Any, Any]: + cached = self._array_cache.get(idx) + if cached is not None: + self._array_cache.move_to_end(idx) + return cached + + if len(self._array_cache) >= self.max_open_shards: + self._array_cache.popitem(last=False) + arr = np.memmap(self.shard_paths[idx], mode="r", dtype=self.dtype) + self._array_cache[idx] = arr + return arr + + def sampled_shard_rows(self, *, min_rows: int = 1) -> list[tuple[Path, int]]: + if min_rows <= 0: + min_rows = 1 + rows: list[tuple[Path, int]] = [] + for idx, count in sorted(self._sampled_rows.items()): + if count >= min_rows: + rows.append((self.shard_paths[idx], count)) + return rows + + def eligible_shard_count(self) -> int: + return len(self.eligible) + + def write_sample_trace(self, path: Path, *, min_rows: int = 1) -> int: + sampled = self.sampled_shard_rows(min_rows=min_rows) + payload = { + "context_length": self.context_length, + "sampling_strategy": self.sampling_strategy, + "sampled_shards": [ + { + "path": str(shard_path), + "rows": rows, + "tokens_windowed": rows * self.context_length, + } + for shard_path, rows in sampled + ], + } + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, indent=2), encoding="utf-8") + return len(sampled) + + def _sample_shard_indices(self, batch_size: int) -> np.ndarray[Any, Any]: + if self.sampling_strategy == "weighted": + return self.np_rng.choice(self._eligible_np, size=batch_size, p=self._weight_probs) + + chosen = np.empty(batch_size, dtype=np.int32) + written = 0 + while written < batch_size: + if self._cycle_pos >= int(self._cycle_order.size): + self._cycle_order = self.np_rng.permutation(self._eligible_np) + self._cycle_pos = 0 + remaining_cycle = int(self._cycle_order.size) - self._cycle_pos + take = min(batch_size - written, remaining_cycle) + chosen[written : written + take] = self._cycle_order[ + self._cycle_pos : self._cycle_pos + take + ] + self._cycle_pos += take + written += take + return chosen + + def close(self) -> None: + self._array_cache.clear() + + def __del__(self) -> None: + self.close() def sample_batch(self, batch_size: int) -> tuple[Tensor, Tensor]: if batch_size <= 0: raise ValueError("batch_size must be > 0") x = np.zeros((batch_size, self.context_length), dtype=np.int64) y = np.zeros((batch_size, self.context_length), dtype=np.int64) - chosen = self.np_rng.choice(self._eligible_np, size=batch_size, p=self._weight_probs) + chosen = self._sample_shard_indices(batch_size) for arr_idx in np.unique(chosen): rows = np.nonzero(chosen == arr_idx)[0] - arr = self.arrays[int(arr_idx)] - max_start = int(arr.size) - self.context_length - 1 + idx = int(arr_idx) + arr = self._get_array(idx) + max_start = int(self.shard_sizes[idx]) - self.context_length - 1 if max_start <= 0: raise ValueError("encountered shard smaller than context window") starts = self.np_rng.integers(0, max_start + 1, size=rows.size, dtype=np.int64) gather_idx = starts[:, None] + self._offsets[None, :] x[rows, :] = arr[gather_idx] y[rows, :] = arr[gather_idx + 1] + self._sampled_rows[idx] = self._sampled_rows.get(idx, 0) + int(rows.size) xb = torch.from_numpy(x) yb = torch.from_numpy(y) @@ -393,6 +492,56 @@ def _resolve_amp_mode( return True, torch.float16, True, "fp16" +def _compile_training_model( + model: GPTModel, + *, + enable_compile: bool, + compile_mode: str, + compile_strict: bool, + context_length: int, + device: torch.device, +) -> tuple[torch.nn.Module, bool, str]: + if not enable_compile: + return model, False, "disabled" + if not hasattr(torch, "compile"): + if compile_strict: + raise ValueError("compile_model requested but torch.compile is unavailable") + print("compile_warning=torch.compile unavailable; falling_back_to_eager=1") + return model, False, "unavailable" + + try: + compiled = cast(torch.nn.Module, torch.compile(model, mode=compile_mode)) + except Exception as exc: # pragma: no cover - backend-dependent failure path + if compile_strict: + raise RuntimeError( + f"torch.compile failed in strict mode (mode={compile_mode}): {exc}" + ) from exc + print( + "compile_warning=torch.compile init failed; " + f"detail={type(exc).__name__}:{str(exc).strip()} " + "falling_back_to_eager=1" + ) + return model, False, "init_failed" + + try: + # Trigger compile graph build early so unstable backends fail before long runs. + dummy = torch.zeros((1, context_length), dtype=torch.long, device=device) + with torch.no_grad(): + compiled(dummy, dummy) + except Exception as exc: # pragma: no cover - backend-dependent failure path + if compile_strict: + raise RuntimeError( + f"torch.compile warmup failed in strict mode (mode={compile_mode}): {exc}" + ) from exc + print( + "compile_warning=torch.compile warmup failed; " + f"detail={type(exc).__name__}:{str(exc).strip()} " + "falling_back_to_eager=1" + ) + return model, False, "warmup_failed" + return compiled, True, "ok" + + def _estimate_loss( model: torch.nn.Module, sampler: ShardBatchSampler | None, @@ -738,11 +887,15 @@ def run_training(config: TrainConfig) -> dict[str, Any]: if config.ema_decay > 0.0 and ema_state is None: ema_state = _init_ema_state(model) - train_model: torch.nn.Module = model - if config.compile_model: - if not hasattr(torch, "compile"): - raise ValueError("compile_model requested but torch.compile is unavailable") - train_model = cast(torch.nn.Module, torch.compile(model, mode=config.compile_mode)) + if config.sampler_max_open_shards <= 0: + raise ValueError("sampler_max_open_shards must be > 0") + normalized_sampler_strategy = config.sampler_strategy.strip().lower() + if normalized_sampler_strategy not in {"weighted", "balanced"}: + raise ValueError("sampler_strategy must be one of: weighted, balanced") + if config.sampler_min_full_passes < 0: + raise ValueError("sampler_min_full_passes must be >= 0") + if config.sampled_shards_trace_min_rows <= 0: + raise ValueError("sampled_shards_trace_min_rows must be > 0") train_sampler = ShardBatchSampler( shard_paths=info.train_shards, @@ -750,6 +903,8 @@ def run_training(config: TrainConfig) -> dict[str, Any]: context_length=config.context_length, seed=config.seed, device=device, + max_open_shards=config.sampler_max_open_shards, + sampling_strategy=normalized_sampler_strategy, ) val_sampler = ShardBatchSampler( shard_paths=info.val_shards, @@ -757,13 +912,42 @@ def run_training(config: TrainConfig) -> dict[str, Any]: context_length=config.context_length, seed=config.seed + 1, device=device, + max_open_shards=config.sampler_max_open_shards, + sampling_strategy="weighted", + ) + planned_steps = max(0, config.max_steps - start_step) + planned_sample_rows = planned_steps * config.grad_accum_steps * config.batch_size + train_eligible_shards = train_sampler.eligible_shard_count() + train_guaranteed_full_passes = ( + planned_sample_rows // train_eligible_shards if train_eligible_shards > 0 else 0 ) + if ( + normalized_sampler_strategy == "balanced" + and config.sampler_min_full_passes > 0 + and train_guaranteed_full_passes < config.sampler_min_full_passes + ): + raise ValueError( + "sampler_min_full_passes cannot be met by this run configuration: " + f"requested={config.sampler_min_full_passes} " + f"guaranteed={train_guaranteed_full_passes} " + f"planned_sample_rows={planned_sample_rows} " + f"eligible_shards={train_eligible_shards}" + ) frozen_eval_batches: list[tuple[Tensor, Tensor]] | None = None if config.eval_freeze_batches: frozen_eval_batches = [ val_sampler.sample_batch(config.batch_size) for _ in range(config.eval_steps) ] + train_model, compile_active, compile_state = _compile_training_model( + model, + enable_compile=config.compile_model, + compile_mode=config.compile_mode, + compile_strict=config.compile_strict, + context_length=config.context_length, + device=device, + ) + config.output_dir.mkdir(parents=True, exist_ok=True) (config.output_dir / "run_config.json").write_text( json.dumps( @@ -796,7 +980,15 @@ def run_training(config: TrainConfig) -> dict[str, Any]: print(f"optimizer_decay_tensors={optimizer_meta['decay_tensors']}") print(f"optimizer_no_decay_tensors={optimizer_meta['no_decay_tensors']}") print(f"tf32={int(config.tf32)}") - print(f"compile_model={int(config.compile_model)}") + print(f"compile_model_requested={int(config.compile_model)}") + print(f"compile_model_active={int(compile_active)}") + print(f"compile_state={compile_state}") + print(f"sampler_max_open_shards={config.sampler_max_open_shards}") + print(f"sampler_strategy={normalized_sampler_strategy}") + print(f"sampler_min_full_passes={config.sampler_min_full_passes}") + print(f"train_sampler_eligible_shards={train_eligible_shards}") + print(f"train_sampler_planned_sample_rows={planned_sample_rows}") + print(f"train_sampler_guaranteed_full_passes={train_guaranteed_full_passes}") print(f"eval_freeze_batches={int(config.eval_freeze_batches)}") print(f"fail_on_eval_regression={int(config.fail_on_eval_regression)}") if config.compile_model: @@ -821,107 +1013,119 @@ def run_training(config: TrainConfig) -> dict[str, Any]: tokens_seen = 0 log_tokens_mark = 0 log_time_mark = time.perf_counter() - for step in range(start_step + 1, config.max_steps + 1): - optimizer.zero_grad(set_to_none=True) - lr = _lr_for_step( - step=step, - max_steps=config.max_steps, - base_lr=config.learning_rate, - schedule=config.lr_schedule, - warmup_steps=config.lr_warmup_steps, - min_ratio=config.lr_min_ratio, - ) - _set_optimizer_lr(optimizer, lr) - micro_losses: list[float] = [] - for _ in range(config.grad_accum_steps): - xb, yb = train_sampler.sample_batch(config.batch_size) - tokens_seen += int(xb.numel()) - with torch.autocast(**autocast_kwargs): - _, loss = train_model(xb, yb) - if loss is None: - raise RuntimeError("loss should not be None when targets are provided") - micro_losses.append(float(loss.item())) - loss_to_backward = loss / config.grad_accum_steps + try: + for step in range(start_step + 1, config.max_steps + 1): + optimizer.zero_grad(set_to_none=True) + lr = _lr_for_step( + step=step, + max_steps=config.max_steps, + base_lr=config.learning_rate, + schedule=config.lr_schedule, + warmup_steps=config.lr_warmup_steps, + min_ratio=config.lr_min_ratio, + ) + _set_optimizer_lr(optimizer, lr) + micro_losses: list[float] = [] + for _ in range(config.grad_accum_steps): + xb, yb = train_sampler.sample_batch(config.batch_size) + tokens_seen += int(xb.numel()) + with torch.autocast(**autocast_kwargs): + _, loss = train_model(xb, yb) + if loss is None: + raise RuntimeError("loss should not be None when targets are provided") + micro_losses.append(float(loss.item())) + loss_to_backward = loss / config.grad_accum_steps + if scaler.is_enabled(): + scaler.scale(loss_to_backward).backward() + else: + loss_to_backward.backward() + if scaler.is_enabled(): - scaler.scale(loss_to_backward).backward() + scaler.unscale_(optimizer) + torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=config.grad_clip) + scaler.step(optimizer) + scaler.update() else: - loss_to_backward.backward() + torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=config.grad_clip) + optimizer.step() - if scaler.is_enabled(): - scaler.unscale_(optimizer) - torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=config.grad_clip) - scaler.step(optimizer) - scaler.update() - else: - torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=config.grad_clip) - optimizer.step() - - if ( - ema_state is not None - and step >= config.ema_start_step - and (step % config.ema_update_every) == 0 - ): - _update_ema_state(ema_state, model, config.ema_decay) - - train_loss = sum(micro_losses) / len(micro_losses) - if step == 1 or step % config.log_interval == 0: - now = time.perf_counter() - elapsed = max(now - log_time_mark, 1e-9) - tok_window = tokens_seen - log_tokens_mark - toks_per_sec = tok_window / elapsed - print( - f"step={step} train_loss={train_loss:.6f} lr={lr:.8f} " - f"tokens_seen={tokens_seen} toks_per_sec={toks_per_sec:.2f}" - ) - log_tokens_mark = tokens_seen - log_time_mark = now - - should_eval = step == 1 or step % config.eval_interval == 0 or step == config.max_steps - if should_eval: - val_loss = _estimate_loss( - train_model, - val_sampler if frozen_eval_batches is None else None, - batch_size=config.batch_size, - eval_steps=config.eval_steps, - amp_enabled=amp_enabled, - amp_dtype=amp_dtype, - device_type=device.type, - fixed_batches=frozen_eval_batches, - ) - val_ppl = float(math.exp(min(val_loss, 20.0))) - prev_best_val_ppl = best_val_ppl - if best_val_loss is None or val_loss < best_val_loss: - best_val_loss = val_loss - if best_val_ppl is None or val_ppl < best_val_ppl: - best_val_ppl = val_ppl - ckpt = _save_checkpoint( - output_dir=config.output_dir, - step=step, - model=model, - optimizer=optimizer, - scaler=scaler, - config=config, - model_config=model_config, - info=info, - lr=lr, - best_val_loss=best_val_loss, - best_val_ppl=best_val_ppl, - ema_state=ema_state, - ) - print( - f"step={step} val_loss={val_loss:.6f} val_ppl={val_ppl:.4f} " - f"best_val_ppl={best_val_ppl:.4f} checkpoint={ckpt}" - ) if ( - config.fail_on_eval_regression - and prev_best_val_ppl is not None - and val_ppl > (prev_best_val_ppl * (1.0 + config.eval_regression_tolerance)) + ema_state is not None + and step >= config.ema_start_step + and (step % config.ema_update_every) == 0 ): - threshold = prev_best_val_ppl * (1.0 + config.eval_regression_tolerance) - raise RuntimeError( - "held-out eval regression gate failed: " - f"val_ppl={val_ppl:.4f} exceeded threshold={threshold:.4f}" + _update_ema_state(ema_state, model, config.ema_decay) + + train_loss = sum(micro_losses) / len(micro_losses) + if step == 1 or step % config.log_interval == 0: + now = time.perf_counter() + elapsed = max(now - log_time_mark, 1e-9) + tok_window = tokens_seen - log_tokens_mark + toks_per_sec = tok_window / elapsed + print( + f"step={step} train_loss={train_loss:.6f} lr={lr:.8f} " + f"tokens_seen={tokens_seen} toks_per_sec={toks_per_sec:.2f}" + ) + log_tokens_mark = tokens_seen + log_time_mark = now + + should_eval = step == 1 or step % config.eval_interval == 0 or step == config.max_steps + if should_eval: + val_loss = _estimate_loss( + train_model, + val_sampler if frozen_eval_batches is None else None, + batch_size=config.batch_size, + eval_steps=config.eval_steps, + amp_enabled=amp_enabled, + amp_dtype=amp_dtype, + device_type=device.type, + fixed_batches=frozen_eval_batches, + ) + val_ppl = float(math.exp(min(val_loss, 20.0))) + prev_best_val_ppl = best_val_ppl + if best_val_loss is None or val_loss < best_val_loss: + best_val_loss = val_loss + if best_val_ppl is None or val_ppl < best_val_ppl: + best_val_ppl = val_ppl + ckpt = _save_checkpoint( + output_dir=config.output_dir, + step=step, + model=model, + optimizer=optimizer, + scaler=scaler, + config=config, + model_config=model_config, + info=info, + lr=lr, + best_val_loss=best_val_loss, + best_val_ppl=best_val_ppl, + ema_state=ema_state, + ) + print( + f"step={step} val_loss={val_loss:.6f} val_ppl={val_ppl:.4f} " + f"best_val_ppl={best_val_ppl:.4f} checkpoint={ckpt}" + ) + if ( + config.fail_on_eval_regression + and prev_best_val_ppl is not None + and val_ppl > (prev_best_val_ppl * (1.0 + config.eval_regression_tolerance)) + ): + threshold = prev_best_val_ppl * (1.0 + config.eval_regression_tolerance) + raise RuntimeError( + "held-out eval regression gate failed: " + f"val_ppl={val_ppl:.4f} exceeded threshold={threshold:.4f}" + ) + finally: + if config.sampled_shards_trace is not None: + try: + trace_count = train_sampler.write_sample_trace( + config.sampled_shards_trace, + min_rows=config.sampled_shards_trace_min_rows, ) + print(f"sampled_shards_trace={config.sampled_shards_trace}") + print(f"sampled_shards_count={trace_count}") + except OSError as exc: + print(f"sampled_shards_trace_write_error={exc}") return { "output_dir": str(config.output_dir), diff --git a/tests/test_eval_suites.py b/tests/test_eval_suites.py index a04274d..cb1ae78 100644 --- a/tests/test_eval_suites.py +++ b/tests/test_eval_suites.py @@ -15,8 +15,16 @@ class EvalSuiteConfigTests(unittest.TestCase): + def _case_suite_paths(self) -> list[Path]: + paths: list[Path] = [] + for path in sorted(EVAL_DIR.glob("*.json")): + payload = json.loads(path.read_text(encoding="utf-8")) + if isinstance(payload.get("cases"), list): + paths.append(path) + return paths + def test_eval_suites_use_supported_check_keys(self) -> None: - for suite_path in sorted(EVAL_DIR.glob("*suite*.json")): + for suite_path in self._case_suite_paths(): suite = json.loads(suite_path.read_text(encoding="utf-8")) self.assertIsInstance(suite.get("name"), str, msg=str(suite_path)) cases = suite.get("cases") @@ -41,13 +49,24 @@ def test_phase1_talk_wrapper_uses_talk_configs(self) -> None: encoding="utf-8" ) self.assertIn("configs/eval/english_talk_suite_v1.json", wrapper) - self.assertIn("configs/eval/generation_talk_smoke_v1.json", wrapper) - self.assertIn("configs/eval/promotion_policy_talk_v1.json", wrapper) + self.assertIn("configs/eval/generation_talk_quality_v2.json", wrapper) + self.assertIn("configs/eval/english_talk_holdout_suite_v1.json", wrapper) + self.assertIn("configs/eval/promotion_policy_talk_recovery_v2.json", wrapper) - def test_talk_promotion_policy_schema(self) -> None: - policy_path = EVAL_DIR / "promotion_policy_talk_v1.json" - policy = json.loads(policy_path.read_text(encoding="utf-8")) - for section in ("absolute", "regression", "improvement"): - self.assertIn(section, policy, msg=str(policy_path)) - self.assertIsInstance(policy[section], dict, msg=str(policy_path)) + def test_phase1_holdout_suite_is_non_coding(self) -> None: + suite_path = EVAL_DIR / "english_talk_holdout_suite_v1.json" + suite = json.loads(suite_path.read_text(encoding="utf-8")) + categories = {str(case.get("category", "")).lower() for case in suite.get("cases", [])} + self.assertNotIn("coding", categories) + def test_talk_promotion_policy_schema(self) -> None: + policy_files = ( + "promotion_policy_talk_v1.json", + "promotion_policy_talk_recovery_v2.json", + ) + for policy_name in policy_files: + policy_path = EVAL_DIR / policy_name + policy = json.loads(policy_path.read_text(encoding="utf-8")) + for section in ("absolute", "regression", "improvement"): + self.assertIn(section, policy, msg=str(policy_path)) + self.assertIsInstance(policy[section], dict, msg=str(policy_path)) diff --git a/tests/test_ops_scripts.py b/tests/test_ops_scripts.py index 46b1643..53e9ba1 100644 --- a/tests/test_ops_scripts.py +++ b/tests/test_ops_scripts.py @@ -65,6 +65,94 @@ def test_train_supervisor_help_lists_unique_input_gate(self) -> None: self.assertIn("--min-train-tokens", proc.stdout) self.assertIn("--train-stall-kill-seconds", proc.stdout) self.assertIn("--dedupe-report-keep", proc.stdout) + self.assertIn("--quality-rollback-streak", proc.stdout) + self.assertIn("--quality-rollback-cooldown-steps", proc.stdout) + self.assertIn("--holdout-suite", proc.stdout) + self.assertIn("--holdout-every-chunks", proc.stdout) + self.assertIn("--promotion-min-quality-streak", proc.stdout) + self.assertIn("--sampler-strategy", proc.stdout) + self.assertIn("--sampler-min-full-passes", proc.stdout) + self.assertIn("--hot-shard-warmup-workers", proc.stdout) + self.assertIn("--warm-shards-root", proc.stdout) + self.assertIn("--no-hot-shard-warmup-background", proc.stdout) + self.assertIn("--hot-shard-warmup-background-interval-seconds", proc.stdout) + + def test_cli_train_help_lists_sampler_and_compile_safety_flags(self) -> None: + proc = subprocess.run( + [sys.executable, "-m", "llm.cli", "train", "--help"], + cwd=Path(__file__).resolve().parents[1], + env={**os.environ, "PYTHONPATH": "src"}, + capture_output=True, + text=True, + check=False, + ) + self.assertEqual(proc.returncode, 0, msg=proc.stderr) + self.assertIn("--sampler-max-open-shards", proc.stdout) + self.assertIn("--sampler-strategy", proc.stdout) + self.assertIn("--sampler-min-full-passes", proc.stdout) + self.assertIn("--sampled-shards-trace", proc.stdout) + self.assertIn("--compile-strict", proc.stdout) + + def test_hot_shard_warmup_hydrates_missing_and_symlinked_files(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + shards_root = root / "shards" + warm_root = root / "warm" + batch = "batch_0001" + local_batch = shards_root / batch + warm_batch = warm_root / batch + local_batch.mkdir(parents=True, exist_ok=True) + warm_batch.mkdir(parents=True, exist_ok=True) + + warm_train = warm_batch / "train_000000.bin" + warm_val = warm_batch / "val_000000.bin" + warm_train.write_bytes(b"train-bytes") + warm_val.write_bytes(b"val-bytes") + + # train shard missing locally, val shard present as symlink. + local_val = local_batch / "val_000000.bin" + local_val.symlink_to(warm_val.resolve()) + + manifest = { + "train": {"shards": [{"path": "train_000000.bin"}]}, + "val": {"shards": [{"path": "val_000000.bin"}]}, + } + (local_batch / "manifest.json").write_text(json.dumps(manifest), encoding="utf-8") + report = root / "report.json" + + proc = subprocess.run( + [ + sys.executable, + "scripts/hot_shard_warmup.py", + "--shards-root", + str(shards_root), + "--warm-shards-root", + str(warm_root), + "--report-output", + str(report), + "--workers", + "2", + ], + cwd=Path(__file__).resolve().parents[1], + capture_output=True, + text=True, + check=False, + ) + self.assertEqual(proc.returncode, 0, msg=proc.stderr) + self.assertIn("hot_shard_warmup_done", proc.stdout) + + local_train = local_batch / "train_000000.bin" + self.assertTrue(local_train.exists()) + self.assertFalse(local_train.is_symlink()) + self.assertEqual(local_train.read_bytes(), b"train-bytes") + self.assertTrue(local_val.exists()) + self.assertFalse(local_val.is_symlink()) + self.assertEqual(local_val.read_bytes(), b"val-bytes") + + payload = json.loads(report.read_text(encoding="utf-8")) + summary = payload["summary"] + self.assertEqual(summary["hydrated_files"], 2) + self.assertEqual(summary["missing_warm_files"], 0) def test_stage_loop_help_lists_stage_copy_jobs(self) -> None: proc = subprocess.run( @@ -79,8 +167,22 @@ def test_stage_loop_help_lists_stage_copy_jobs(self) -> None: self.assertIn("--stage-min-free-gib", proc.stdout) self.assertIn("--auto-tune-shard-jobs", proc.stdout) self.assertIn("--no-auto-tune-stage-copy-jobs", proc.stdout) + self.assertIn("--expected-unique-input-files", proc.stdout) self.assertIn("--sync-background", proc.stdout) + def test_benchmark_ctx_profiles_help_lists_profile_option(self) -> None: + proc = subprocess.run( + ["bash", "scripts/benchmark_rtx5070_context_profiles.sh", "--help"], + cwd=Path(__file__).resolve().parents[1], + capture_output=True, + text=True, + check=False, + ) + self.assertEqual(proc.returncode, 0, msg=proc.stderr) + self.assertIn("--profiles", proc.stdout) + self.assertIn("--compile-model", proc.stdout) + self.assertIn("--sample-seconds", proc.stdout) + def test_install_user_systemd_services_help(self) -> None: proc = subprocess.run( ["bash", "scripts/install_user_systemd_services.sh", "--help"], @@ -103,6 +205,7 @@ def test_stage_watchdog_help_lists_cleanup_option(self) -> None: ) self.assertEqual(proc.returncode, 0, msg=proc.stderr) self.assertIn("--no-cleanup-stale-workers", proc.stdout) + self.assertIn("--expected-unique-input-files", proc.stdout) self.assertIn("--lock-file", proc.stdout) self.assertIn("--no-adopt-existing-loop", proc.stdout) @@ -236,18 +339,6 @@ def test_checkpoint_offload_prune_help(self) -> None: self.assertIn("--keep-local-runs", proc.stdout) self.assertIn("--sync-only", proc.stdout) - def test_fineweb_prefetch_help_lists_auto_skip_options(self) -> None: - proc = subprocess.run( - ["bash", "scripts/fineweb_prefetch_hot_queue.sh", "--help"], - cwd=Path(__file__).resolve().parents[1], - capture_output=True, - text=True, - check=False, - ) - self.assertEqual(proc.returncode, 0, msg=proc.stderr) - self.assertIn("--auto-skip-state-dir", proc.stdout) - self.assertIn("--no-auto-skip", proc.stdout) - def test_revalidate_bad_parquet_help(self) -> None: proc = subprocess.run( [sys.executable, "scripts/revalidate_bad_parquet.py", "--help"], @@ -260,6 +351,79 @@ def test_revalidate_bad_parquet_help(self) -> None: self.assertIn("--restage-valid", proc.stdout) self.assertIn("--no-rewrite-bad-list", proc.stdout) + def test_revalidate_bad_parquet_max_entries_keeps_untouched(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + bad_list = root / "bad_parquet_files.txt" + warm_dir = root / "warm" + report = root / "report.json" + bad_list.write_text( + "000_00001.parquet\n000_00002.parquet\n000_00003.parquet\n", + encoding="utf-8", + ) + warm_dir.mkdir(parents=True, exist_ok=True) + + proc = subprocess.run( + [ + sys.executable, + "scripts/revalidate_bad_parquet.py", + "--bad-list", + str(bad_list), + "--warm-dir", + str(warm_dir), + "--report-output", + str(report), + "--max-entries", + "2", + "--workers", + "2", + ], + cwd=Path(__file__).resolve().parents[1], + capture_output=True, + text=True, + check=False, + ) + self.assertEqual(proc.returncode, 0, msg=proc.stderr) + self.assertIn("input=3", proc.stdout) + self.assertTrue(report.exists()) + payload = json.loads(report.read_text(encoding="utf-8")) + summary = payload.get("summary", {}) + self.assertEqual(summary.get("processed_entries"), 2) + self.assertEqual(summary.get("untouched_entries"), 1) + self.assertEqual(summary.get("retained_bad_entries"), 3) + rewritten = bad_list.read_text(encoding="utf-8").splitlines() + self.assertEqual( + rewritten, + ["000_00001.parquet", "000_00002.parquet", "000_00003.parquet"], + ) + + def test_offload_shard_bins_skip_if_missing_trained_registry(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + shards_root = root / "shards" + warm_root = root / "warm" + shards_root.mkdir(parents=True, exist_ok=True) + warm_root.mkdir(parents=True, exist_ok=True) + proc = subprocess.run( + [ + sys.executable, + "scripts/offload_shard_bins_to_warm.py", + "--shards-root", + str(shards_root), + "--warm-shards-root", + str(warm_root), + "--require-trained-batches-file", + str(root / "missing_trained.txt"), + "--skip-if-trained-file-missing", + ], + cwd=Path(__file__).resolve().parents[1], + capture_output=True, + text=True, + check=False, + ) + self.assertEqual(proc.returncode, 0, msg=proc.stderr) + self.assertIn("shard_offload_skip", proc.stdout) + def test_set_swappiness_help(self) -> None: proc = subprocess.run( ["bash", "scripts/set_swappiness.sh", "--help"], @@ -506,6 +670,233 @@ def test_pipeline_live_view_reports_train_token_gate(self) -> None: self.assertEqual(proc.returncode, 0, msg=proc.stderr) self.assertIn("Supervisor: gate=waiting_train_tokens 123456/999999", proc.stdout) + def test_pipeline_live_view_reports_quality_heartbeat(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + warm = root / "warm" + hot = root / "hot" + shards = root / "shards" + stage_dir = root / "stage" + sup_dir = root / "supervisor" + for path in [warm, hot, shards, stage_dir, sup_dir]: + path.mkdir(parents=True, exist_ok=True) + + (sup_dir / "eval_trend.tsv").write_text( + "\n".join( + [ + ( + "run_tag\tstep\teval_rc\tpass_rate\tcheck_pass_rate\tavg_case_score\t" + "cases_passed\tcases_total\treport_json" + ), + "run1\t1000\t0\t0.20\t0.70\t0.65\t3\t15\treport1.json", + "run2\t2000\t0\t0.30\t0.75\t0.70\t5\t15\treport2.json", + ] + ) + + "\n", + encoding="utf-8", + ) + (sup_dir / "generation_trend.tsv").write_text( + "\n".join( + [ + ( + "run_tag\tstep\tgeneration_rc\tpass_rate\tcheck_pass_rate\t" + "avg_case_score\tcases_passed\tcases_total\tregression_pass\t" + "baseline_report\treport_json" + ), + "run1\t1000\t0\t0.80\t0.85\t0.80\t4\t5\tTrue\tbase.json\tgen1.json", + "run2\t2000\t0\t1.00\t1.00\t1.00\t5\t5\tTrue\tgen1.json\tgen2.json", + ] + ) + + "\n", + encoding="utf-8", + ) + + proc = subprocess.run( + [ + sys.executable, + "scripts/pipeline_live_view.py", + "--once", + "--no-alt-screen", + "--refresh-seconds", + "0.1", + "--warm-dir", + str(warm), + "--hot-dir", + str(hot), + "--shards-root", + str(shards), + "--stage-state-dir", + str(stage_dir), + "--supervisor-state-dir", + str(sup_dir), + "--expected-parquet-files", + "510", + "--expected-bytes", + "1061360917731", + "--train-target-step", + "100000", + ], + cwd=Path(__file__).resolve().parents[1], + capture_output=True, + text=True, + check=False, + ) + self.assertEqual(proc.returncode, 0, msg=proc.stderr) + self.assertIn("Quality: heartbeat=improving", proc.stdout) + self.assertIn("eval=improving", proc.stdout) + self.assertIn("gen=improving", proc.stdout) + + def test_pipeline_live_view_quality_heartbeat_honors_eval_regression_flag(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + warm = root / "warm" + hot = root / "hot" + shards = root / "shards" + stage_dir = root / "stage" + sup_dir = root / "supervisor" + for path in [warm, hot, shards, stage_dir, sup_dir]: + path.mkdir(parents=True, exist_ok=True) + + (sup_dir / "eval_trend.tsv").write_text( + "\n".join( + [ + ( + "run_tag\tstep\teval_rc\tpass_rate\tcheck_pass_rate\tavg_case_score\t" + "cases_passed\tcases_total\tregression_pass\tpromotion_pass\t" + "failed_checks\tbaseline_report\treport_json" + ), + "run1\t1000\t0\t0.60\t0.80\t0.70\t9\t15\tTrue\tTrue\tnone\tNA\treport1.json", + "run2\t2000\t0\t0.65\t0.82\t0.72\t10\t15\tFalse\tFalse\tnone\tNA\treport2.json", + ] + ) + + "\n", + encoding="utf-8", + ) + (sup_dir / "generation_trend.tsv").write_text( + "\n".join( + [ + ( + "run_tag\tstep\tgeneration_rc\tpass_rate\tcheck_pass_rate\t" + "avg_case_score\tcases_passed\tcases_total\tregression_pass\t" + "baseline_report\treport_json" + ), + "run1\t2000\t0\t0.90\t0.90\t0.90\t4\t5\tTrue\tbase.json\tgen1.json", + ] + ) + + "\n", + encoding="utf-8", + ) + + proc = subprocess.run( + [ + sys.executable, + "scripts/pipeline_live_view.py", + "--once", + "--no-alt-screen", + "--refresh-seconds", + "0.1", + "--warm-dir", + str(warm), + "--hot-dir", + str(hot), + "--shards-root", + str(shards), + "--stage-state-dir", + str(stage_dir), + "--supervisor-state-dir", + str(sup_dir), + "--expected-parquet-files", + "510", + "--expected-bytes", + "1061360917731", + "--train-target-step", + "100000", + ], + cwd=Path(__file__).resolve().parents[1], + capture_output=True, + text=True, + check=False, + ) + self.assertEqual(proc.returncode, 0, msg=proc.stderr) + self.assertIn("Quality: heartbeat=regressed", proc.stdout) + self.assertIn("regression_pass=False", proc.stdout) + + def test_pipeline_live_view_quality_heartbeat_includes_holdout(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + warm = root / "warm" + hot = root / "hot" + shards = root / "shards" + stage_dir = root / "stage" + sup_dir = root / "supervisor" + for path in [warm, hot, shards, stage_dir, sup_dir]: + path.mkdir(parents=True, exist_ok=True) + + (sup_dir / "eval_trend.tsv").write_text( + "\n".join( + [ + "run_tag\tstep\teval_rc\tpass_rate\tcheck_pass_rate\tavg_case_score\tcases_passed\tcases_total\tregression_pass", + "run1\t1000\t0\t0.70\t0.80\t0.75\t7\t10\tTrue", + ] + ) + + "\n", + encoding="utf-8", + ) + (sup_dir / "generation_trend.tsv").write_text( + "\n".join( + [ + "run_tag\tstep\tgeneration_rc\tpass_rate\tcheck_pass_rate\tavg_case_score\tcases_passed\tcases_total\tregression_pass\tbaseline_report\treport_json", + "run1\t1000\t0\t0.80\t0.85\t0.80\t4\t5\tTrue\tbase.json\tgen1.json", + ] + ) + + "\n", + encoding="utf-8", + ) + (sup_dir / "holdout_trend.tsv").write_text( + "\n".join( + [ + "run_tag\tstep\tholdout_rc\tpass_rate\tcheck_pass_rate\tavg_case_score\tcases_passed\tcases_total\tregression_pass\tbaseline_report\treport_json", + "run1\t1000\t1\t0.40\t0.40\t0.40\t2\t5\tFalse\tbase.json\tholdout1.json", + ] + ) + + "\n", + encoding="utf-8", + ) + + proc = subprocess.run( + [ + sys.executable, + "scripts/pipeline_live_view.py", + "--once", + "--no-alt-screen", + "--refresh-seconds", + "0.1", + "--warm-dir", + str(warm), + "--hot-dir", + str(hot), + "--shards-root", + str(shards), + "--stage-state-dir", + str(stage_dir), + "--supervisor-state-dir", + str(sup_dir), + "--expected-parquet-files", + "510", + "--expected-bytes", + "1061360917731", + "--train-target-step", + "100000", + ], + cwd=Path(__file__).resolve().parents[1], + capture_output=True, + text=True, + check=False, + ) + self.assertEqual(proc.returncode, 0, msg=proc.stderr) + self.assertIn("Quality: heartbeat=regressed", proc.stdout) + self.assertIn("holdout=regressed", proc.stdout) + def test_pipeline_live_view_auto_detects_train_target_step(self) -> None: if ( subprocess.run( @@ -677,6 +1068,9 @@ def test_pipeline_eta_report_accepts_once_flag(self) -> None: payload = json.loads(out_json.read_text(encoding="utf-8")) self.assertIn("trainer_stall_seconds", payload["metrics"]) self.assertIn("offload_eligible_batches", payload["metrics"]) + self.assertIn("quality_heartbeat", payload) + self.assertIn("status_confidence", payload) + self.assertIn("overall_score", payload["status_confidence"]) def test_pipeline_eta_report_uses_coverage_fallback_rate(self) -> None: with tempfile.TemporaryDirectory() as tmp: @@ -773,6 +1167,57 @@ def test_pipeline_eta_report_uses_coverage_fallback_rate(self) -> None: ) self.assertNotEqual(payload["eta_human"]["manifest_unique_inputs"], "unknown") + def test_pipeline_eta_report_counts_offloaded_manifests_in_coverage(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + warm = root / "warm" + shards = root / "shards" + stage_dir = root / "stage" + sup_dir = root / "supervisor" + for path in [warm, shards, stage_dir, sup_dir]: + path.mkdir(parents=True, exist_ok=True) + + (shards / "batch_0001").mkdir(parents=True, exist_ok=True) + (shards / "batch_0001" / "manifest.offloaded.json").write_text( + json.dumps({"input_files": ["000_00077.parquet"]}), + encoding="utf-8", + ) + + out_json = root / "status.json" + out_txt = root / "status.txt" + state_json = root / "state.json" + proc = subprocess.run( + [ + sys.executable, + "scripts/pipeline_eta_report.py", + "--warm-dir", + str(warm), + "--shards-root", + str(shards), + "--stage-state-dir", + str(stage_dir), + "--supervisor-state-dir", + str(sup_dir), + "--expected-parquet-files", + "10", + "--expected-bytes", + "100", + "--output-json", + str(out_json), + "--output-text", + str(out_txt), + "--state-file", + str(state_json), + ], + cwd=Path(__file__).resolve().parents[1], + capture_output=True, + text=True, + check=False, + ) + self.assertEqual(proc.returncode, 0, msg=proc.stderr) + payload = json.loads(out_json.read_text(encoding="utf-8")) + self.assertEqual(payload["metrics"]["manifest_unique_input_files"], 1) + def test_pipeline_live_view_uses_eta_status_rate_fallback(self) -> None: if ( subprocess.run( @@ -856,10 +1301,7 @@ def test_pipeline_live_view_uses_eta_status_rate_fallback(self) -> None: def test_pipeline_eta_report_pgrep_root_count_dedupes_children(self) -> None: marker = f"eta_root_count_{int(time.time() * 1_000_000)}" - cmd = ( - f"exec -a {marker} bash -lc " - f"'exec -a {marker} sleep 15 & wait'" - ) + cmd = f"exec -a {marker} bash -lc " f"'exec -a {marker} sleep 15 & wait'" proc = subprocess.Popen( ["bash", "-lc", cmd], cwd=Path(__file__).resolve().parents[1], diff --git a/tests/test_train.py b/tests/test_train.py index 0f85af0..91f4b7c 100644 --- a/tests/test_train.py +++ b/tests/test_train.py @@ -10,6 +10,7 @@ from llm.tokenizer import BPETokenizer, tokenizer_contract_fingerprint, tokenizer_fingerprint from llm.train import ( ShardBatchSampler, + TrainConfig, _apply_resume_context_policy, _compute_keep_steps, _init_ema_state, @@ -18,12 +19,14 @@ _resolve_amp_mode, _update_ema_state, collect_shard_training_info, + run_training, ) except ModuleNotFoundError: torch = None BPETokenizer = None tokenizer_contract_fingerprint = None tokenizer_fingerprint = None + TrainConfig = None ShardBatchSampler = None _apply_resume_context_policy = None _compute_keep_steps = None @@ -33,6 +36,7 @@ _resolve_amp_mode = None _update_ema_state = None collect_shard_training_info = None + run_training = None def _write_tokenizer(path: Path, text: str, vocab_size: int = 256) -> int: @@ -196,6 +200,82 @@ def test_collect_shard_training_info_and_sampler(self) -> None: self.assertEqual(xb.dtype, torch.long) self.assertTrue(torch.all(xb[:, 1:] == yb[:, :-1])) + def test_sampler_limits_open_shard_cache_and_writes_trace(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + ds = root / "dataset" + tok = root / "tok.json" + + vocab_size = _write_tokenizer(tok, "a b c d e f g h i") + for idx in range(6): + _write_shard(ds / f"train_{idx:06d}.bin", list(range(80))) + _write_shard(ds / "val_000000.bin", list(range(40))) + manifest = { + "input_path": "x.txt", + "tokenizer_path": str(tok), + "tokenizer_hash": tokenizer_fingerprint(tok), + "tokenizer_contract_hash": tokenizer_contract_fingerprint(tok), + "tokenizer_vocab_size": vocab_size, + "token_dtype": "uint16", + "shard_size_tokens": 1024, + "val_ratio": 0.01, + "seed": 42, + "max_lines": 0, + "line_count": 10, + "train": { + "total_tokens": 480, + "shards": [{"path": f"train_{idx:06d}.bin", "tokens": 80} for idx in range(6)], + }, + "val": {"total_tokens": 40, "shards": [{"path": "val_000000.bin", "tokens": 40}]}, + } + (ds / "manifest.json").write_text(json.dumps(manifest, indent=2), encoding="utf-8") + + info = collect_shard_training_info(ds) + sampler = ShardBatchSampler( + shard_paths=info.train_shards, + token_dtype=info.token_dtype, + context_length=8, + seed=7, + device=torch.device("cpu"), + max_open_shards=2, + ) + for _ in range(20): + sampler.sample_batch(batch_size=8) + self.assertLessEqual(len(sampler._array_cache), 2) + + trace_path = root / "sample_trace.json" + count = sampler.write_sample_trace(trace_path) + self.assertGreater(count, 0) + payload = json.loads(trace_path.read_text(encoding="utf-8")) + self.assertIn("sampled_shards", payload) + self.assertTrue(payload["sampled_shards"]) + self.assertEqual(payload["sampling_strategy"], "weighted") + + def test_sampler_balanced_strategy_distributes_evenly(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + shard_paths: list[Path] = [] + for idx in range(5): + shard_path = root / f"train_{idx:06d}.bin" + _write_shard(shard_path, list(range(80))) + shard_paths.append(shard_path) + + sampler = ShardBatchSampler( + shard_paths=shard_paths, + token_dtype="uint16", + context_length=8, + seed=123, + device=torch.device("cpu"), + sampling_strategy="balanced", + ) + for _ in range(17): + sampler.sample_batch(batch_size=3) + + sampled = dict(sampler.sampled_shard_rows()) + self.assertEqual(len(sampled), len(shard_paths)) + counts = [sampled[path.resolve()] for path in shard_paths] + self.assertLessEqual(max(counts) - min(counts), 1) + def test_ema_state_update(self) -> None: model = torch.nn.Linear(2, 2, bias=False) with torch.no_grad(): @@ -262,6 +342,106 @@ def test_apply_resume_context_policy(self) -> None: allow_extension=False, ) + def test_run_training_writes_sampled_trace(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + ds = root / "dataset" + out = root / "out" + tok = root / "tok.json" + trace = root / "sampled_trace.json" + + vocab_size = _write_tokenizer(tok, "simple tiny corpus for training") + _write_shard(ds / "train_000000.bin", list(range(200))) + _write_shard(ds / "val_000000.bin", list(range(120))) + _write_manifest( + ds / "manifest.json", + tokenizer_path=tok, + train_shard="train_000000.bin", + val_shard="val_000000.bin", + vocab_size=vocab_size, + ) + + result = run_training( + TrainConfig( + shards_path=ds, + output_dir=out, + max_steps=1, + batch_size=2, + context_length=16, + grad_accum_steps=1, + eval_interval=1, + eval_steps=1, + log_interval=1, + device="cpu", + n_layers=1, + n_heads=1, + d_model=32, + sampled_shards_trace=trace, + sampled_shards_trace_min_rows=1, + ) + ) + + self.assertEqual(result["max_steps"], 1) + self.assertTrue(trace.exists()) + payload = json.loads(trace.read_text(encoding="utf-8")) + self.assertIn("sampled_shards", payload) + self.assertTrue(payload["sampled_shards"]) + + def test_run_training_rejects_unmet_sampler_min_full_passes(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + ds = root / "dataset" + out = root / "out" + tok = root / "tok.json" + + vocab_size = _write_tokenizer(tok, "simple tiny corpus for training") + _write_shard(ds / "train_000000.bin", list(range(200))) + _write_shard(ds / "train_000001.bin", list(range(200))) + _write_shard(ds / "val_000000.bin", list(range(120))) + manifest = { + "input_path": "x.txt", + "tokenizer_path": str(tok), + "tokenizer_hash": tokenizer_fingerprint(tok), + "tokenizer_contract_hash": tokenizer_contract_fingerprint(tok), + "tokenizer_vocab_size": vocab_size, + "token_dtype": "uint16", + "shard_size_tokens": 1024, + "val_ratio": 0.01, + "seed": 42, + "max_lines": 0, + "line_count": 10, + "train": { + "total_tokens": 400, + "shards": [ + {"path": "train_000000.bin", "tokens": 200}, + {"path": "train_000001.bin", "tokens": 200}, + ], + }, + "val": {"total_tokens": 120, "shards": [{"path": "val_000000.bin", "tokens": 120}]}, + } + (ds / "manifest.json").write_text(json.dumps(manifest, indent=2), encoding="utf-8") + + with self.assertRaises(ValueError): + run_training( + TrainConfig( + shards_path=ds, + output_dir=out, + max_steps=1, + batch_size=1, + context_length=16, + grad_accum_steps=1, + eval_interval=1, + eval_steps=1, + log_interval=1, + device="cpu", + n_layers=1, + n_heads=1, + d_model=32, + sampler_strategy="balanced", + sampler_min_full_passes=1, + ) + ) + if __name__ == "__main__": unittest.main() diff --git a/wiki/Architecture-and-Roadmap.md b/wiki/Architecture-and-Roadmap.md index a02a67d..20e3d03 100644 --- a/wiki/Architecture-and-Roadmap.md +++ b/wiki/Architecture-and-Roadmap.md @@ -31,7 +31,7 @@ Current implemented modules: ### Phase 3 - Sampling strategies (greedy, top-k, top-p) - Experiment configs and reproducibility controls -- Runtime performance profiling +- Runtime performance profiling (`scripts/benchmark_rtx5070_context_profiles.sh`) ### Phase 4 - CI automation diff --git a/wiki/Data-Pipeline-and-Versioning.md b/wiki/Data-Pipeline-and-Versioning.md index a242c15..dcc83dc 100644 --- a/wiki/Data-Pipeline-and-Versioning.md +++ b/wiki/Data-Pipeline-and-Versioning.md @@ -130,6 +130,12 @@ bash scripts/hf_download_watchdog.sh \ The watchdog monitors parquet/incomplete byte growth and restarts the resumable worker if it exits or stalls. Use `--exit-on-complete` with expected targets so the watchdog exits when the download is actually complete. +Simple full FineWeb-Edu sync (direct to Ceph path): +```bash +export HF_TOKEN=hf_xxx # optional but recommended +bash scripts/sync_fineweb_edu_full.sh /mnt/pve/cephfs/llm/data/fineweb/fineweb-edu-full +``` + Hot-queue prefetch worker (stage on demand while training): ```bash bash scripts/fineweb_prefetch_hot_queue.sh \ @@ -255,6 +261,18 @@ Rehydrate local hot workspace from warm storage: bash scripts/hydrate_from_warm_storage.sh /mnt/ceph/llm/data ``` +Pre-wipe safety checklist: +```bash +git fetch --all --prune +git status --short --branch +git push +bash scripts/sync_warm_storage.sh /mnt/ceph/llm/data +stamp=$(date -u +%Y%m%dT%H%M%SZ) +git ls-files --others --ignored --exclude-standard | sort \ + > /mnt/ceph/llm/data/logs/git_untracked_all_${stamp}.txt +``` +Only wipe after sync completion and a fresh `last_sync_utc.txt` timestamp. + ## First-Pass Talking Profile For an English prose-first pass, generate include/exclude manifests and optionally move excluded ZIMs to warm storage: diff --git a/wiki/Home.md b/wiki/Home.md index 63ad0e9..f439725 100644 --- a/wiki/Home.md +++ b/wiki/Home.md @@ -38,4 +38,17 @@ make typecheck make train ``` +## Pre-Wipe Checklist +Before wiping or rebuilding the hot workspace, run: + +```bash +git fetch --all --prune +git status --short --branch +git push +bash scripts/sync_warm_storage.sh /mnt/ceph/llm/data +``` + +Then wait for sync completion and verify +`/mnt/ceph/llm/data/logs/last_sync_utc.txt` was updated. + For server-specific setup and storage guidance, see [Setup and Tooling](Setup-and-Tooling) and [Data Pipeline and Versioning](Data-Pipeline-and-Versioning). diff --git a/wiki/Setup-and-Tooling.md b/wiki/Setup-and-Tooling.md index bc0ad0b..24cd23d 100644 --- a/wiki/Setup-and-Tooling.md +++ b/wiki/Setup-and-Tooling.md @@ -34,6 +34,7 @@ make hydrate-warm make offload-zim make checkpoint-offload-prune make set-swappiness +make benchmark-rtx5070 ``` `make sync-warm` now includes raw/training inputs (`data/raw_zim`, `data/fineweb`, @@ -58,11 +59,15 @@ make set-swappiness - Saved configs: `configs/train/rtx5070/` - Recommended launcher: ```bash -bash scripts/train_rtx5070_fineweb_bpe_v1_big.sh +bash scripts/train_rtx5070_fineweb_350bt_bpe_v2.sh ``` - Training defaults now include warmup+cosine LR and fixed held-out eval batches. - For VRAM pressure, increase effective batch with `--grad-accum-steps`. - For long-context continuation, use `bash scripts/train_rtx5070_fineweb_350bt_bpe_v2_ctx1024.sh`. +- For reproducible context/batch benchmarking, use: +```bash +bash scripts/benchmark_rtx5070_context_profiles.sh --max-steps 1200 --compile-model +``` - For release bundles, use `scripts/package_inference_bundle.py` or `--include-safetensors` in HF prepare script. ## systemd Services @@ -79,10 +84,6 @@ Units installed from `deploy/systemd/`: - `llm-checkpoint-offload-prune.timer` - `llm-vm-swappiness.service` -Optional prefetch service (only if you want separate prefetch in addition to stage-loop staging): -- `llm-fineweb-prefetch.service` -- install with: `bash scripts/install_systemd_services.sh --install-watchdog --install-prefetch` - `deploy/systemd/llm.env.example` includes tuned loop args for this host profile (`--hot-queue-min-files 10`, `--stage-copy-jobs 4`, `--stage-min-free-gib 80`, `--auto-tune-shard-jobs`, `--sync-background`, `--shard-size-tokens 20000000`).