From 3bf6ccf17b55a5c1bc8ce3958741fb5a202e26d6 Mon Sep 17 00:00:00 2001 From: PieterjanDeClippel Date: Sun, 12 Jul 2026 19:19:37 +0200 Subject: [PATCH 01/12] docs(M38): PRD + plan to reduce per-game boilerplate (supersedes PR #27) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A three-agent audit mapped the remaining copy-paste across three layers and the staleness of PR #27 (M36 INetworkTelemetrySource + M37 DqnGrowth edited exactly the campaign bodies that PR relocates, so a rebase drops net-growth). Adds docs/prd/BOILERPLATE_REDUCTION_PRD.md and a phased, revert-friendly M38 milestone in PLAN.md. Behaviour-preserving refactor (SHA256-bitwise-identical training); re-cuts DqnScoreCampaign to absorb the M36/M37 duplication rather than fight it, keeps PR #27's still-valid web RefreshingCheckpoint + checkpoint README, and fixes two drift bugs the duplication caused (frontend poller try/catch, RushHour puzzle validator). Docs only — no code change yet. Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/prd/BOILERPLATE_REDUCTION_PRD.md | 173 ++++++++++++++++++++++++++ docs/prd/PLAN.md | 58 +++++++++ 2 files changed, 231 insertions(+) create mode 100644 docs/prd/BOILERPLATE_REDUCTION_PRD.md diff --git a/docs/prd/BOILERPLATE_REDUCTION_PRD.md b/docs/prd/BOILERPLATE_REDUCTION_PRD.md new file mode 100644 index 0000000..e29f118 --- /dev/null +++ b/docs/prd/BOILERPLATE_REDUCTION_PRD.md @@ -0,0 +1,173 @@ +# Reduce per-game boilerplate — campaigns, web services, frontend — PRD + +**Status:** Planned · 2026-07-12 · branch TBD (off `master`) · **supersedes the stale [PR #27](https://github.com/MintPlayer/MintPlayer.AI/pull/27)** +**Owner:** Pieterjan +**Milestone:** [PLAN.md](PLAN.md) M38 · **Depends on:** nothing new — a behaviour-preserving refactor across the Lab training harness (§8 of [../ARCHITECTURE.md](../ARCHITECTURE.md)), the web model-service layer (§7), and the Angular playground (§10). + +## 1. Problem + +Adding each game has left behind **near-identical copy-paste** in three layers. The duplication is not +harmful *today*, but every copy is a place a fix has to land N times, and several copies have **already +drifted** — one copy has a bug its siblings don't. This is exactly the change-amplification / unknown-unknowns +cost Ousterhout warns about. + +PR #27 ("reduce per-game boilerplate") took a first cut at this on 2026-07-10, but it is now **stale and +conflicting**: M36 (network visualizer / `INetworkTelemetrySource`) and M37 (progressive net growth / +`DqnGrowth`) landed afterward and edited precisely the campaign method bodies and fields that PR #27 deletes +and relocates. A straight rebase would silently drop net-growth. This PRD **re-scopes** the effort: it keeps +the parts of PR #27 that still apply verbatim, re-cuts the campaign refactor to *absorb* the new M36/M37 +duplication rather than fight it, and adds the further duplication a three-agent audit surfaced. + +**What "boilerplate" means here** (audited 2026-07-12, three parallel agents; counts verified against `master`): + +### Lab / training campaigns (`tools/…Lab/`) +- **DQN campaign twins.** `SnakeDqnCampaign.cs` (213 L) and `FruitCakeDqnCampaign.cs` (236 L) are the *same* + score-maximizing Double+Dueling DQN campaign. `TrainChunk` is byte-identical; warm-start/resume, the + save-best gate, the eval loop, and the telemetry block are all parallel. FruitCake's only real extras: a + noisy `StateId` line and the plain→noisy / `GrowInput` warm-net adaptation. +- **`INetworkTelemetrySource` duplicated 6×** (M36). Every campaign — Snake, FruitCake, RushHourImitation, + CubeImitation, CubeEfficient, CubeDavi — carries a ~30-line explicit-interface block + (`SnapshotParameters`/`Sample`/`SampleIo`/`SampleActivations`). The two DQN copies are ~90% identical twins. +- **Net-growth wiring** (M37) repeated per campaign: `grow`/`growEvery` ctor params, a `_growRng` field, and a + `DqnGrowth.Maybe(...)` / `PolicyGrowth.Maybe(...)` line in `TrainChunk`. +- **CLI flag parsing hand-rolled 6×.** Every `*Lab.cs` re-derives the `i+1 < Length` bounds guard and + `args[++i]` walk for the shared flags (`--hours`/`--data`/`--seed`/`--lr`/`--eval-only`, plus + `--grow`/`--grow-every` in five). **Live inconsistency:** culture handling is ad-hoc — `double`/`float` + mostly pass `InvariantCulture`, `int`/`long`/`ulong` inconsistently do not. A latent locale-dependent bug. +- **Host-bootstrap + `runner.Run` tail duplicated in all 6 labs** (~12 L each): `AIHost.CreateBuilder` → + resolve store/runner → CSV path → new campaign → `VizLauncher.TryStart` → `runner.Run` → `WaitForViewer`. + `WaitForViewer` currently lives in `SnakeLab` and is called cross-type from every other lab (a smell). +- **Imitation/policy lifecycle plumbing 3×** (RushHour, CubeImitation, CubeEfficient): Adam load/save, net + resume-or-init, window-mean fields+reset, and the growth one-liner are identical. (Their `Evaluate` and + data-generation *are* genuinely different algorithms — those must stay bespoke.) +- **Trivia:** identical one-line `Log` in all 6 campaigns; growth-RNG seed constant differs by family. + +### Web model-service layer (`src/RLDemo.Web/`) +- **The cadence-refresh checkpoint getter — 4 copies** (the most bug-prone): "re-read on a cadence, + double-checked-locked, swallow a corrupt/mid-write read and keep the previous value, optional post-load + hook." CubeModelService ×3 (policy/value/efficient) + RushHourModelService ×1. This is the copy PR #27 + correctly targeted; **still 4 verbatim copies on `master`** (`TryOpenRead` appears 7× across the 3 services). +- **Model-service startup/readiness quartet — 3 copies:** `Status`/`Error` snapshot + lazy agent getter + + `TryLoadFromStore` + `Initialize` ("load, else `Status=Failed` + warn"). Cube/Game2048/RushHour. +- **`RushHourController.TryBuildPuzzle` duplicated verbatim in `RushHourDeckStore`** (~26 L; a comment already + admits the mirror). **Two validators of one contract will drift** — the deck store could accept a board the + solver rejects. +- **Controller `/status` + 503 gating — 3 copies;** and `Game2048Controller` needlessly re-declares + `Status2048Response` where Cube+RushHour share `StatusResponse`. + +### Angular playground (`src/RLDemo.Web/ClientApp/`) +- **Component `pollStatus()` self-rescheduling poller — 3 copies, with a real divergence bug:** `cube.ts` has + a `try/catch`; `game-2048.ts` and `rush-hour.ts` **do not** → an uncaught rejection on a backend blip. +- **`*-api.ts` `status()` + 503→`loading` classification — 3 copies.** +- **Watch-mode wake-lock + `visibilitychange` scaffolding — 3 copies** (snake, mountaincar, fruit-cake); + `onVisibilityChange()` is byte-identical. +- **Director watch-loop skeleton — 2 copies** (snake, mountaincar; fruit-cake's rAF loop differs — leave it). +- **Atomic temp-file write — 2 copies** (`GalleryStore`, `RushHourDeckStore`; trivial). + +## 2. Goal & success criteria + +**Reduce the duplication above into deep, well-named modules — with zero behaviour change — and fix the two +latent bugs the duplication has already caused.** + +- **Behaviour-preserving (the hard gate).** Training stays **bitwise-identical**: a `--game snake` / `fruitcake` + run (and the imitation/policy campaigns) produces a **SHA256-equal** checkpoint before and after. The web API + contract tests and the campaign/checkpoint test suites stay green. Only log wording may change. +- **Deep, not shallow.** Every extraction must hide more than it exposes (Ousterhout): a `RefreshingCheckpoint` + exposes `.Current` and hides the lock + cadence + keep-previous-on-corrupt; a `CliArgs` reader hides the + bounds-check + culture + `++i`. We explicitly **do not** create a shallow `ModelService` base or a + shared *solve*-controller base — the solve endpoints are genuinely different algorithms/DTOs and stay bespoke + (PR #27 got this call right). +- **Fix the drift bugs found in the audit** as part of the relevant step: the missing `try/catch` in two + frontend pollers, and the duplicated Rush Hour puzzle validator that can drift out of sync with the solver. +- **Absorb M36/M37, don't regress them.** The re-cut `DqnScoreCampaign` base **owns** net-growth and + implements `INetworkTelemetrySource` itself, so both DQN games lose their growth wiring and telemetry twin — + *more* reduction than PR #27, with growth preserved. +- **Net line reduction** across the touched files, with the shared modules documented (interface comment stating + what each promises and hides). + +**Non-goals.** No new features, no API/DTO changes visible to clients, no change to checkpoint formats or the +wire protocol. No merging of the three imitation/policy *campaigns* (only their plumbing is extracted). No +touching the `.pg` single-source FruitCake path. No frontend visual change. + +## 3. Design — the shared modules + +Grouped by the concern each hides. Each is a **deep module**: small surface, meaningful implementation hidden. + +### Lab +1. **`DqnScoreCampaign` (abstract base)** — the score-maximizing DQN spine. Owns: seeds/state/warm-net fields, + the save-best gate, `Resume`/`TrainChunk`/`IsComplete`/`Evaluate`/`Checkpoint`/`Dispose`, **net-growth** + (`grow`/`growEvery` + `_growRng` + `DqnGrowth.Maybe` inside its `TrainChunk`), **and + `INetworkTelemetrySource`** (`NetKind`, `SnapshotParameters`, `Sample`, `SampleIo`, `SampleActivations`, + reading `State?.Online ?? WarmNet`). Subclasses supply only: `Environment`, `TrainEnv`, `BaseOptions`, + `EvaluateNet`, `ObservationSize`/`InputLabels`/`OutputLabels`, and (FruitCake) an `AdaptWarmNet` hook for + plain→noisy + `GrowInput`. *This is the re-cut of PR #27 change 1, enlarged to fold in the M36 telemetry + twin and the M37 growth wiring.* +2. **`CliArgs` value-reader + `CommonLabArgs.Parse`** — `CliArgs` hides the `i+1`** — `(store, envId, algoId, load, refresh, onReload?)` exposing just `.Current`. + Hides TTL + double-checked lock + keep-previous-on-corrupt (defines the corrupt-read error out of + existence). Cube's GPU resident-forward rebuild rides the `onReload` hook. Collapses the 4 getters to a + field + one-line property. *(PR #27's `RefreshingModel`, renamed for clarity.)* +7. **`StartupCheckpoint`** — the load-at-startup readiness holder (`Status`/`Error` + single-place + `Initialize`). Applied by **composition**, not a base class (a `ModelService` base would be shallow/leaky). + Covers the same concern as #6, so ship them as a pair. +8. **`RushHourPuzzleDto.TryBuild(...)`** on the Environments layer — one validator; controller + deck store + both call it. Kills the drift bug. +9. **`ControllerBase.ModelStatusResult(...)` extension** + **delete `Status2048Response`** (use the shared + `StatusResponse`). Small; solve endpoints stay bespoke. + +### Web (frontend) +10. **`pollModelStatus(api, set, intervalMs)`** — one poller with the `try/catch` built in (fixes the 2048 / + rush-hour divergence). `fetchStatus(path)` + `classifySolve(...)` for the `*-api.ts` `status()`/503 union + (keep each game's typed `Solve*Result`). +11. **`WatchWakeLock`** helper/directive driven by the component `mode` signal — hides the sentinel + + `visibilitychange` bookkeeping (not just a wrapper around acquire/release). +12. **`runDirectorLoop({tick, step, onFrame, onDone})`** for the 2 `setInterval` directors (kept minimal — only + 2 call sites). **`AtomicFile.Write`** for the 2 temp-write copies (trivial, backend). + +### Checkpoints doc +13. **`Core/Checkpoints/README.md`** — the "when to use which checkpoint type" decision table from PR #27 + (pure docs, still accurate). Land verbatim. + +## 4. Deliberately left alone + +- **Per-game *solve* controllers + DTOs** (beam search / policy-A\* / expectimax) — different algorithms, + different validation. A shared base would be a leaky, shallow abstraction. (Same call PR #27 made.) +- **The 3 imitation/policy *campaigns* as wholes** — only plumbing (§3.4) is extracted; `Evaluate`/data-gen + stay bespoke. +- **`CubeDaviLab`'s 46-flag + appsettings block**, `CampaignRunner`, `DqnGrowth`/`PolicyGrowth`, + `CubePolicyTraining`, `CubeViz`, `VizLauncher`, `VizServer` — already deep/good. +- **fruit-cake's rAF watch loop** — genuinely different from the `setInterval` directors. + +## 5. Risks + +- **Silent behaviour change is the whole risk.** Mitigation: the SHA256 bitwise-identical checkpoint gate per + DQN game (the M36 methodology already exists), plus the existing campaign/checkpoint/web-API test suites, run + after **each** step. Any per-step diff in a checkpoint = stop and reconcile. +- **The telemetry-in-base move must read the same fields the 6× copies read.** Verify `--viz` still renders each + game live (dev-only manual check) after the base absorbs telemetry. +- **Order matters.** Land the low-risk, self-contained pieces (docs, bug-fix extracts, web checkpoint holders) + before the campaign re-cut, so a regression is bisectable. + +## 6. Verification + +- `dotnet build` clean (0 warn / 0 err) after every step. +- `dotnet test --filter "Category!=Slow"` green (campaign contract/runner, checkpoint round-trip, web API/service + suites). +- **Bitwise gate:** a short fixed-seed `--game snake` and `--game fruitcake` run yields a SHA256-equal + `*.dqn.ckpt` + `*.dqn-state.ckpt` vs a pre-refactor baseline; likewise a short imitation/policy run after §3.4. +- **`--viz` smoke** (dev-only): each game still streams a live net after the telemetry move. +- Net **negative** line diff with the shared modules documented. + +See [PLAN.md](PLAN.md) M38 for the phased, revert-friendly step order. diff --git a/docs/prd/PLAN.md b/docs/prd/PLAN.md index d631627..cd491be 100644 --- a/docs/prd/PLAN.md +++ b/docs/prd/PLAN.md @@ -1286,6 +1286,64 @@ note the identity diagonals in the just-deepened layers' heatmaps), both with ** 320 fast tests green (incl. Net2Net forward-equality for DuelingQNet + PolicyValueNet, and a v1-checkpoint-load test). Screenshots: `docs/screenshots/m36-network-grows.png` (DQN), `m37-policy-net-grows.png` (policy). +## M38 — Reduce per-game boilerplate (campaigns · web services · frontend) *(planned 2026-07-12; supersedes stale PR #27; see `BOILERPLATE_REDUCTION_PRD.md`)* 🔜 + +**Problem.** Adding each game left near-identical copy-paste in three layers, and several copies have already +**drifted** (one has a bug its siblings don't). PR #27 took a first cut on 2026-07-10 but is now stale/conflicting: +M36 (`INetworkTelemetrySource`) and M37 (`DqnGrowth`) edited exactly the campaign bodies/fields PR #27 relocates, +so a rebase would silently drop net-growth. This milestone re-scopes: keep what still applies, re-cut the campaign +refactor to **absorb** the M36/M37 duplication, and add the further duplication a three-agent audit surfaced. A +**behaviour-preserving** refactor — training stays SHA256-bitwise-identical. + +**Audit (2026-07-12, 3 parallel agents; counts verified vs `master`).** Lab: two DQN campaign twins (~200 shared +lines), `INetworkTelemetrySource` copied **6×**, net-growth wiring per campaign, CLI flag parsing hand-rolled 6× +(with an ad-hoc-culture latent bug), host-bootstrap tail 6×, imitation/policy plumbing 3×. Web: the cadence-refresh +"keep-previous-on-corrupt" checkpoint getter **4×** (highest bug-risk), startup/readiness quartet 3×, `TryBuildPuzzle` +duplicated controller↔deck-store (drift bug), `/status`+503 3× (+ a needless `Status2048Response` fork). Frontend: +`pollStatus()` 3× (2 **missing the `try/catch`** cube has — a latent unhandled-rejection bug), `*-api.ts` status/503 +3×, watch wake-lock scaffolding 3×, director loop 2×, atomic-write 2×. + +**Phased plan (each step ends on a green build + the relevant test suite; ordered low-risk→high-risk so a regression +is bisectable).** + +- **B0 — docs + standalone bug-fix extracts (no behaviour risk).** Land `Core/Checkpoints/README.md` (the + when-to-use-which-checkpoint table, from PR #27, still accurate). Fix + de-dup the two drift bugs: one + `RushHourPuzzleDto.TryBuild` on the Environments layer (controller + `RushHourDeckStore` both call it), and a + frontend `pollModelStatus(api, set, ms)` with `try/catch` built in (fixes 2048 + rush-hour). Delete + `Status2048Response` in favour of the shared `StatusResponse`. **Gate:** web API tests green; frontend builds. +- **B1 — web checkpoint concern (deep pair).** `RefreshingCheckpoint` (`.Current` hides TTL + double-checked + lock + keep-previous-on-corrupt + `onReload` hook) and `StartupCheckpoint` (readiness holder, applied by + **composition** — no shallow `ModelService` base). Collapse the 4 refreshing getters + 3 startup quartets in + Cube/Game2048/RushHour services; Cube's GPU resident-forward rebuild rides `onReload`. `ModelStatusResult` + controller extension for the `/status` trio (solve endpoints stay bespoke). **Gate:** web service + API tests + green; net negative diff. +- **B2 — re-cut `DqnScoreCampaign` base (the M36/M37 absorption).** Shared score-max DQN spine owning + resume/train/save-best/checkpoint **plus** net-growth (`grow`/`growEvery` + `_growRng` + `DqnGrowth.Maybe` in its + `TrainChunk`) **plus** `INetworkTelemetrySource` itself; Snake/FruitCake supply only env + `BaseOptions` + + `EvaluateNet` + labels + FruitCake's `AdaptWarmNet` (plain→noisy + `GrowInput`). **Gate (the hard one):** a + fixed-seed `--game snake`/`fruitcake` run yields a **SHA256-equal** `*.dqn.ckpt` + `*.dqn-state.ckpt` vs a + pre-refactor baseline; `--viz` still streams both nets live. +- **B3 — Lab CLI + host plumbing.** `CliArgs` value-reader (hides bounds-check + `++i` + `InvariantCulture` — fixes + the locale inconsistency) + `CommonLabArgs.Parse` for the shared 6 flags; `LabHost.Run(...)` owning + DI+GPU+CSV+viz+viewer-lifetime (`WaitForViewer` moves off `SnakeLab`). Apply to all 6 labs. `LabLog.Line`, + `GrowthRng(seed)` trivia. **Gate:** every `--game` still parses + runs identically (bitwise spot-check retained). +- **B4 — imitation/policy plumbing extract.** `SupervisedNetState` (net+Adam load-or-init/save + growth one-liner) + + `WindowMean`, applied to RushHour/CubeImitation/CubeEfficient. **Campaigns stay separate** — only plumbing is + shared (their `Evaluate`/data-gen are different algorithms). **Gate:** a short imitation/policy run is + SHA256-equal to baseline. +- **B5 — frontend watch scaffolding (optional polish).** `WatchWakeLock` (mode-signal-driven; hides sentinel + + `visibilitychange`) for snake/mountaincar/fruit-cake; `runDirectorLoop(...)` for the 2 `setInterval` directors; + `fetchStatus`/`classifySolve` in `*-api.ts`; `AtomicFile.Write` for the 2 temp writes. **Gate:** frontend builds; + manual watch-mode smoke. + +**Deliberately left alone:** per-game *solve* controllers/DTOs (different algorithms — a shared base would be +shallow/leaky, PR #27's correct call), the 3 imitation/policy campaigns *as wholes*, `CubeDaviLab`'s 46-flag block, +`CampaignRunner`/`DqnGrowth`/`PolicyGrowth`/`VizServer` (already deep), fruit-cake's rAF loop. + +**Gate (whole milestone):** full solution builds 0/0; `dotnet test --filter "Category!=Slow"` green; DQN + a +policy campaign SHA256-bitwise-identical vs baseline; `--viz` live for every game; net **negative** line diff with +every shared module carrying an interface comment. See `BOILERPLATE_REDUCTION_PRD.md`. + ## Testing strategy (cross-cutting, from research) 1. **Known-solved thresholds** as integration tests (median over ≥3 seeds) — slow bucket. From 46726eee8ac72d7722a7775d7bd021e0215830b5 Mon Sep 17 00:00:00 2001 From: PieterjanDeClippel Date: Sun, 12 Jul 2026 19:31:35 +0200 Subject: [PATCH 02/12] M38 B0: checkpoint docs + de-dup drift bugs (no behaviour change) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Lowest-risk first step of the boilerplate-reduction plan. - Core/Checkpoints/README.md: when-to-use-which-checkpoint decision table (from PR #27, still accurate). - RushHour: one shared RushHourBoardDto.TryBuildPuzzle used by both the controller (analyze/solve) and RushHourDeckStore.Upsert — kills the duplicated validator that could drift so the deck store accepts a board the solver rejects. - Frontend: shared pollModelStatus() with try/catch built in, replacing 3 hand-rolled pollers — two of which (2048, rush-hour) lacked the catch and could raise an unhandled rejection on a backend blip. - Collapse Status2048Response into the shared StatusResponse DTO (moved to its own file); identical wire shape, so the API tests are unaffected. Build 0 errors; 320 fast tests green (incl. Cube/RushHour API + service). Co-Authored-By: Claude Opus 4.8 (1M context) --- .../Checkpoints/README.md | 48 +++++++++++++ src/RLDemo.Web/ClientApp/src/app/cube/cube.ts | 13 +--- .../ClientApp/src/app/game-2048/game-2048.ts | 9 +-- .../ClientApp/src/app/model-status.ts | 25 +++++++ .../ClientApp/src/app/rush-hour/rush-hour.ts | 9 +-- .../Controllers/Game2048Controller.cs | 4 +- .../Controllers/RushHourController.cs | 68 ++++++++++--------- src/RLDemo.Web/Controllers/StatusResponse.cs | 8 +++ src/RLDemo.Web/Services/RushHourDeckStore.cs | 29 +------- 9 files changed, 125 insertions(+), 88 deletions(-) create mode 100644 src/MintPlayer.AI.ReinforcementLearning.Core/Checkpoints/README.md create mode 100644 src/RLDemo.Web/ClientApp/src/app/model-status.ts create mode 100644 src/RLDemo.Web/Controllers/StatusResponse.cs diff --git a/src/MintPlayer.AI.ReinforcementLearning.Core/Checkpoints/README.md b/src/MintPlayer.AI.ReinforcementLearning.Core/Checkpoints/README.md new file mode 100644 index 0000000..e7aa499 --- /dev/null +++ b/src/MintPlayer.AI.ReinforcementLearning.Core/Checkpoints/README.md @@ -0,0 +1,48 @@ +# Checkpoints + +Versioned, resumable persistence for training runs and deployable models. Everything here sits +between the in-memory runtime objects (networks, optimizer, agents) and raw `Stream`s; the +`FileModelStore` then puts those streams on disk atomically. + +## When do I use which? + +One rule covers most cases: + +- **Resuming training** → save the *composite state* (`DqnTrainingState` for DQN, `TabularCheckpoint` + for tabular). It carries everything needed to continue *bitwise-identically* — nets, optimizer, + replay buffer, RNG streams. +- **Shipping a trained brain for inference** (the web app, an eval-only run) → save the *individual + net* checkpoint (`DuelingQNetCheckpoint` / `MlpCheckpoint` / `ResidualMlpCheckpoint`). Just shape + + weights; no optimizer or buffer. + +The campaigns do both: they persist the full resume state under `/-state` and, keep-best +gated, the deployable net under `/dqn` — the id the web `ModelService` loads. + +## The types + +| Checkpoint | Serializes | Standalone or embedded | Format | +|---|---|---|---| +| `MlpCheckpoint` / `ResidualMlpCheckpoint` / `DuelingQNetCheckpoint` | one network's architecture + weights | standalone deployable file **or** embedded via `QNetCheckpoint` | binary | +| `QNetCheckpoint` | any `IValueNet` (type-tagged: `Mlp` \| `DuelingQNet`) | embedded (inside a training state) | binary | +| `AdamCheckpoint` | optimizer hyperparameters + moment estimates — needs the net's parameters to reload | embedded only | binary | +| `ReplayBufferCheckpoint` | DQN replay transitions | embedded only | binary | +| `DqnTrainingState` | **the whole DQN run** — online+target nets + Adam + replay buffer + RNGs + env snapshot | top-level resume file | binary | +| `TabularCheckpoint` | a Q-table | standalone resume **and** deploy (small enough to be both) | JSON (human-inspectable, lossless) | + +`CheckpointFormat` holds the shared binary primitives (4-byte magic, kind + version header, float/int/ +bool arrays, RNG state). Every binary checkpoint above is built from it, so they share one on-disk +convention and one versioning scheme. + +## Persistence seam + +`IModelStore` / `FileModelStore` (`ModelStore.cs`) is the file layer: one *current* checkpoint per +`(environmentId, algorithmId)` pair, written to `/..ckpt` via a temp-file + rename so +a crash mid-save never corrupts the previous checkpoint. It deals only in raw `Stream`s and knows +nothing about net types — the `*Checkpoint` classes above own that knowledge. + +## Versioning + +Each checkpoint kind carries its own format version and stays backward-compatible: a newer reader +loads older files (e.g. `DqnTrainingState` v2 files load with a default noise RNG; `DuelingQNet` v1 +files load as plain non-noisy nets). Bump the version and branch on it in `Read`/`Load` when you add a +field — never reorder existing fields. diff --git a/src/RLDemo.Web/ClientApp/src/app/cube/cube.ts b/src/RLDemo.Web/ClientApp/src/app/cube/cube.ts index c1b2ea7..747528a 100644 --- a/src/RLDemo.Web/ClientApp/src/app/cube/cube.ts +++ b/src/RLDemo.Web/ClientApp/src/app/cube/cube.ts @@ -4,6 +4,7 @@ import * as THREE from 'three'; import { OrbitControls } from 'three/addons/controls/OrbitControls.js'; import { CubeApi, CubeSolveResponse, CubeStatusResponse } from './cube-api'; import { RubiksCube } from './cube-renderer'; +import { pollModelStatus } from '../model-status'; const FACES = ['U', 'D', 'L', 'R', 'F', 'B'] as const; const OPPOSITE: Record = { U: 'D', D: 'U', R: 'L', L: 'R', F: 'B', B: 'F' }; @@ -282,17 +283,7 @@ export class Cube { // ------------------------------------------------------------------ model status private pollStatus(): void { - void (async () => { - try { - const status = await this.api.status(); - this.modelStatus.set(status); - if (status.status === 'loading') { - setTimeout(() => this.pollStatus(), 2000); - } - } catch { - // Backend unreachable: leave the status unknown; the buttons stay usable. - } - })(); + pollModelStatus(() => this.api.status(), (s) => this.modelStatus.set(s)); } // ------------------------------------------------------------------ solution playback diff --git a/src/RLDemo.Web/ClientApp/src/app/game-2048/game-2048.ts b/src/RLDemo.Web/ClientApp/src/app/game-2048/game-2048.ts index 8733279..db216d9 100644 --- a/src/RLDemo.Web/ClientApp/src/app/game-2048/game-2048.ts +++ b/src/RLDemo.Web/ClientApp/src/app/game-2048/game-2048.ts @@ -4,6 +4,7 @@ import { ActivatedRoute } from '@angular/router'; import { Game2048Api, SolveResponse2048, Status2048 } from './game-2048-api'; import { ClassicEngine, RenderTile } from './game-2048-classic'; import { Board, exponentOf } from './game-2048-logic'; +import { pollModelStatus } from '../model-status'; type Mode = 'edit' | 'play' | 'playback'; @@ -313,13 +314,7 @@ export class Game2048 { // ------------------------------------------------------------------ status private pollStatus(): void { - void (async () => { - const status = await this.api.status(); - this.modelStatus.set(status); - if (status.status === 'loading') { - setTimeout(() => this.pollStatus(), 2000); - } - })(); + pollModelStatus(() => this.api.status(), (s) => this.modelStatus.set(s)); } protected playMaxTile = () => this.playMax(); diff --git a/src/RLDemo.Web/ClientApp/src/app/model-status.ts b/src/RLDemo.Web/ClientApp/src/app/model-status.ts new file mode 100644 index 0000000..228dab8 --- /dev/null +++ b/src/RLDemo.Web/ClientApp/src/app/model-status.ts @@ -0,0 +1,25 @@ +/** + * Poll a game's `/status` endpoint until the model stops loading, pushing each reply to a signal. + * + * Hides the self-rescheduling loop and — crucially — swallows a transient fetch failure so a backend + * blip during startup can never surface as an unhandled promise rejection (the bug two of the three + * hand-rolled copies had). Re-schedules only while `status === 'loading'`; a `ready`/`failed` reply + * ends the poll. Generic over the status shape so each game keeps its own typed reply. + */ +export function pollModelStatus( + fetchStatus: () => Promise, + set: (status: T) => void, + intervalMs = 2000, +): void { + void (async () => { + try { + const status = await fetchStatus(); + set(status); + if (status.status === 'loading') { + setTimeout(() => pollModelStatus(fetchStatus, set, intervalMs), intervalMs); + } + } catch { + // Backend unreachable: leave the status as-is; the UI stays usable and a later poll can recover. + } + })(); +} diff --git a/src/RLDemo.Web/ClientApp/src/app/rush-hour/rush-hour.ts b/src/RLDemo.Web/ClientApp/src/app/rush-hour/rush-hour.ts index a726ec0..b47a4e6 100644 --- a/src/RLDemo.Web/ClientApp/src/app/rush-hour/rush-hour.ts +++ b/src/RLDemo.Web/ClientApp/src/app/rush-hour/rush-hour.ts @@ -2,6 +2,7 @@ import { Component, DestroyRef, ElementRef, computed, effect, inject, isDevMode, import { ActivatedRoute } from '@angular/router'; import { AnalyzeResponse, DeckLevel, RushHourApi, SolveResponse, StatusResponse, VehicleDto } from './rush-hour-api'; import { EXIT_ROW, SIZE, canMove, canPlace, initialPositions, isSolved, occupancy } from './rush-hour-logic'; +import { pollModelStatus } from '../model-status'; type Mode = 'edit' | 'play' | 'playback'; type Tool = 'red' | 'red-truck' | 'car-h' | 'car-v' | 'truck-h' | 'truck-v' | 'erase'; @@ -361,13 +362,7 @@ export class RushHour { // ------------------------------------------------------------------ model status private pollStatus(): void { - void (async () => { - const status = await this.api.status(); - this.modelStatus.set(status); - if (status.status === 'loading') { - setTimeout(() => this.pollStatus(), 2000); - } - })(); + pollModelStatus(() => this.api.status(), (s) => this.modelStatus.set(s)); } // ------------------------------------------------------------------ canvas diff --git a/src/RLDemo.Web/Controllers/Game2048Controller.cs b/src/RLDemo.Web/Controllers/Game2048Controller.cs index b0b1502..29422c1 100644 --- a/src/RLDemo.Web/Controllers/Game2048Controller.cs +++ b/src/RLDemo.Web/Controllers/Game2048Controller.cs @@ -24,14 +24,12 @@ public sealed record SolveResponse2048( int MaxTile, bool Reached2048); -public sealed record Status2048Response(string Status, string? Error); - [ApiController] [Route("api/2048")] public sealed class Game2048Controller(Game2048ModelService model, GalleryStore gallery) : ControllerBase { [HttpGet("status")] - public Status2048Response Status() + public StatusResponse Status() { _ = model.Agent; // touch: lazily loads a stored checkpoint return new(model.Status.ToString().ToLowerInvariant(), model.Error); diff --git a/src/RLDemo.Web/Controllers/RushHourController.cs b/src/RLDemo.Web/Controllers/RushHourController.cs index 1af8f5b..5847f58 100644 --- a/src/RLDemo.Web/Controllers/RushHourController.cs +++ b/src/RLDemo.Web/Controllers/RushHourController.cs @@ -7,7 +7,40 @@ namespace RLDemo.Web.Controllers; /// A vehicle as drawn in the browser; index 0 in the array is the red car. public sealed record VehicleDto(int Row, int Col, int Length, bool Horizontal); -public sealed record RushHourBoardDto(VehicleDto[] Vehicles); +public sealed record RushHourBoardDto(VehicleDto[] Vehicles) +{ + /// + /// The single drawn-board → contract (index 0 is the red car): validates the + /// vehicle list and reports a user-facing reason when it is illegal. Shared by the solve/analyze endpoints + /// and the deck store so a board the solver would reject can never be admitted to the curated deck. + /// + public static bool TryBuildPuzzle(VehicleDto[] vehicles, out RushHourPuzzle puzzle, out string? error) + { + puzzle = null!; + error = null; + if (vehicles is not { Length: > 0 }) + { + error = "Draw at least the red car."; + return false; + } + + try + { + puzzle = new RushHourPuzzle([.. vehicles.Select(v => new Vehicle(v.Row, v.Col, v.Length, v.Horizontal))]); + if (puzzle.Vehicles.Any(v => v.Length is < 2 or > 3)) + { + error = "Vehicles must have length 2 (car) or 3 (truck)."; + return false; + } + return true; + } + catch (ArgumentException ex) + { + error = ex.Message; + return false; + } + } +} public sealed record AnalyzeResponse(bool Valid, string? Error, bool Solvable, int OptimalMoves); @@ -25,8 +58,6 @@ public sealed record SolveResponse( TrajectoryStepDto[] OptimalTrajectory, string AiMode); // "greedy" (reactive policy) | "search" (policy-guided A*) | "dqn" (legacy fallback) -public sealed record StatusResponse(string Status, string? Error); - [ApiController] [Route("api/rushhour")] public sealed class RushHourController(RushHourModelService model, GalleryStore gallery) : ControllerBase @@ -42,7 +73,7 @@ public StatusResponse Status() [HttpPost("analyze")] public ActionResult Analyze(RushHourBoardDto board) { - if (TryBuildPuzzle(board, out var puzzle, out string? error)) + if (RushHourBoardDto.TryBuildPuzzle(board.Vehicles, out var puzzle, out string? error)) { int optimal = RushHourSolver.Solve(puzzle); return new AnalyzeResponse(true, null, optimal >= 0, optimal); @@ -54,7 +85,7 @@ public ActionResult Analyze(RushHourBoardDto board) [HttpPost("solve")] public ActionResult Solve(RushHourBoardDto board) { - if (!TryBuildPuzzle(board, out var puzzle, out string? error)) + if (!RushHourBoardDto.TryBuildPuzzle(board.Vehicles, out var puzzle, out string? error)) return BadRequest(new AnalyzeResponse(false, error, false, -1)); var policy = model.PolicyNet; @@ -124,31 +155,4 @@ private static TrajectoryStepDto[] ReplayActions(RushHourPuzzle puzzle, int[] ac } return steps; } - - private static bool TryBuildPuzzle(RushHourBoardDto board, out RushHourPuzzle puzzle, out string? error) - { - puzzle = null!; - error = null; - if (board.Vehicles is not { Length: > 0 }) - { - error = "Draw at least the red car."; - return false; - } - - try - { - puzzle = new RushHourPuzzle([.. board.Vehicles.Select(v => new Vehicle(v.Row, v.Col, v.Length, v.Horizontal))]); - if (puzzle.Vehicles.Any(v => v.Length is < 2 or > 3)) - { - error = "Vehicles must have length 2 (car) or 3 (truck)."; - return false; - } - return true; - } - catch (ArgumentException ex) - { - error = ex.Message; - return false; - } - } } diff --git a/src/RLDemo.Web/Controllers/StatusResponse.cs b/src/RLDemo.Web/Controllers/StatusResponse.cs new file mode 100644 index 0000000..88342b4 --- /dev/null +++ b/src/RLDemo.Web/Controllers/StatusResponse.cs @@ -0,0 +1,8 @@ +namespace RLDemo.Web.Controllers; + +/// +/// The shared model-readiness reply for every game's GET /api/<game>/status: +/// the lowercased plus an optional load error. One shape for all +/// games so the frontend poller and the API tests treat status uniformly. +/// +public sealed record StatusResponse(string Status, string? Error); diff --git a/src/RLDemo.Web/Services/RushHourDeckStore.cs b/src/RLDemo.Web/Services/RushHourDeckStore.cs index 6cbc59b..654c5e0 100644 --- a/src/RLDemo.Web/Services/RushHourDeckStore.cs +++ b/src/RLDemo.Web/Services/RushHourDeckStore.cs @@ -47,7 +47,7 @@ public RushHourDeck Load() public (DeckLevel? Level, string? Error) Upsert(string? id, string? name, VehicleDto[] vehicles) { if (string.IsNullOrWhiteSpace(name)) return (null, "Give the level a name."); - if (!TryBuildPuzzle(vehicles, out var puzzle, out string? error)) return (null, error); + if (!RushHourBoardDto.TryBuildPuzzle(vehicles, out var puzzle, out string? error)) return (null, error); int optimal = RushHourSolver.Solve(puzzle); if (optimal < 0) return (null, "Unsolvable — the red car can never reach the exit."); @@ -82,31 +82,4 @@ private void Write(List levels) File.WriteAllText(temp, JsonSerializer.Serialize(new RushHourDeck(1, levels), Json)); File.Move(temp, FilePath, overwrite: true); } - - // Mirrors RushHourController.TryBuildPuzzle (the drawn-board → puzzle contract): index 0 is the red car. - private static bool TryBuildPuzzle(VehicleDto[] vehicles, out RushHourPuzzle puzzle, out string? error) - { - puzzle = null!; - error = null; - if (vehicles is not { Length: > 0 }) - { - error = "Draw at least the red car."; - return false; - } - try - { - puzzle = new RushHourPuzzle([.. vehicles.Select(v => new Vehicle(v.Row, v.Col, v.Length, v.Horizontal))]); - if (puzzle.Vehicles.Any(v => v.Length is < 2 or > 3)) - { - error = "Vehicles must have length 2 (car) or 3 (truck)."; - return false; - } - return true; - } - catch (ArgumentException ex) - { - error = ex.Message; - return false; - } - } } From 360f257c3c92d4327d29f3fa8413e629516790a5 Mon Sep 17 00:00:00 2001 From: PieterjanDeClippel Date: Sun, 12 Jul 2026 19:38:33 +0200 Subject: [PATCH 03/12] M38 B1: deep web checkpoint modules (RefreshingCheckpoint + StartupCheckpoint) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extract the two copy-paste families in the web model-service layer into ModelServiceInfrastructure.cs and apply them by composition (no shallow ModelService base): - RefreshingCheckpoint: "re-read the checkpoint on a cadence, double-checked lock, keep the previous net on a corrupt/mid-write read" — was copy-pasted 4x (Cube policy/value/efficient + RushHour policy). Corrupt-read error defined out of existence: callers only ever see a usable net or null. Cube's resident-GPU forward rebuild rides the onReload hook, sharing the service lock so it stays serialized against Dispose. - StartupCheckpoint: load-at-startup readiness (Status/Error + lazy Value + Initialize), was hand-rolled 3x. CubeModelService 235->~160, RushHour 109->~50, 2048 43->~24; ModelStatus / IModelStartupService / ModelStartupHostedService relocated into the infra file. Behaviour-preserving (only the model-unavailable warning wording changed). Build 0 errors; 320 fast tests green (Cube/RushHour API + service suites). Co-Authored-By: Claude Opus 4.8 (1M context) --- src/RLDemo.Web/Services/CubeModelService.cs | 235 ++++++------------ .../Services/Game2048ModelService.cs | 43 +--- .../Services/ModelServiceInfrastructure.cs | 131 ++++++++++ .../Services/RushHourModelService.cs | 109 ++------ 4 files changed, 227 insertions(+), 291 deletions(-) create mode 100644 src/RLDemo.Web/Services/ModelServiceInfrastructure.cs diff --git a/src/RLDemo.Web/Services/CubeModelService.cs b/src/RLDemo.Web/Services/CubeModelService.cs index b34a636..f94dccc 100644 --- a/src/RLDemo.Web/Services/CubeModelService.cs +++ b/src/RLDemo.Web/Services/CubeModelService.cs @@ -12,200 +12,89 @@ namespace RLDemo.Web.Services; /// committed to models/ via Git LFS — the web never trains, PRD §14). Holds the device-resident GPU /// forwards for the deep solvers when a CUDA device is present. /// -public sealed class CubeModelService(IModelStore store, AdaptiveBackend backend, ILogger logger) : IModelStartupService, IDisposable +public sealed class CubeModelService : IModelStartupService, IDisposable { public const string EnvironmentId = "cube"; public const string AlgorithmId = "dqn"; + public const string PolicyAlgorithmId = "policy"; + public const string ValueDaviAlgorithmId = "value-davi-res"; + public const string EfficientPolicyAlgorithmId = "policy-efficient"; public const int MaxMoves = 20; + private readonly AdaptiveBackend _backend; + private readonly ILogger _logger; + + // Shared by every refreshing net's reload hook and Dispose, so a resident-GPU rebuild can never race the + // teardown of the forward it rebuilds. private readonly object _lock = new(); - private GreedyQAgent? _agent; + private DeviceResidualMlp? _residentValue; + private DeviceMlp? _residentEfficient; - public ModelStatus Status { get; private set; } = ModelStatus.Loading; - public string? Error { get; private set; } + private readonly StartupCheckpoint _dqn; + private readonly RefreshingCheckpoint _policy; + private readonly RefreshingCheckpoint _value; + private readonly RefreshingCheckpoint _efficient; - /// The greedy inference agent, or null while the model is not ready. Loads lazily from the store. - public GreedyQAgent? Agent + public CubeModelService(IModelStore store, AdaptiveBackend backend, ILogger logger) { - get - { - if (_agent is null && Status == ModelStatus.Loading) TryLoadFromStore(); - return _agent; - } + _backend = backend; + _logger = logger; + + _dqn = new(store, EnvironmentId, AlgorithmId, + stream => new GreedyQAgent(MlpCheckpoint.Load(stream), RubiksCubeEnv.ActionCount), + logger, "Rubik's Cube DQN model"); + _policy = new(store, EnvironmentId, PolicyAlgorithmId, CubePolicyNet.Load, logger, "cube policy net"); + // The value / efficient nets rebuild a resident GPU forward on each reload, under the shared _lock. + _value = new(store, EnvironmentId, ValueDaviAlgorithmId, ResidualMlpCheckpoint.Load, logger, + "cube DAVI value net", onReload: RebuildResidentValue, gate: _lock); + _efficient = new(store, EnvironmentId, EfficientPolicyAlgorithmId, CubePolicyNet.Load, logger, + "EfficientCube policy net", onReload: RebuildResidentEfficient, gate: _lock); } - public const string PolicyAlgorithmId = "policy"; - private CubePolicyNet? _policyNet; - private DateTime _policyLoadedUtc = DateTime.MinValue; - private static readonly TimeSpan PolicyRefresh = TimeSpan.FromMinutes(5); + public ModelStatus Status => _dqn.Status; + public string? Error => _dqn.Error; + + /// The greedy inference agent, or null while the model is not ready. Loads lazily from the store. + public GreedyQAgent? Agent => _dqn.Value; /// - /// The Kociemba-imitation policy/value net (preferred over the DQN when present, PLAN - /// M16), or null if the store has none. Re-read every few minutes so a long-running - /// Lab campaign's improving checkpoints are picked up without restarting the host. + /// The Kociemba-imitation policy/value net (preferred over the DQN when present, PLAN M16), or null if the + /// store has none. Re-read every few minutes so a long-running Lab campaign's improving checkpoints are picked + /// up without restarting the host. /// - public CubePolicyNet? PolicyNet - { - get - { - if (DateTime.UtcNow - _policyLoadedUtc < PolicyRefresh) return _policyNet; - lock (_lock) - { - if (DateTime.UtcNow - _policyLoadedUtc < PolicyRefresh) return _policyNet; - try - { - using var stream = store.TryOpenRead(EnvironmentId, PolicyAlgorithmId); - if (stream is not null) - { - _policyNet = CubePolicyNet.Load(stream); - logger.LogInformation("Loaded cube policy net from the store."); - } - } - catch (Exception ex) - { - // A mid-write or corrupt checkpoint must not break solving; keep the previous net. - logger.LogWarning(ex, "Failed to (re)load the cube policy net; keeping the previous one."); - } - _policyLoadedUtc = DateTime.UtcNow; - return _policyNet; - } - } - } - - public const string ValueDaviAlgorithmId = "value-davi-res"; - private ResidualMlp? _valueNet; - private DeviceResidualMlp? _residentValue; - private DateTime _valueLoadedUtc = DateTime.MinValue; + public CubePolicyNet? PolicyNet => _policy.Current; /// - /// The teacher-free DAVI value net (the "self-taught AI" shortest-move solver, PLAN M21), or null - /// if the store has none. Re-read on the same cadence as so an improving - /// Lab campaign's checkpoints are picked up without restarting the host. Accessing this also keeps + /// The teacher-free DAVI value net (the "self-taught AI" shortest-move solver, PLAN M21), or null if the store + /// has none. Re-read on the same cadence as ; accessing it keeps /// in sync (rebuilt on the GPU whenever the net reloads). /// - public ResidualMlp? ValueNet - { - get - { - if (DateTime.UtcNow - _valueLoadedUtc < PolicyRefresh) return _valueNet; - lock (_lock) - { - if (DateTime.UtcNow - _valueLoadedUtc < PolicyRefresh) return _valueNet; - try - { - using var stream = store.TryOpenRead(EnvironmentId, ValueDaviAlgorithmId); - if (stream is not null) - { - _valueNet = ResidualMlpCheckpoint.Load(stream); - // Mirror the freshly-loaded weights onto the GPU as a resident forward when a CUDA - // device is present — the host-span path is transfer-bound and barely beats CPU, so - // the resident forward (weights on device) is what makes the deep solver fast. - _residentValue?.Dispose(); - _residentValue = backend.Gpu is { } gpu ? gpu.CreateResidentForward(_valueNet) : null; - logger.LogInformation("Loaded cube DAVI value net from the store ({Mode}).", - _residentValue is not null ? "resident GPU forward" : "CPU forward"); - } - } - catch (Exception ex) - { - // A mid-write or corrupt checkpoint must not break solving; keep the previous net. - logger.LogWarning(ex, "Failed to (re)load the cube DAVI value net; keeping the previous one."); - } - _valueLoadedUtc = DateTime.UtcNow; - return _valueNet; - } - } - } + public ResidualMlp? ValueNet => _value.Current; /// - /// A device-resident GPU forward over the current value net, or null when no CUDA device is present - /// (the solver then scores on the CPU). Kept in sync by the getter — read that - /// first. Thread-safe: the underlying forward serializes GPU access internally. + /// A device-resident GPU forward over the current value net, or null when no CUDA device is present (the solver + /// then scores on the CPU). Kept in sync by the getter — read that first. Thread-safe: + /// the underlying forward serializes GPU access internally. /// public CubeValueSearch.BatchForward? ResidentValueForward => _residentValue is { } dev ? dev.Forward : null; - public const string EfficientPolicyAlgorithmId = "policy-efficient"; - private CubePolicyNet? _efficientNet; - private DeviceMlp? _residentEfficient; - private DateTime _efficientLoadedUtc = DateTime.MinValue; - /// /// The teacher-free EfficientCube policy net (the website's "self-taught AI", trained self-supervised on - /// scramble reversals — no Kociemba, no value-iteration bootstrap), or null if the store has none. Solved - /// by beam search (). - /// Re-read on the same cadence as the other nets so an improving Lab campaign's checkpoints are picked up - /// without restarting the host; accessing this keeps in sync (the - /// policy head's weights mirrored onto the GPU whenever the net reloads). + /// scramble reversals — no Kociemba, no value-iteration bootstrap), or null if the store has none. Solved by + /// beam search (). + /// Re-read on the same cadence as the other nets; accessing it keeps in + /// sync (the policy head's weights mirrored onto the GPU whenever the net reloads). /// - public CubePolicyNet? EfficientPolicyNet - { - get - { - if (DateTime.UtcNow - _efficientLoadedUtc < PolicyRefresh) return _efficientNet; - lock (_lock) - { - if (DateTime.UtcNow - _efficientLoadedUtc < PolicyRefresh) return _efficientNet; - try - { - using var stream = store.TryOpenRead(EnvironmentId, EfficientPolicyAlgorithmId); - if (stream is not null) - { - _efficientNet = CubePolicyNet.Load(stream); - // Beam search runs the bulk of the forwards; mirror the policy head onto the GPU as a - // resident forward when a CUDA device is present (host-span is transfer-bound). - _residentEfficient?.Dispose(); - _residentEfficient = backend.Gpu is { } gpu ? gpu.CreateResidentForward(_efficientNet.PolicyAsMlp()) : null; - logger.LogInformation("Loaded EfficientCube policy net from the store ({Mode}).", - _residentEfficient is not null ? "resident GPU forward" : "CPU forward"); - } - } - catch (Exception ex) - { - // A mid-write or corrupt checkpoint must not break solving; keep the previous net. - logger.LogWarning(ex, "Failed to (re)load the EfficientCube policy net; keeping the previous one."); - } - _efficientLoadedUtc = DateTime.UtcNow; - return _efficientNet; - } - } - } + public CubePolicyNet? EfficientPolicyNet => _efficient.Current; /// - /// A device-resident GPU forward over the current EfficientCube policy head (rows × 12 logits), or null when - /// no CUDA device is present (beam search then scores on the CPU). Kept in sync by the + /// A device-resident GPU forward over the current EfficientCube policy head (rows × 12 logits), or null when no + /// CUDA device is present (beam search then scores on the CPU). Kept in sync by the /// getter — read that first. Thread-safe: the forward serializes GPU access. /// public Func? ResidentEfficientForward => _residentEfficient is { } dev ? dev.Forward : null; - public bool TryLoadFromStore() - { - lock (_lock) - { - if (_agent is not null) return true; - using var stream = store.TryOpenRead(EnvironmentId, AlgorithmId); - if (stream is null) return false; - var network = MlpCheckpoint.Load(stream); - _agent = new GreedyQAgent(network, RubiksCubeEnv.ActionCount); - Status = ModelStatus.Ready; - logger.LogInformation("Loaded Rubik's Cube model from the store."); - return true; - } - } - - /// Loads the shipped DQN baseline at startup. The web does not train (PRD §14); the deep solver - /// nets (policy / DAVI value / EfficientCube) load lazily through their getters. - public void Initialize(CancellationToken cancellationToken) - { - if (TryLoadFromStore()) return; - lock (_lock) - { - Status = ModelStatus.Failed; - Error = "No trained Rubik's Cube model in the store."; - } - logger.LogWarning("No Rubik's Cube DQN model in the store — train it on a dev machine and commit the checkpoint to models/ (Git LFS)."); - } - /// /// Greedy rollout on a drawn cube: up to quarter-turns under /// the same no-undo mask the agent was trained with, reported honestly whether they @@ -230,6 +119,30 @@ public static (bool Solved, List Moves) Rollout(GreedyQAgent agent, Face return (cube.IsSolved, moves); } + /// Loads the shipped DQN baseline at startup. The web does not train (PRD §14); the deep solver nets + /// (policy / DAVI value / EfficientCube) load lazily through their getters. + public void Initialize(CancellationToken cancellationToken) => _dqn.Initialize(); + + // Mirror the freshly-loaded value-net weights onto the GPU as a resident forward when a CUDA device is present: + // the host-span path is transfer-bound and barely beats CPU, so the resident forward (weights on device) is what + // makes the deep solver fast. Runs under _lock (RefreshingCheckpoint holds the shared gate), serialized vs Dispose. + private void RebuildResidentValue(ResidualMlp net) + { + _residentValue?.Dispose(); + _residentValue = _backend.Gpu is { } gpu ? gpu.CreateResidentForward(net) : null; + _logger.LogInformation("Rebuilt cube DAVI value forward ({Mode}).", + _residentValue is not null ? "resident GPU" : "CPU"); + } + + // Beam search runs the bulk of the forwards; mirror the policy head onto the GPU when a device is present. + private void RebuildResidentEfficient(CubePolicyNet net) + { + _residentEfficient?.Dispose(); + _residentEfficient = _backend.Gpu is { } gpu ? gpu.CreateResidentForward(net.PolicyAsMlp()) : null; + _logger.LogInformation("Rebuilt EfficientCube policy forward ({Mode}).", + _residentEfficient is not null ? "resident GPU" : "CPU"); + } + public void Dispose() { lock (_lock) diff --git a/src/RLDemo.Web/Services/Game2048ModelService.cs b/src/RLDemo.Web/Services/Game2048ModelService.cs index 75db53e..f228bc8 100644 --- a/src/RLDemo.Web/Services/Game2048ModelService.cs +++ b/src/RLDemo.Web/Services/Game2048ModelService.cs @@ -13,45 +13,14 @@ public sealed class Game2048ModelService(IModelStore store, ILogger _model = + new(store, EnvironmentId, AlgorithmId, NTuple2048Agent.Load, logger, "2048 n-tuple model"); - public ModelStatus Status { get; private set; } = ModelStatus.Loading; - public string? Error { get; private set; } + public ModelStatus Status => _model.Status; + public string? Error => _model.Error; /// The trained agent, or null while not ready. Loads lazily from the store. - public NTuple2048Agent? Agent - { - get - { - if (_agent is null && Status == ModelStatus.Loading) TryLoadFromStore(); - return _agent; - } - } + public NTuple2048Agent? Agent => _model.Value; - public bool TryLoadFromStore() - { - lock (_lock) - { - if (_agent is not null) return true; - using var stream = store.TryOpenRead(EnvironmentId, AlgorithmId); - if (stream is null) return false; - _agent = NTuple2048Agent.Load(stream); - Status = ModelStatus.Ready; - logger.LogInformation("Loaded 2048 n-tuple model from the store."); - return true; - } - } - - /// Loads the shipped checkpoint at startup. The web does not train (PRD §14). - public void Initialize(CancellationToken cancellationToken) - { - if (TryLoadFromStore()) return; - lock (_lock) - { - Status = ModelStatus.Failed; - Error = "No trained 2048 model in the store."; - } - logger.LogWarning("No 2048 model in the store — game unavailable. Train it on a dev machine and commit the checkpoint to models/ (Git LFS)."); - } + public void Initialize(CancellationToken cancellationToken) => _model.Initialize(); } diff --git a/src/RLDemo.Web/Services/ModelServiceInfrastructure.cs b/src/RLDemo.Web/Services/ModelServiceInfrastructure.cs new file mode 100644 index 0000000..4339703 --- /dev/null +++ b/src/RLDemo.Web/Services/ModelServiceInfrastructure.cs @@ -0,0 +1,131 @@ +using MintPlayer.AI.ReinforcementLearning.Core.Checkpoints; + +namespace RLDemo.Web.Services; + +public enum ModelStatus { Loading, Ready, Failed } + +/// +/// A game service with startup work (load its trained checkpoint from the store, or warm a cache). The web does +/// NOT train: models are produced on a dev machine (the Lab campaigns / Console) and committed to models/ +/// via Git LFS, so a single training path lives off the request path. A missing checkpoint = the game is +/// unavailable, not a trigger to train in-process. +/// +public interface IModelStartupService +{ + void Initialize(CancellationToken cancellationToken); +} + +/// Runs every game's startup work (load checkpoint / warm caches) once, in parallel, off the request path. +public sealed class ModelStartupHostedService(IEnumerable models) : BackgroundService +{ + protected override Task ExecuteAsync(CancellationToken stoppingToken) + => Task.WhenAll(models.Select(m => Task.Run(() => m.Initialize(stoppingToken), stoppingToken))); +} + +/// +/// A trained model loaded once from the store and then served read-only for a game's lifetime: it owns the +/// readiness snapshot ( / ) behind the /status endpoint, loads +/// lazily on first access (or eagerly at startup via ), and turns a missing checkpoint +/// into "unavailable" rather than an exception (the web never trains — the checkpoint ships in models/). +/// Composed into a game's model service, so one service can hold several (e.g. a DQN baseline alongside the +/// refreshing deep-solver nets); the load recipe is the only per-game knowledge, passed as . +/// +public sealed class StartupCheckpoint( + IModelStore store, string environmentId, string algorithmId, Func load, + ILogger logger, string modelName) where T : class +{ + private readonly object _lock = new(); + private T? _value; + + public ModelStatus Status { get; private set; } = ModelStatus.Loading; + public string? Error { get; private set; } + + /// The loaded model, or null while unavailable. Loads lazily from the store on first access. + public T? Value + { + get + { + if (_value is null && Status == ModelStatus.Loading) TryLoad(); + return _value; + } + } + + /// Loads the stored checkpoint if one exists; safe to call any time. Returns whether a model is loaded. + public bool TryLoad() + { + lock (_lock) + { + if (_value is not null) return true; + using var stream = store.TryOpenRead(environmentId, algorithmId); + if (stream is null) return false; + _value = load(stream); + Status = ModelStatus.Ready; + logger.LogInformation("Loaded {ModelName} from the store.", modelName); + return true; + } + } + + /// Startup: load the shipped checkpoint, or mark the game unavailable and warn how to provide one. + public void Initialize() + { + if (TryLoad()) return; + lock (_lock) + { + Status = ModelStatus.Failed; + Error = $"No trained {modelName} in the store."; + } + logger.LogWarning( + "No {ModelName} in the store — game unavailable. Train it on a dev machine and commit the checkpoint to models/ (Git LFS).", + modelName); + } +} + +/// +/// A single trained net kept fresh from the model store: the getter re-reads the checkpoint +/// on a cadence so a long-running Lab campaign's improving weights are picked up without restarting the host, +/// while a corrupt or mid-write read is swallowed and the previous net kept — the caller only ever sees a usable +/// net or null, never an exception (the error is defined out of existence). The re-read is double-checked against +/// a lock so concurrent readers do at most one load per cadence. runs under that lock +/// right after a successful load — Cube uses it to mirror the fresh weights onto a resident GPU forward, and +/// passes its own service lock as so the rebuild serializes against Dispose. +/// +public sealed class RefreshingCheckpoint( + IModelStore store, string environmentId, string algorithmId, Func load, + ILogger logger, string name, TimeSpan? refresh = null, Action? onReload = null, object? gate = null) + where T : class +{ + private readonly object _lock = gate ?? new object(); + private readonly TimeSpan _refresh = refresh ?? TimeSpan.FromMinutes(5); + private T? _value; + private DateTime _loadedUtc = DateTime.MinValue; + + /// The current net, or null until the store first has one. Re-reads on the cadence; never throws. + public T? Current + { + get + { + if (DateTime.UtcNow - _loadedUtc < _refresh) return _value; + lock (_lock) + { + if (DateTime.UtcNow - _loadedUtc < _refresh) return _value; + try + { + using var stream = store.TryOpenRead(environmentId, algorithmId); + if (stream is not null) + { + _value = load(stream); + onReload?.Invoke(_value); + logger.LogInformation("Loaded {Name} from the store.", name); + } + } + catch (Exception ex) + { + // A mid-write or corrupt checkpoint must not break solving; keep the previous net. + logger.LogWarning(ex, "Failed to (re)load {Name}; keeping the previous one.", name); + } + _loadedUtc = DateTime.UtcNow; + return _value; + } + } + } +} diff --git a/src/RLDemo.Web/Services/RushHourModelService.cs b/src/RLDemo.Web/Services/RushHourModelService.cs index 699a94f..1364021 100644 --- a/src/RLDemo.Web/Services/RushHourModelService.cs +++ b/src/RLDemo.Web/Services/RushHourModelService.cs @@ -5,19 +5,6 @@ namespace RLDemo.Web.Services; -public enum ModelStatus { Loading, Ready, Failed } - -/// -/// A game service with startup work (load its trained checkpoint from the store, or warm a cache). The web does -/// NOT train: models are produced on a dev machine (the Lab campaigns / Console) and committed to models/ -/// via Git LFS, so a single training path lives off the request path. A missing checkpoint = the game is -/// unavailable, not a trigger to train in-process. -/// -public interface IModelStartupService -{ - void Initialize(CancellationToken cancellationToken); -} - /// /// Owns the Rush Hour DQN: loads it from the model store at startup (the checkpoint is trained on a dev machine /// and shipped in models/ via Git LFS — the web never trains, PRD §14). Also serves the imitation policy @@ -27,94 +14,30 @@ public sealed class RushHourModelService(IModelStore store, ILogger _dqn = new( + store, EnvironmentId, AlgorithmId, + stream => new GreedyQAgent(MlpCheckpoint.Load(stream), RushHourBoard.ActionCount), + logger, "Rush Hour model"); - public ModelStatus Status { get; private set; } = ModelStatus.Loading; - public string? Error { get; private set; } + private readonly RefreshingCheckpoint _policy = new( + store, EnvironmentId, PolicyAlgorithmId, RushHourPolicyNet.Load, logger, "Rush Hour policy net"); - /// The greedy inference agent, or null while the model is not ready. Loads lazily from the store. - public GreedyQAgent? Agent - { - get - { - if (_agent is null && Status == ModelStatus.Loading) TryLoadFromStore(); - return _agent; - } - } + public ModelStatus Status => _dqn.Status; + public string? Error => _dqn.Error; - public const string PolicyAlgorithmId = "policy"; - private RushHourPolicyNet? _policyNet; - private DateTime _policyLoadedUtc = DateTime.MinValue; - private static readonly TimeSpan PolicyRefresh = TimeSpan.FromMinutes(5); + /// The greedy inference agent, or null while the model is not ready. Loads lazily from the store. + public GreedyQAgent? Agent => _dqn.Value; /// - /// The imitation-learned policy/value net (preferred over the DQN when present), or - /// null if the store has none. Re-read every few minutes so a long-running training - /// campaign's improving checkpoints are picked up without restarting the host. + /// The imitation-learned policy/value net (preferred over the DQN when present), or null if the store has + /// none. Re-read every few minutes so a long-running training campaign's improving checkpoints are picked up + /// without restarting the host. /// - public RushHourPolicyNet? PolicyNet - { - get - { - if (DateTime.UtcNow - _policyLoadedUtc < PolicyRefresh) return _policyNet; - lock (_lock) - { - if (DateTime.UtcNow - _policyLoadedUtc < PolicyRefresh) return _policyNet; - try - { - using var stream = store.TryOpenRead(EnvironmentId, PolicyAlgorithmId); - if (stream is not null) - { - _policyNet = RushHourPolicyNet.Load(stream); - logger.LogInformation("Loaded Rush Hour policy net from the store."); - } - } - catch (Exception ex) - { - // A mid-write or corrupt checkpoint must not break solving; keep the previous net. - logger.LogWarning(ex, "Failed to (re)load the Rush Hour policy net; keeping the previous one."); - } - _policyLoadedUtc = DateTime.UtcNow; - return _policyNet; - } - } - } - - /// Loads a stored checkpoint if one exists; safe to call at any time. - public bool TryLoadFromStore() - { - lock (_lock) - { - if (_agent is not null) return true; - using var stream = store.TryOpenRead(EnvironmentId, AlgorithmId); - if (stream is null) return false; - var network = MlpCheckpoint.Load(stream); - _agent = new GreedyQAgent(network, RushHourBoard.ActionCount); - Status = ModelStatus.Ready; - logger.LogInformation("Loaded Rush Hour model from the store."); - return true; - } - } + public RushHourPolicyNet? PolicyNet => _policy.Current; /// Loads the shipped checkpoint at startup. The web does not train (PRD §14). - public void Initialize(CancellationToken cancellationToken) - { - if (TryLoadFromStore()) return; - lock (_lock) - { - Status = ModelStatus.Failed; - Error = "No trained Rush Hour model in the store."; - } - logger.LogWarning("No Rush Hour model in the store — game unavailable. Train it on a dev machine (Lab/Console) and commit the checkpoint to models/ (Git LFS)."); - } -} - -/// Runs every game's startup work (load checkpoint / warm caches) once, in parallel, off the request path. -public sealed class ModelStartupHostedService(IEnumerable models) : BackgroundService -{ - protected override Task ExecuteAsync(CancellationToken stoppingToken) - => Task.WhenAll(models.Select(m => Task.Run(() => m.Initialize(stoppingToken), stoppingToken))); + public void Initialize(CancellationToken cancellationToken) => _dqn.Initialize(); } From a4c3d79c5c1a7f05ee136cce95ec6eed694cfac5 Mon Sep 17 00:00:00 2001 From: PieterjanDeClippel Date: Sun, 12 Jul 2026 19:52:08 +0200 Subject: [PATCH 04/12] =?UTF-8?q?M38=20B2:=20DqnScoreCampaign=20base=20?= =?UTF-8?q?=E2=80=94=20absorb=20the=20M36/M37=20duplication?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Re-cut the shared score-maximizing DQN spine (the change PR #27 predated). Beyond the resume / per-chunk-train / keep-best / checkpoint lifecycle, the base now OWNS the two features that landed after PR #27 and had been copy- pasted per game: - Progressive net growth (M37): grow/growEvery + _growRng + DqnGrowth.Maybe inside the base TrainChunk — removed from both subclasses. - Live telemetry (M36): the base implements INetworkTelemetrySource itself (SnapshotParameters/Sample/SampleIo/SampleActivations), reading the current net + gate + targetSteps it already owns — the ~90-line twin is gone from both subclasses, which now only supply ObservationSize + input/output labels. Snake 213->110, FruitCake 236->110; base is 190 lines. Subclasses supply env, BaseOptions, EvaluateNet, labels, and (FruitCake) AdaptWarmNet (plain->noisy + GrowInput) + noisy StateId. Behaviour-preserving, PROVEN bitwise-identical: fresh fixed-seed `--game snake` and `--game fruitcake` runs (6000 steps, seed 1) produce SHA256-equal snake.dqn.ckpt / snake.dqn-state.ckpt / fruitcake.dqn.ckpt / fruitcake.dqn-state.ckpt vs the pre-refactor build (deployable net AND full resume state). 320 fast tests green. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../DqnScoreCampaign.cs | 190 +++++++++++++++++ .../FruitCakeDqnCampaign.cs | 198 ++++-------------- .../SnakeDqnCampaign.cs | 173 ++------------- 3 files changed, 248 insertions(+), 313 deletions(-) create mode 100644 tools/MintPlayer.AI.ReinforcementLearning.Lab/DqnScoreCampaign.cs diff --git a/tools/MintPlayer.AI.ReinforcementLearning.Lab/DqnScoreCampaign.cs b/tools/MintPlayer.AI.ReinforcementLearning.Lab/DqnScoreCampaign.cs new file mode 100644 index 0000000..ac641c9 --- /dev/null +++ b/tools/MintPlayer.AI.ReinforcementLearning.Lab/DqnScoreCampaign.cs @@ -0,0 +1,190 @@ +using MintPlayer.AI.ReinforcementLearning.Core.Checkpoints; +using MintPlayer.AI.ReinforcementLearning.Core.Environments; +using MintPlayer.AI.ReinforcementLearning.Core.Nn; +using MintPlayer.AI.ReinforcementLearning.Core.Numerics; +using MintPlayer.AI.ReinforcementLearning.Core.Random; +using MintPlayer.AI.ReinforcementLearning.Core.Telemetry; +using MintPlayer.AI.ReinforcementLearning.Core.Training; + +/// +/// The shared spine of a score-maximizing DQN campaign (Snake, FruitCake): an open-ended game whose eval is +/// a mean episodic score, not a solve rate. It owns everything these campaigns do identically — resume-or-warm-start +/// from the model store, per-chunk training against a raised absolute , the +/// keep-best gate on the deployable net (DQN eval is noisy, so the shipped net is only overwritten when an eval +/// beats the best seen), the full-state checkpoint, progressive net growth (, PLAN M37), and +/// the live-telemetry seam (, PLAN M36) a viewer samples while it trains. +/// +/// A subclass supplies only what genuinely differs: its env + DQN hyperparameters (), how +/// to evaluate the net (), the neuron labels, and — for a net whose warm-start needs +/// adaptation (plain→noisy, grow-input) — . Resume/train/checkpoint stay byte-identical +/// to the hand-rolled campaigns, so a run is bitwise-reproducible across the refactor. +/// +/// +internal abstract class DqnScoreCampaign(ulong seed, int chunkSteps, long targetSteps, float learningRate, bool grow, int growEvery) + : ITrainingCampaign, INetworkTelemetrySource +{ + protected const string NetId = "dqn"; // deployable DuelingQNet — the id the web loads + + // Progressive-growth demo (DqnGrowth): grow the net wider+deeper on the growEvery cadence. Seeded independently + // of the training streams so growth is deterministic and never perturbs the trainer's RNG. + private readonly Xoshiro256StarStar _growRng = new(seed ^ 0x9E3779B97F4A7C15UL); + protected readonly SeedSequence Seeds = new(seed); + + protected DqnTrainingState? State; + private IValueNet? _warmNet; // deployable net to warm-start from when there's no full resume state + // Save-best: the deployable net is only overwritten when the gate metric IMPROVES on the best seen (seeded from + // the starting net, so a noisy dip never regresses a good shipped model). The resume state tracks the latest net. + private double _bestGate = double.NegativeInfinity; + private double _lastGate = double.NegativeInfinity; + + // ── Game-specific surface ────────────────────────────────────────────────────────────────────────────────── + public abstract string Environment { get; } + /// Model-store id for the full resume state (default "dqn-state"; a separate line, e.g. noisy, overrides). + protected virtual string StateId => "dqn-state"; + protected abstract IEnvironment TrainEnv { get; } + protected abstract DqnOptions BaseOptions { get; } + /// The progress unit shown in logs/metrics ("steps" / "drops"). + protected abstract string StepNoun { get; } + /// The gate metric's label for logs ("food@12" / "mean score"). + protected abstract string GateLabel { get; } + /// Human display name for the campaign ("Snake DQN" / "FruitCake DQN"). + protected abstract string DisplayName { get; } + /// Optional suffix on the fresh-start log (e.g. " (train 6×6, eval 12×12)"); empty by default. + protected virtual string FreshStartDetail => ""; + /// The net's observation width, used to guard the telemetry forward pass. + protected abstract int ObservationSize { get; } + protected abstract IReadOnlyList? InputLabels { get; } + protected abstract IReadOnlyList? OutputLabels { get; } + /// Adapt a freshly-loaded deployable net before warm-starting; identity by default (FruitCake promotes + /// plain→noisy and grows the input to fit an enriched observation). + protected virtual IValueNet AdaptWarmNet(DuelingQNet loaded) => loaded; + /// Run the greedy eval and return the keep-best Gate value plus the game-specific middle metrics + /// and a summary body; the base frames them with the step count and loss. + protected abstract (double Gate, IReadOnlyList Metrics, string Summary) EvaluateNet(IValueNet net); + + /// The net being trained/served: the live online net, else the warm-start net (for eval-only runs). + protected IValueNet? CurrentNet => State?.Online ?? _warmNet; + + public bool Resume(IModelStore store) + { + bool resumed = false; + using (var s = store.TryOpenRead(Environment, StateId)) + { + if (s is not null) + { + State = DqnTrainingState.Load(s); + Log($"resumed {DisplayName} at {State.StepsCompleted:N0} {StepNoun} (last eval return {State.LastEval:F2})"); + resumed = true; + } + } + // No full resume state, but the deployable net may be present (the shipped checkpoint). Warm-start from it: a + // fresh optimizer/replay buffer continues the trained net rather than discarding its weights. + if (!resumed) + { + using var net = store.TryOpenRead(Environment, NetId); + if (net is not null) + { + _warmNet = AdaptWarmNet(DuelingQNetCheckpoint.Load(net)); + Log($"warm-starting from the deployable {DisplayName} net '{NetId}' (fresh optimizer + replay buffer)"); + resumed = true; + } + } + + // Seed save-best from the starting net's score so training only ever ships a net that BEATS it. + var startNet = CurrentNet; + if (startNet is not null) + { + _bestGate = EvaluateNet(startNet).Gate; + Log($"baseline {GateLabel}: {_bestGate:F2} (will only re-save the deployable net when an eval beats this)"); + } + else + { + Log($"starting fresh {DisplayName} training{FreshStartDetail}"); + } + return resumed; + } + + public long TrainChunk() + { + int from = State?.StepsCompleted ?? 0; + int to = from + chunkSteps; + if (targetSteps > 0) to = (int)Math.Min(to, targetSteps); + // EvalEvery == the chunk size so the trainer's own eval fires at most once per chunk — the campaign's + // authoritative eval is Evaluate(). MaxSteps is ABSOLUTE: resuming raises the ceiling. + var options = BaseOptions with { MaxSteps = to, EvalEvery = Math.Max(1, chunkSteps) }; + // First chunk: resume the full state if present, else warm-start from the deployable net (if any). + var result = DqnTrainer.Train(TrainEnv, options, Seeds, resume: State, warmStart: State is null ? _warmNet : null); + State = result.State; + State = DqnGrowth.Maybe(State, grow, growEvery, learningRate, _growRng, Log); + return State.StepsCompleted; + } + + /// Score-maximizing: no hard goal. Stops on the runner's time budget, or an optional absolute step cap. + public bool IsComplete => targetSteps > 0 && (State?.StepsCompleted ?? 0) >= targetSteps; + + public CampaignEval Evaluate() + { + int steps = State?.StepsCompleted ?? 0; + // Eval the trained net if we've trained this run, else the warm-start net (so eval-only works on the shipped + // checkpoint too); only "no model at all" yields the placeholder. + var net = CurrentNet; + if (net is null) + return new CampaignEval([new(StepNoun, 0, "0")], "no model yet (train first)"); + + var (gate, metrics, summary) = EvaluateNet(net); + _lastGate = gate; // Checkpoint() ships the net only when this beats the best seen + float loss = State?.LastLoss ?? 0f; + + var full = new List(metrics.Count + 2) { new(StepNoun, steps, "0") }; + full.AddRange(metrics); + full.Add(new("loss", loss, "F4")); + return new CampaignEval(full, $"{StepNoun} {steps:N0} | {summary} | loss {loss:F4}"); + } + + public void Checkpoint(IModelStore store) + { + if (State is null) return; + // The resume state always tracks the latest net (so a continuation picks up where it left off). + store.Save(Environment, StateId, s => State.Save(s)); + // The DEPLOYABLE net is save-best: only overwrite it when this eval beat the best seen (DQN eval is noisy). + if (_lastGate > _bestGate) + { + _bestGate = _lastGate; + store.Save(Environment, NetId, s => DuelingQNetCheckpoint.Save((DuelingQNet)State.Online, s)); + Log($"new best {GateLabel} {_bestGate:F2} → saved deployable net '{NetId}'"); + } + } + + public virtual void Dispose() { } + + // ── Live telemetry (INetworkTelemetrySource): read-only; a viewer samples the current net as it trains. ────── + string INetworkTelemetrySource.NetKind => "dueling-q"; + IReadOnlyList? INetworkTelemetrySource.SnapshotParameters() + => CurrentNet is { } net ? [.. net.Parameters()] : null; + NetworkMetrics INetworkTelemetrySource.Sample() + => new(State?.StepsCompleted ?? 0, targetSteps, State?.LastLoss ?? double.NaN, _lastGate, double.NaN); + IReadOnlyList? INetworkTelemetrySource.InputLabels => InputLabels; + IReadOnlyList? INetworkTelemetrySource.OutputLabels => OutputLabels; + (float[] Input, float[] Output)? INetworkTelemetrySource.SampleIo() + { + var obs = State?.CurrentObs; + if (CurrentNet is not { } net || obs is null || obs.Length != ObservationSize) return null; + try + { + // Forward the most-recent observation for the net's current per-action Q-values. Read-only (no Backward), + // so a concurrent training step is unaffected; a fresh array so the viewer never sees it mutated. + var input = (float[])obs.Clone(); + return (input, net.Forward(new Tensor(input, 1, input.Length)).Data.AsSpan().ToArray()); + } + catch { return null; } + } + float[][]? INetworkTelemetrySource.SampleActivations() + { + var obs = State?.CurrentObs; + if (CurrentNet is not DuelingQNet dqn || obs is null || obs.Length != ObservationSize) return null; + try { return dqn.LayerActivations(new Tensor((float[])obs.Clone(), 1, obs.Length)); } + catch { return null; } + } + + protected static void Log(string message) => Console.WriteLine($"{DateTime.UtcNow:HH:mm:ss} {message}"); +} diff --git a/tools/MintPlayer.AI.ReinforcementLearning.Lab/FruitCakeDqnCampaign.cs b/tools/MintPlayer.AI.ReinforcementLearning.Lab/FruitCakeDqnCampaign.cs index 9a7d967..0ec62fa 100644 --- a/tools/MintPlayer.AI.ReinforcementLearning.Lab/FruitCakeDqnCampaign.cs +++ b/tools/MintPlayer.AI.ReinforcementLearning.Lab/FruitCakeDqnCampaign.cs @@ -1,48 +1,40 @@ -using MintPlayer.AI.ReinforcementLearning.Core.Checkpoints; +using MintPlayer.AI.ReinforcementLearning.Core.Environments; using MintPlayer.AI.ReinforcementLearning.Core.Nn; -using MintPlayer.AI.ReinforcementLearning.Core.Numerics; using MintPlayer.AI.ReinforcementLearning.Core.Random; using MintPlayer.AI.ReinforcementLearning.Core.Schedules; -using MintPlayer.AI.ReinforcementLearning.Core.Telemetry; using MintPlayer.AI.ReinforcementLearning.Core.Training; using MintPlayer.AI.ReinforcementLearning.Environments.FruitCake; /// -/// FruitCake DQN campaign (`--game fruitcake`, PRD FRUITCAKE_AI §4.4/§7-A2) as an -/// on — the score-maximizing paradigm (an open-ended game whose eval is the mean -/// episodic score, not a solve rate). Trains a Double+Dueling on the headless -/// (one step = one drop, simulated to rest in pure compute; rotation off). Resumable -/// bitwise-identically via ; each chunk raises the absolute -/// and continues. Persists the deployable net under `fruitcake`/`dqn` (the id -/// the web's FruitCakeModelService will load, A3) plus the full resume state under `fruitcake`/`dqn-state`. +/// FruitCake DQN campaign (`--game fruitcake`, PRD FRUITCAKE_AI §4.4/§7-A2) on the shared +/// spine — the score-maximizing paradigm (an open-ended game whose eval is the mean +/// episodic score). Trains a Double+Dueling on the headless +/// (one step = one drop, simulated to rest in pure compute; rotation off). This type supplies the env + reward +/// shaping, the hyperparameters, the mean-score eval, and the plain→noisy / grow-input warm-net adaptation; the +/// base owns resume/train/keep-best/checkpoint/growth/telemetry. /// internal sealed class FruitCakeDqnCampaign(ulong seed, int chunkSteps, long targetSteps, int evalEpisodes, float learningRate, float epsilonStart, int[] hidden, double gamma, bool noisy = false, int nStep = 1, bool shapeRewards = false, bool grow = false, int growEvery = 2000) - : ITrainingCampaign, INetworkTelemetrySource + : DqnScoreCampaign(seed, chunkSteps, targetSteps, learningRate, grow, growEvery) { - // Progressive-growth demo (shared with Snake via DqnGrowth): grow the net wider+deeper on the growEvery cadence. - private readonly Xoshiro256StarStar _growRng = new(seed ^ 0x9E3779B97F4A7C15UL); - private const string NetId = "dqn"; // deployable DuelingQNet — the id the web loads (shared by both lines) - // Noisy training is a SEPARATE line with its own resume state: it must never resume a PLAIN state as a plain - // net (which would then train with ε=0 and NO exploration). The deployable NetId is shared and keep-best gated. - private string StateId => noisy ? "dqn-noisy-state" : "dqn-state"; - - // Training env carries the reward shaping (ShapingGamma matches the learner's γ for policy-invariance); the - // eval env stays a plain game so keep-best/A/B always judge real merge points, never the shaped signal. + // Training env carries the reward shaping (ShapingGamma matches the learner's γ for policy-invariance); the eval + // env stays a plain game so keep-best/A/B always judge real merge points, never the shaped signal. private readonly FruitCakeEnv _env = new() { ShapeRewards = shapeRewards, ShapingGamma = gamma }; private readonly FruitCakeEnv _evalEnv = new(); - private readonly SeedSequence _seeds = new(seed); - - private DqnTrainingState? _state; - private IValueNet? _warmNet; // deployable net to warm-start from when there's no full resume state - // Save-best: DQN eval is noisy, so the deployable net is only overwritten when the eval mean-score IMPROVES on - // the best seen (seeded from the starting net, so a noisy dip never regresses a good shipped model). - private double _bestScore = double.NegativeInfinity; - private double _lastEvalScore = double.NegativeInfinity; - public string Environment => "fruitcake"; + public override string Environment => "fruitcake"; + // Noisy training is a SEPARATE line with its own resume state: it must never resume a PLAIN state as a plain net + // (which would then train with ε=0 and NO exploration). The deployable NetId is shared and keep-best gated. + protected override string StateId => noisy ? "dqn-noisy-state" : "dqn-state"; + protected override IEnvironment TrainEnv => _env; + protected override string StepNoun => "drops"; + protected override string GateLabel => "mean score"; + protected override string DisplayName => "FruitCake DQN"; + protected override int ObservationSize => FruitCakeEnv.ObservationSize; + protected override IReadOnlyList? InputLabels => FruitCakeEnv.ObservationLabels; + protected override IReadOnlyList? OutputLabels => FruitCakeEnv.ActionLabels; /// Double+Dueling DQN; MaxSteps is managed per chunk by the runner (drops, not physics sub-steps). - private DqnOptions BaseOptions => new() + protected override DqnOptions BaseOptions => new() { Dueling = true, DoubleDqn = true, @@ -62,114 +54,35 @@ internal sealed class FruitCakeDqnCampaign(ulong seed, int chunkSteps, long targ EvalEpisodes = 10, }; - public bool Resume(IModelStore store) + protected override IValueNet AdaptWarmNet(DuelingQNet loaded) { - bool resumed = false; - using (var s = store.TryOpenRead(Environment, StateId)) + // Promote-plain→noisy: the shipped net is plain, but a noisy run needs a noisy net. Copy its trained weights + // into the means + add fresh σ — behaviorally identical with noise off, so the run continues the ~current + // policy and merely adds learnable exploration. + IValueNet warm = noisy && !loaded.Noisy ? loaded.ToNoisy(Seeds.CreateRng(RngStreams.Init)) : loaded; + // If the observation has gained features since this net was trained (e.g. the big-fruit-position inputs), + // grow its input to fit — function-preserving (new inputs zero-init), so the baseline eval and warm-start + // continue the trained net rather than retraining from scratch. + if (warm.InputSize != FruitCakeEnv.ObservationSize) { - if (s is not null) - { - _state = DqnTrainingState.Load(s); - Log($"resumed FruitCake DQN at {_state.StepsCompleted:N0} drops (last eval return {_state.LastEval:F2})"); - resumed = true; - } + Log($"growing the loaded net's input {warm.InputSize} → {FruitCakeEnv.ObservationSize} (observation gained features; weights preserved)"); + warm = warm.GrowInput(FruitCakeEnv.ObservationSize); } - // No full resume state, but the deployable net may exist (e.g. the shipped checkpoint). Warm-start from it: - // a fresh optimizer/replay buffer continues the trained net rather than discarding its weights. - if (!resumed) - { - using var net = store.TryOpenRead(Environment, NetId); - if (net is not null) - { - var loaded = DuelingQNetCheckpoint.Load(net); - // Promote-plain→noisy: the shipped net is plain, but a noisy run needs a noisy net. Copy its - // trained weights into the means + add fresh σ — behaviorally identical with noise off, so the - // run continues the ~current policy and merely adds learnable exploration. - IValueNet warm = noisy && !loaded.Noisy ? loaded.ToNoisy(_seeds.CreateRng(RngStreams.Init)) : loaded; - // If the observation has gained features since this net was trained (e.g. the big-fruit-position - // inputs), grow its input to fit — function-preserving (new inputs zero-init), so the baseline eval - // and warm-start continue the trained net rather than retraining from scratch. - if (warm.InputSize != FruitCakeEnv.ObservationSize) - { - Log($"growing the loaded net's input {warm.InputSize} → {FruitCakeEnv.ObservationSize} (observation gained features; weights preserved)"); - warm = warm.GrowInput(FruitCakeEnv.ObservationSize); - } - _warmNet = warm; - Log($"warm-starting from the deployable FruitCake net '{NetId}'{(noisy ? " (promoted to noisy)" : "")} (fresh optimizer + replay buffer)"); - resumed = true; - } - } - - // Seed save-best from the starting net's score so training only ever ships a net that BEATS it. - var startNet = _state?.Online ?? _warmNet; - if (startNet is not null) - { - _bestScore = EvalNet(startNet).Score; - Log($"baseline mean score: {_bestScore:F2} (will only re-save the deployable net when an eval beats this)"); - } - else - { - Log("starting fresh FruitCake DQN training"); - } - return resumed; + return warm; } - public long TrainChunk() - { - int from = _state?.StepsCompleted ?? 0; - int to = from + chunkSteps; - if (targetSteps > 0) to = (int)Math.Min(to, targetSteps); - // EvalEvery == the chunk size so the trainer's own eval fires at most once per chunk — the campaign's - // authoritative eval is the mean score in Evaluate(). MaxSteps is ABSOLUTE: resuming raises the ceiling. - var options = BaseOptions with { MaxSteps = to, EvalEvery = Math.Max(1, chunkSteps) }; - var result = DqnTrainer.Train(_env, options, _seeds, resume: _state, warmStart: _state is null ? _warmNet : null); - _state = result.State; - _state = DqnGrowth.Maybe(_state, grow, growEvery, learningRate, _growRng, Log); - return _state.StepsCompleted; - } - - /// Score-maximizing: no hard goal. Stops on the runner's time budget, or an optional absolute drop cap. - public bool IsComplete => targetSteps > 0 && (_state?.StepsCompleted ?? 0) >= targetSteps; - - public CampaignEval Evaluate() + protected override (double Gate, IReadOnlyList Metrics, string Summary) EvaluateNet(IValueNet net) { - int steps = _state?.StepsCompleted ?? 0; - var net = _state?.Online ?? _warmNet; - if (net is null) - return new CampaignEval([new("drops", 0, "0")], "no model yet (train first)"); - var (score, maxTier, meanReturn) = EvalNet(net); - _lastEvalScore = score; // Checkpoint() ships the net only when this beats the best seen - - float loss = _state?.LastLoss ?? 0f; - var metrics = new List + var metrics = new CampaignMetric[] { - new("drops", steps, "0"), new("score", score, "F2"), new("maxTier", maxTier, "F2"), new("return", meanReturn, "F2"), - new("loss", loss, "F4"), }; - return new CampaignEval(metrics, - $"drops {steps:N0} | score {score:F2} | maxTier {maxTier:F2} | return {meanReturn:F2} | loss {loss:F4}"); + return (score, metrics, $"score {score:F2} | maxTier {maxTier:F2} | return {meanReturn:F2}"); } - public void Checkpoint(IModelStore store) - { - if (_state is null) return; - // The resume state always tracks the latest net (so a continuation picks up where it left off). - store.Save(Environment, StateId, s => _state.Save(s)); - // The DEPLOYABLE net is save-best: only overwrite it when this eval beat the best seen (DQN eval is noisy). - if (_lastEvalScore > _bestScore) - { - _bestScore = _lastEvalScore; - store.Save(Environment, NetId, s => DuelingQNetCheckpoint.Save((DuelingQNet)_state.Online, s)); - Log($"new best mean score {_bestScore:F2} → saved deployable net '{NetId}'"); - } - } - - public void Dispose() { } - /// Mean episodic score, mean max-tier reached, and mean return over fixed-seed greedy episodes. private (double Score, double MaxTier, double Return) EvalNet(IValueNet net) { @@ -196,41 +109,4 @@ public void Dispose() { } } return (totalScore / evalEpisodes, totalMaxTier / evalEpisodes, totalReturn / evalEpisodes); } - - // --- Live telemetry (INetworkTelemetrySource): read-only; a viewer samples the current net as it trains. --- - string INetworkTelemetrySource.NetKind => "dueling-q"; - IReadOnlyList? INetworkTelemetrySource.SnapshotParameters() - => (_state?.Online ?? _warmNet) is { } net ? [.. net.Parameters()] : null; - NetworkMetrics INetworkTelemetrySource.Sample() - => new(_state?.StepsCompleted ?? 0, targetSteps, _state?.LastLoss ?? double.NaN, _lastEvalScore, double.NaN); - - // Environment-aware neuron labels + live values, so the viewer can say what each input/output MEANS and its - // current value: each input is a named observation feature; each output is the Q-value of dropping in a column. - IReadOnlyList? INetworkTelemetrySource.InputLabels => FruitCakeEnv.ObservationLabels; - IReadOnlyList? INetworkTelemetrySource.OutputLabels => FruitCakeEnv.ActionLabels; - (float[] Input, float[] Output)? INetworkTelemetrySource.SampleIo() - { - var net = _state?.Online ?? _warmNet; - var obs = _state?.CurrentObs; - if (net is null || obs is null || obs.Length != FruitCakeEnv.ObservationSize) return null; - try - { - // Forward the most-recent observation to get the net's current per-column Q-values. Read-only: it never - // writes weights/grads (no Backward), so a concurrent training step is unaffected (torn reads only - // flicker a displayed number). A fresh input array so the viewer never sees it mutated mid-serialize. - var input = (float[])obs.Clone(); - float[] q = net.Forward(new Tensor(input, 1, input.Length)).Data; - return (input, q.AsSpan().ToArray()); - } - catch { return null; } - } - float[][]? INetworkTelemetrySource.SampleActivations() - { - var obs = _state?.CurrentObs; - if ((_state?.Online ?? _warmNet) is not DuelingQNet dqn || obs is null || obs.Length != FruitCakeEnv.ObservationSize) return null; - try { return dqn.LayerActivations(new Tensor((float[])obs.Clone(), 1, obs.Length)); } - catch { return null; } - } - - private static void Log(string message) => Console.WriteLine($"{DateTime.UtcNow:HH:mm:ss} {message}"); } diff --git a/tools/MintPlayer.AI.ReinforcementLearning.Lab/SnakeDqnCampaign.cs b/tools/MintPlayer.AI.ReinforcementLearning.Lab/SnakeDqnCampaign.cs index 9c81395..c60e667 100644 --- a/tools/MintPlayer.AI.ReinforcementLearning.Lab/SnakeDqnCampaign.cs +++ b/tools/MintPlayer.AI.ReinforcementLearning.Lab/SnakeDqnCampaign.cs @@ -1,46 +1,35 @@ -using MintPlayer.AI.ReinforcementLearning.Core.Checkpoints; +using MintPlayer.AI.ReinforcementLearning.Core.Environments; using MintPlayer.AI.ReinforcementLearning.Core.Nn; -using MintPlayer.AI.ReinforcementLearning.Core.Random; -using MintPlayer.AI.ReinforcementLearning.Core.Numerics; using MintPlayer.AI.ReinforcementLearning.Core.Schedules; -using MintPlayer.AI.ReinforcementLearning.Core.Telemetry; using MintPlayer.AI.ReinforcementLearning.Core.Training; using MintPlayer.AI.ReinforcementLearning.Environments.Snake; /// -/// Snake DQN campaign (`--game snake`, PLAN M22) as an on -/// (PLAN M25) — the score-maximizing paradigm: an *infinite-goal* game whose eval is -/// the mean episodic score (food), not a solve rate. Trains the masked Double+Dueling on a -/// small 6×6 grid (fast, dense food; the size-invariant observation transfers to the demo grid) and evaluates mean -/// food on the deployed 12×12 grid. Resumable bitwise-identically via : each chunk -/// raises the absolute and continues from the prior state. Persists the deployable -/// net under `snake`/`dqn` (the id the web's SnakeModelService loads) plus the full resume state under -/// `snake`/`dqn-state`. +/// Snake DQN campaign (`--game snake`, PLAN M22) on the shared spine — the +/// score-maximizing paradigm: an *infinite-goal* game whose eval is the mean episodic score (food), not a solve +/// rate. Trains the masked Double+Dueling on a small 6×6 grid (fast, dense food; the +/// size-invariant observation transfers to the demo grid) and evaluates mean food on the deployed 12×12 grid. All +/// resume/train/keep-best/checkpoint/growth/telemetry plumbing lives in the base; this type supplies only the env, +/// the M22 hyperparameters, and the 12×12 food eval. /// internal sealed class SnakeDqnCampaign(ulong seed, int trainGrid, int evalGrid, int chunkSteps, long targetSteps, int evalEpisodes, float learningRate, float epsilonStart, int[] hidden, double gamma, float stepPenalty, bool safeMask, bool grow = false, int growEvery = 5000) - : ITrainingCampaign, INetworkTelemetrySource + : DqnScoreCampaign(seed, chunkSteps, targetSteps, learningRate, grow, growEvery) { - // Progressive-growth demo (shared with FruitCake via DqnGrowth): grow the net wider+deeper on the growEvery cadence. - private readonly Xoshiro256StarStar _growRng = new(seed ^ 0x9E3779B97F4A7C15UL); - private const string NetId = "dqn"; // deployable DuelingQNet — the id the web loads - private const string StateId = "dqn-state"; // full DqnTrainingState for lossless resume - private readonly SnakeEnv _env = new(trainGrid, stepPenalty, safeMask); private readonly SnakeEnv _evalEnv = new(evalGrid, stepPenalty, safeMask); - private readonly SeedSequence _seeds = new(seed); - - private DqnTrainingState? _state; - private IValueNet? _warmNet; // the deployable net to warm-start from when there's no full resume state - // Save-best: DQN eval is noisy, so the deployable net is only overwritten when the 12×12 eval IMPROVES on the - // best seen (seeded from the starting net, so a noisy dip never regresses a good shipped model). The resume - // state always tracks the latest net. - private double _bestFood = double.NegativeInfinity; - private double _lastEvalFood = double.NegativeInfinity; - public string Environment => "snake"; + public override string Environment => "snake"; + protected override IEnvironment TrainEnv => _env; + protected override string StepNoun => "steps"; + protected override string GateLabel => $"food@{evalGrid}"; + protected override string DisplayName => "Snake DQN"; + protected override string FreshStartDetail => $" (train {trainGrid}×{trainGrid}, eval {evalGrid}×{evalGrid})"; + protected override int ObservationSize => SnakeEnv.ObservationSize; + protected override IReadOnlyList? InputLabels => SnakeEnv.ObservationLabels; + protected override IReadOnlyList? OutputLabels => SnakeEnv.ActionLabels; /// The proven M22 config (masked Double+Dueling DQN); MaxSteps is managed per chunk by the runner. - private DqnOptions BaseOptions => new() + protected override DqnOptions BaseOptions => new() { Dueling = true, DoubleDqn = true, @@ -57,105 +46,17 @@ internal sealed class SnakeDqnCampaign(ulong seed, int trainGrid, int evalGrid, EvalEpisodes = 20, }; - public bool Resume(IModelStore store) - { - bool resumed = false; - using (var s = store.TryOpenRead(Environment, StateId)) - { - if (s is not null) - { - _state = DqnTrainingState.Load(s); - Log($"resumed Snake DQN at {_state.StepsCompleted:N0} steps (last eval return {_state.LastEval:F2})"); - resumed = true; - } - } - // No full resume state, but the deployable net may be present (e.g. the shipped checkpoint, or one produced - // by the old web one-shot trainer). Warm-start from it: a fresh optimizer/replay buffer continues the - // trained net rather than throwing away its weights. Lets us further-train (and eval) the shipped model. - if (!resumed) - { - using var net = store.TryOpenRead(Environment, NetId); - if (net is not null) - { - _warmNet = DuelingQNetCheckpoint.Load(net); - Log($"warm-starting from the deployable Snake net '{NetId}' (fresh optimizer + replay buffer)"); - resumed = true; - } - } - - // Seed save-best from the starting net's score so training only ever ships a net that BEATS it. - var startNet = _state?.Online ?? _warmNet; - if (startNet is not null) - { - _bestFood = EvalNet(startNet).Food; - Log($"baseline food@{evalGrid}: {_bestFood:F2} (will only re-save the deployable net when an eval beats this)"); - } - else - { - Log($"starting fresh Snake DQN training (train {trainGrid}×{trainGrid}, eval {evalGrid}×{evalGrid})"); - } - return resumed; - } - - public long TrainChunk() + protected override (double Gate, IReadOnlyList Metrics, string Summary) EvaluateNet(IValueNet net) { - int from = _state?.StepsCompleted ?? 0; - int to = from + chunkSteps; - if (targetSteps > 0) to = (int)Math.Min(to, targetSteps); - // EvalEvery == the chunk size so the trainer's own (6×6) eval fires at most once per chunk — the campaign's - // authoritative eval is the 12×12 food in Evaluate(). MaxSteps is ABSOLUTE: resuming raises the ceiling. - var options = BaseOptions with { MaxSteps = to, EvalEvery = Math.Max(1, chunkSteps) }; - // First chunk: resume the full state if present, else warm-start from the deployable net (if any). - var result = DqnTrainer.Train(_env, options, _seeds, resume: _state, warmStart: _state is null ? _warmNet : null); - _state = result.State; - _state = DqnGrowth.Maybe(_state, grow, growEvery, learningRate, _growRng, Log); - return _state.StepsCompleted; - } - - /// Score-maximizing: no hard goal. Stops on the runner's time budget, or an optional absolute step cap. - public bool IsComplete => targetSteps > 0 && (_state?.StepsCompleted ?? 0) >= targetSteps; - - public CampaignEval Evaluate() - { - int steps = _state?.StepsCompleted ?? 0; - // Eval the trained net if we've trained this run, else the warm-start net (so eval-only works on the - // shipped checkpoint too); only "no model at all" yields the placeholder. - var net = _state?.Online ?? _warmNet; - if (net is null) - return new CampaignEval([new("steps", 0, "0")], "no model yet (train first)"); - var (food, meanReturn) = EvalNet(net); - _lastEvalFood = food; // Checkpoint() ships the net only when this beats the best seen - - float loss = _state?.LastLoss ?? 0f; - var metrics = new List + var metrics = new CampaignMetric[] { - new("steps", steps, "0"), new($"food{evalGrid}", food, "F2"), new("return", meanReturn, "F2"), - new("loss", loss, "F4"), }; - return new CampaignEval(metrics, - $"steps {steps:N0} | food@{evalGrid} {food:F2} | return {meanReturn:F2} | loss {loss:F4}"); - } - - public void Checkpoint(IModelStore store) - { - if (_state is null) return; - // The resume state always tracks the latest net (so a continuation picks up where it left off). - store.Save(Environment, StateId, s => _state.Save(s)); - // The DEPLOYABLE net is save-best: only overwrite it when this eval beat the best seen — DQN eval is - // noisy and the last net is often not the best (here the 270k net scored 23.3 but 300k fell to 19.5). - if (_lastEvalFood > _bestFood) - { - _bestFood = _lastEvalFood; - store.Save(Environment, NetId, s => DuelingQNetCheckpoint.Save((DuelingQNet)_state.Online, s)); - Log($"new best food@{evalGrid} {_bestFood:F2} → saved deployable net '{NetId}'"); - } + return (food, metrics, $"food@{evalGrid} {food:F2} | return {meanReturn:F2}"); } - public void Dispose() { } - /// Mean food + return over fixed-seed greedy episodes on the DEPLOYED grid (food is the gate metric). private (double Food, double Return) EvalNet(IValueNet net) { @@ -178,36 +79,4 @@ public void Dispose() { } } return (totalFood / evalEpisodes, totalReturn / evalEpisodes); } - - // --- Live telemetry (INetworkTelemetrySource): read-only; a viewer samples the current net as it trains. --- - string INetworkTelemetrySource.NetKind => "dueling-q"; - IReadOnlyList? INetworkTelemetrySource.SnapshotParameters() - => (_state?.Online ?? _warmNet) is { } net ? [.. net.Parameters()] : null; - NetworkMetrics INetworkTelemetrySource.Sample() - => new(_state?.StepsCompleted ?? 0, targetSteps, _state?.LastLoss ?? double.NaN, _lastEvalFood, double.NaN); - - // Environment-aware neuron labels + live values: inputs are named observation features (177-dim egocentric - // vision + scalars), outputs are the 4 move directions with their Q-values, hidden neurons show activations. - IReadOnlyList? INetworkTelemetrySource.InputLabels => SnakeEnv.ObservationLabels; - IReadOnlyList? INetworkTelemetrySource.OutputLabels => SnakeEnv.ActionLabels; - (float[] Input, float[] Output)? INetworkTelemetrySource.SampleIo() - { - var obs = _state?.CurrentObs; - if ((_state?.Online ?? _warmNet) is not { } net || obs is null || obs.Length != SnakeEnv.ObservationSize) return null; - try - { - var input = (float[])obs.Clone(); - return (input, net.Forward(new Tensor(input, 1, input.Length)).Data.AsSpan().ToArray()); - } - catch { return null; } - } - float[][]? INetworkTelemetrySource.SampleActivations() - { - var obs = _state?.CurrentObs; - if ((_state?.Online ?? _warmNet) is not DuelingQNet dqn || obs is null || obs.Length != SnakeEnv.ObservationSize) return null; - try { return dqn.LayerActivations(new Tensor((float[])obs.Clone(), 1, obs.Length)); } - catch { return null; } - } - - private static void Log(string message) => Console.WriteLine($"{DateTime.UtcNow:HH:mm:ss} {message}"); } From 87a9cca6bd9dfcc7a5bc0f82669c5fd8a2c8470b Mon Sep 17 00:00:00 2001 From: PieterjanDeClippel Date: Sun, 12 Jul 2026 20:10:40 +0200 Subject: [PATCH 05/12] =?UTF-8?q?M38=20B3:=20LabHost.Run=20=E2=80=94=20one?= =?UTF-8?q?=20shared=20Lab=20bootstrap=20tail?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every game Lab hand-rolled the same ~15-line tail: build the AIHost (+ AddGpuBackend for the cube labs), resolve store/runner, construct the campaign, attach the --viz viewer, runner.Run(CampaignOptions), then keep the process alive for the viewer. Extracted into LabHost.Run(args, dataDir, hours, evalOnly, useGpu, build, onEval): the caller supplies only the campaign factory (pulling AdaptiveBackend from the provider for GPU labs) and the eval IO hook. WaitForViewer moves into LabHost (off SnakeLab — every other lab was calling it cross-type, a smell). Applied to all six labs (snake, fruitcake, rushhour, cube, cube-policy, cube-davi). Also fixed the two CS9107 warnings B2 introduced by exposing the base's LearningRate as a property instead of re-capturing the ctor param. Behaviour-preserving. Smoke-tested eval-only: snake/fruitcake (CPU DQN), rushhour (CPU imitation) print the expected placeholders/eval; cube-policy boots the GPU path (AddGpuBackend → AdaptiveBackend → real RTX 3060 detected) and evaluates. Build 0 errors / no new warnings; 320 fast tests green. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../CubeDaviLab.cs | 28 ++--------- .../CubeLab.cs | 23 ++------- .../CubePolicyLab.cs | 30 ++---------- .../DqnScoreCampaign.cs | 4 ++ .../FruitCakeDqnCampaign.cs | 2 +- .../FruitCakeLab.cs | 23 ++------- .../LabHost.cs | 49 +++++++++++++++++++ .../RushHourLab.cs | 23 ++------- .../SnakeDqnCampaign.cs | 2 +- .../SnakeLab.cs | 32 ++---------- 10 files changed, 77 insertions(+), 139 deletions(-) create mode 100644 tools/MintPlayer.AI.ReinforcementLearning.Lab/LabHost.cs diff --git a/tools/MintPlayer.AI.ReinforcementLearning.Lab/CubeDaviLab.cs b/tools/MintPlayer.AI.ReinforcementLearning.Lab/CubeDaviLab.cs index 8a194e4..62d07f4 100644 --- a/tools/MintPlayer.AI.ReinforcementLearning.Lab/CubeDaviLab.cs +++ b/tools/MintPlayer.AI.ReinforcementLearning.Lab/CubeDaviLab.cs @@ -1,10 +1,5 @@ using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Hosting; -using MintPlayer.AI.ReinforcementLearning.Core.Checkpoints; -using MintPlayer.AI.ReinforcementLearning.Core.Training; -using MintPlayer.AI.ReinforcementLearning.Hosting; using MintPlayer.AI.ReinforcementLearning.Ilgpu; -using MintPlayer.AI.ReinforcementLearning.Ilgpu.Hosting; /// /// `--game cube-davi` entry point: resolves the long-lived campaign config (code defaults → appsettings.json → @@ -158,23 +153,10 @@ public static void Run(string[] args) EvalEpisodes = evalEpisodes, }; - // DI all the way: the model store, clock, GPU backend and CampaignRunner are resolved from the AIHost - // container. AddGpuBackend() registers the shared AdaptiveBackend (DAVI's wide value net wins on GPU). - var builder = AIHost.CreateBuilder(dataDir); - builder.Services.AddGpuBackend(); - using var host = builder.Build(); - var store = host.Services.GetRequiredService(); - var runner = host.Services.GetRequiredService(); - var backend = host.Services.GetRequiredService(); - - var campaign = new CubeDaviCampaign(backend, settings); - using var viz = VizLauncher.TryStart(args, campaign, host.Services.GetRequiredService()); - runner.Run(campaign, store, new CampaignOptions - { - Duration = TimeSpan.FromHours(hours), - EvalOnly = evalOnly, - OnEval = CampaignCli.Console(), - }); - SnakeLab.WaitForViewer(viz); + // GPU: DAVI's wide value net wins on GPU, so the campaign runs on the AdaptiveBackend (useGpu: true). + // The campaign owns its two depth-column CSVs, so the console-only eval hook is used (no generic metric CSV). + LabHost.Run(args, dataDir, hours, evalOnly, useGpu: true, + sp => new CubeDaviCampaign(sp.GetRequiredService(), settings), + CampaignCli.Console()); } } diff --git a/tools/MintPlayer.AI.ReinforcementLearning.Lab/CubeLab.cs b/tools/MintPlayer.AI.ReinforcementLearning.Lab/CubeLab.cs index d8116e1..15e08c7 100644 --- a/tools/MintPlayer.AI.ReinforcementLearning.Lab/CubeLab.cs +++ b/tools/MintPlayer.AI.ReinforcementLearning.Lab/CubeLab.cs @@ -1,9 +1,4 @@ using System.Globalization; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Hosting; -using MintPlayer.AI.ReinforcementLearning.Core.Checkpoints; -using MintPlayer.AI.ReinforcementLearning.Core.Training; -using MintPlayer.AI.ReinforcementLearning.Hosting; /// /// `--game cube` entry point: parses the campaign flags and runs the @@ -34,21 +29,9 @@ public static void Run(string[] args) else if (args[i] == "--grow-every" && i + 1 < args.Length) growEvery = int.Parse(args[++i]); } - // DI all the way: the model store, clock and CampaignRunner are resolved from the AIHost container. - using var host = AIHost.CreateBuilder(dataDir).Build(); - var store = host.Services.GetRequiredService(); - var runner = host.Services.GetRequiredService(); - string csvPath = Path.Combine(dataDir, "logs", "cube-imitation.csv"); - - var campaign = new CubeImitationCampaign(seed, learningRate, width, grow, growEvery); - using var viz = VizLauncher.TryStart(args, campaign, host.Services.GetRequiredService()); - runner.Run(campaign, store, new CampaignOptions - { - Duration = TimeSpan.FromHours(hours), - EvalOnly = evalOnly, - OnEval = CampaignCli.ConsoleAndCsv(csvPath), - }); - SnakeLab.WaitForViewer(viz); + LabHost.Run(args, dataDir, hours, evalOnly, useGpu: false, + _ => new CubeImitationCampaign(seed, learningRate, width, grow, growEvery), + CampaignCli.ConsoleAndCsv(Path.Combine(dataDir, "logs", "cube-imitation.csv"))); } } diff --git a/tools/MintPlayer.AI.ReinforcementLearning.Lab/CubePolicyLab.cs b/tools/MintPlayer.AI.ReinforcementLearning.Lab/CubePolicyLab.cs index df399dd..b01eb1c 100644 --- a/tools/MintPlayer.AI.ReinforcementLearning.Lab/CubePolicyLab.cs +++ b/tools/MintPlayer.AI.ReinforcementLearning.Lab/CubePolicyLab.cs @@ -1,11 +1,6 @@ using System.Globalization; using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Hosting; -using MintPlayer.AI.ReinforcementLearning.Core.Checkpoints; -using MintPlayer.AI.ReinforcementLearning.Core.Training; -using MintPlayer.AI.ReinforcementLearning.Hosting; using MintPlayer.AI.ReinforcementLearning.Ilgpu; -using MintPlayer.AI.ReinforcementLearning.Ilgpu.Hosting; /// /// `--game cube-policy` entry point: parses the campaign flags and runs the EfficientCube @@ -42,25 +37,10 @@ public static void Run(string[] args) else if (args[i] == "--grow-every" && i + 1 < args.Length) growEvery = int.Parse(args[++i]); } - // DI all the way: the model store, clock, GPU backend and CampaignRunner are resolved from the AIHost - // container. AddGpuBackend() registers the shared AdaptiveBackend (the cube nets are large enough to win on GPU). - var builder = AIHost.CreateBuilder(dataDir); - builder.Services.AddGpuBackend(); - using var host = builder.Build(); - var store = host.Services.GetRequiredService(); - var runner = host.Services.GetRequiredService(); - var backend = host.Services.GetRequiredService(); - string csvPath = Path.Combine(dataDir, "logs", "cube-policy.csv"); - - var campaign = new CubeEfficientCampaign(backend, seed, learningRate, width, maxScramble, beamWidth, evalEpisodes, grow, growEvery); - using var viz = VizLauncher.TryStart(args, campaign, host.Services.GetRequiredService()); - runner.Run(campaign, store, - new CampaignOptions - { - Duration = TimeSpan.FromHours(hours), - EvalOnly = evalOnly, - OnEval = CampaignCli.ConsoleAndCsv(csvPath), - }); - SnakeLab.WaitForViewer(viz); + // GPU: the cube nets are large enough to win on GPU, so the campaign runs on the AdaptiveBackend + // (useGpu: true → LabHost registers it and this build pulls it from the container). + LabHost.Run(args, dataDir, hours, evalOnly, useGpu: true, + sp => new CubeEfficientCampaign(sp.GetRequiredService(), seed, learningRate, width, maxScramble, beamWidth, evalEpisodes, grow, growEvery), + CampaignCli.ConsoleAndCsv(Path.Combine(dataDir, "logs", "cube-policy.csv"))); } } diff --git a/tools/MintPlayer.AI.ReinforcementLearning.Lab/DqnScoreCampaign.cs b/tools/MintPlayer.AI.ReinforcementLearning.Lab/DqnScoreCampaign.cs index ac641c9..f971e5d 100644 --- a/tools/MintPlayer.AI.ReinforcementLearning.Lab/DqnScoreCampaign.cs +++ b/tools/MintPlayer.AI.ReinforcementLearning.Lab/DqnScoreCampaign.cs @@ -65,6 +65,10 @@ internal abstract class DqnScoreCampaign(ulong seed, int chunkSteps, long target /// The net being trained/served: the live online net, else the warm-start net (for eval-only runs). protected IValueNet? CurrentNet => State?.Online ?? _warmNet; + /// The learning rate, exposed so a subclass's reuses the value it already + /// hands the base (which owns it for optimizer rebuilds on growth) rather than capturing the ctor param twice. + protected float LearningRate => learningRate; + public bool Resume(IModelStore store) { bool resumed = false; diff --git a/tools/MintPlayer.AI.ReinforcementLearning.Lab/FruitCakeDqnCampaign.cs b/tools/MintPlayer.AI.ReinforcementLearning.Lab/FruitCakeDqnCampaign.cs index 0ec62fa..720efca 100644 --- a/tools/MintPlayer.AI.ReinforcementLearning.Lab/FruitCakeDqnCampaign.cs +++ b/tools/MintPlayer.AI.ReinforcementLearning.Lab/FruitCakeDqnCampaign.cs @@ -43,7 +43,7 @@ internal sealed class FruitCakeDqnCampaign(ulong seed, int chunkSteps, long targ Hidden = hidden, Gamma = gamma, NStep = nStep, // n-step returns: propagate the sparse high-tier reward backward faster (PRD §4.C C4) - LearningRate = learningRate, + LearningRate = LearningRate, BufferCapacity = 100_000, BatchSize = 128, WarmupSteps = 2_000, diff --git a/tools/MintPlayer.AI.ReinforcementLearning.Lab/FruitCakeLab.cs b/tools/MintPlayer.AI.ReinforcementLearning.Lab/FruitCakeLab.cs index c5417a1..8085b39 100644 --- a/tools/MintPlayer.AI.ReinforcementLearning.Lab/FruitCakeLab.cs +++ b/tools/MintPlayer.AI.ReinforcementLearning.Lab/FruitCakeLab.cs @@ -1,9 +1,4 @@ using System.Globalization; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Hosting; -using MintPlayer.AI.ReinforcementLearning.Core.Checkpoints; -using MintPlayer.AI.ReinforcementLearning.Core.Training; -using MintPlayer.AI.ReinforcementLearning.Hosting; /// /// `--game fruitcake` entry point: parses the campaign flags and runs the score-maximizing @@ -84,20 +79,8 @@ public static void Run(string[] args) return; } - using var host = AIHost.CreateBuilder(dataDir).Build(); - var store = host.Services.GetRequiredService(); - var runner = host.Services.GetRequiredService(); - string csvPath = Path.Combine(dataDir, "logs", "fruitcake-dqn.csv"); - - var campaign = new FruitCakeDqnCampaign(seed, chunkSteps, targetSteps, evalEpisodes, learningRate, explore, hidden, gamma, noisy, nStep, shape, grow, growEvery); - using var viz = VizLauncher.TryStart(args, campaign, host.Services.GetRequiredService()); - runner.Run(campaign, store, - new CampaignOptions - { - Duration = TimeSpan.FromHours(hours), - EvalOnly = evalOnly, - OnEval = CampaignCli.ConsoleAndCsv(csvPath), - }); - SnakeLab.WaitForViewer(viz); + LabHost.Run(args, dataDir, hours, evalOnly, useGpu: false, + _ => new FruitCakeDqnCampaign(seed, chunkSteps, targetSteps, evalEpisodes, learningRate, explore, hidden, gamma, noisy, nStep, shape, grow, growEvery), + CampaignCli.ConsoleAndCsv(Path.Combine(dataDir, "logs", "fruitcake-dqn.csv"))); } } diff --git a/tools/MintPlayer.AI.ReinforcementLearning.Lab/LabHost.cs b/tools/MintPlayer.AI.ReinforcementLearning.Lab/LabHost.cs new file mode 100644 index 0000000..8483d99 --- /dev/null +++ b/tools/MintPlayer.AI.ReinforcementLearning.Lab/LabHost.cs @@ -0,0 +1,49 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using MintPlayer.AI.ReinforcementLearning.Core.Checkpoints; +using MintPlayer.AI.ReinforcementLearning.Core.Training; +using MintPlayer.AI.ReinforcementLearning.Hosting; +using MintPlayer.AI.ReinforcementLearning.Ilgpu.Hosting; + +/// +/// The bootstrap tail every game Lab runs: build the DI container (optionally with the GPU +/// backend), resolve the model store + , let the caller construct its campaign from the +/// resolved services, attach the live --viz viewer, run the campaign under the given options, and keep the +/// process alive afterward so the final net stays inspectable. A Lab is then just "parse flags → build campaign"; +/// the DI wiring, GPU opt-in, and viewer lifetime live here once instead of copy-pasted per game. +/// +internal static class LabHost +{ + /// Constructs the campaign from the host's services — GPU labs pull the + /// AdaptiveBackend from it; CPU labs ignore the argument. + /// The runner's per-eval IO hook (console+CSV for most games, console-only for the ones + /// that own their own CSVs) — see . + public static void Run(string[] args, string dataDir, double hours, bool evalOnly, bool useGpu, + Func build, Action onEval) + { + // DI all the way: the model store, clock, (optional) GPU backend and CampaignRunner come from the container. + var builder = AIHost.CreateBuilder(dataDir); + if (useGpu) builder.Services.AddGpuBackend(); + using var host = builder.Build(); + var store = host.Services.GetRequiredService(); + var runner = host.Services.GetRequiredService(); + + var campaign = build(host.Services); + using var viz = VizLauncher.TryStart(args, campaign, host.Services.GetRequiredService()); + runner.Run(campaign, store, new CampaignOptions + { + Duration = TimeSpan.FromHours(hours), + EvalOnly = evalOnly, + OnEval = onEval, + }); + WaitForViewer(viz); + } + + /// Keep the process (and its live viewer) alive after training so the final net can be inspected. + private static void WaitForViewer(VizServer? viz) + { + if (viz is null || Console.IsInputRedirected) return; + Console.WriteLine($"training finished — viewer still live at {viz.Url}; press Enter to exit."); + Console.ReadLine(); + } +} diff --git a/tools/MintPlayer.AI.ReinforcementLearning.Lab/RushHourLab.cs b/tools/MintPlayer.AI.ReinforcementLearning.Lab/RushHourLab.cs index b2a0163..b30ac41 100644 --- a/tools/MintPlayer.AI.ReinforcementLearning.Lab/RushHourLab.cs +++ b/tools/MintPlayer.AI.ReinforcementLearning.Lab/RushHourLab.cs @@ -1,9 +1,4 @@ using System.Globalization; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Hosting; -using MintPlayer.AI.ReinforcementLearning.Core.Checkpoints; -using MintPlayer.AI.ReinforcementLearning.Core.Training; -using MintPlayer.AI.ReinforcementLearning.Hosting; /// /// The Lab's default `--game rushhour` entry point: parses the campaign flags and runs the @@ -32,20 +27,8 @@ public static void Run(string[] args) else if (args[i] == "--grow-every" && i + 1 < args.Length) growEvery = int.Parse(args[++i]); } - // DI all the way: the model store, clock and CampaignRunner are resolved from the AIHost container. - using var host = AIHost.CreateBuilder(dataDir).Build(); - var store = host.Services.GetRequiredService(); - var runner = host.Services.GetRequiredService(); - string csvPath = Path.Combine(dataDir, "logs", "imitation.csv"); - - var campaign = new RushHourImitationCampaign(seed, learningRate, grow, growEvery); - using var viz = VizLauncher.TryStart(args, campaign, host.Services.GetRequiredService()); - runner.Run(campaign, store, new CampaignOptions - { - Duration = TimeSpan.FromHours(hours), - EvalOnly = evalOnly, - OnEval = CampaignCli.ConsoleAndCsv(csvPath), - }); - SnakeLab.WaitForViewer(viz); + LabHost.Run(args, dataDir, hours, evalOnly, useGpu: false, + _ => new RushHourImitationCampaign(seed, learningRate, grow, growEvery), + CampaignCli.ConsoleAndCsv(Path.Combine(dataDir, "logs", "imitation.csv"))); } } diff --git a/tools/MintPlayer.AI.ReinforcementLearning.Lab/SnakeDqnCampaign.cs b/tools/MintPlayer.AI.ReinforcementLearning.Lab/SnakeDqnCampaign.cs index c60e667..0633b6e 100644 --- a/tools/MintPlayer.AI.ReinforcementLearning.Lab/SnakeDqnCampaign.cs +++ b/tools/MintPlayer.AI.ReinforcementLearning.Lab/SnakeDqnCampaign.cs @@ -35,7 +35,7 @@ internal sealed class SnakeDqnCampaign(ulong seed, int trainGrid, int evalGrid, DoubleDqn = true, Hidden = hidden, Gamma = gamma, - LearningRate = learningRate, + LearningRate = LearningRate, BufferCapacity = 100_000, BatchSize = 128, WarmupSteps = 2_000, diff --git a/tools/MintPlayer.AI.ReinforcementLearning.Lab/SnakeLab.cs b/tools/MintPlayer.AI.ReinforcementLearning.Lab/SnakeLab.cs index 1d9c909..3e6e624 100644 --- a/tools/MintPlayer.AI.ReinforcementLearning.Lab/SnakeLab.cs +++ b/tools/MintPlayer.AI.ReinforcementLearning.Lab/SnakeLab.cs @@ -1,11 +1,6 @@ using System.Diagnostics; using System.Globalization; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Hosting; -using MintPlayer.AI.ReinforcementLearning.Core.Checkpoints; -using MintPlayer.AI.ReinforcementLearning.Core.Training; using MintPlayer.AI.ReinforcementLearning.Environments.Snake; -using MintPlayer.AI.ReinforcementLearning.Hosting; /// /// `--game snake` entry point: parses the campaign flags and runs the score-maximizing @@ -84,30 +79,9 @@ public static void Run(string[] args) return; } - // DI all the way: the model store, clock and CampaignRunner are resolved from the AIHost container. - using var host = AIHost.CreateBuilder(dataDir).Build(); - var store = host.Services.GetRequiredService(); - var runner = host.Services.GetRequiredService(); - string csvPath = Path.Combine(dataDir, "logs", "snake-dqn.csv"); - - var campaign = new SnakeDqnCampaign(seed, trainGrid, evalGrid, chunkSteps, targetSteps, evalEpisodes, learningRate, explore, hidden, gamma, stepPenalty, safeMask, grow, growEvery); - using var viz = VizLauncher.TryStart(args, campaign, host.Services.GetRequiredService()); - runner.Run(campaign, store, - new CampaignOptions - { - Duration = TimeSpan.FromHours(hours), - EvalOnly = evalOnly, - OnEval = CampaignCli.ConsoleAndCsv(csvPath), - }); - WaitForViewer(viz); - } - - /// Keep the process (and its live viewer) alive after training so the final net can be inspected. - internal static void WaitForViewer(VizServer? viz) - { - if (viz is null || Console.IsInputRedirected) return; - Console.WriteLine($"training finished — viewer still live at {viz.Url}; press Enter to exit."); - Console.ReadLine(); + LabHost.Run(args, dataDir, hours, evalOnly, useGpu: false, + _ => new SnakeDqnCampaign(seed, trainGrid, evalGrid, chunkSteps, targetSteps, evalEpisodes, learningRate, explore, hidden, gamma, stepPenalty, safeMask, grow, growEvery), + CampaignCli.ConsoleAndCsv(Path.Combine(dataDir, "logs", "snake-dqn.csv"))); } /// From 0e51b7c4897632418e3adf679c0d6387e77504d5 Mon Sep 17 00:00:00 2001 From: PieterjanDeClippel Date: Sun, 12 Jul 2026 20:17:45 +0200 Subject: [PATCH 06/12] M38 B4: extract SupervisedTraining plumbing (AdamState + TrainWindow) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The three imitation/policy campaigns (RushHour, Cube, EfficientCube) each copy-pasted two self-contained bits: the Adam moment load-or-init/save dance (BinaryReader/Writer + UTF-8 + leaveOpen + AdamCheckpoint.Read/Write; the "no stored optimizer" branch) and the rolling window-mean of the (CE, Huber, acc) batch losses (four fields + guarded divide + reset). Extracted into two deep helpers: - AdamState.LoadOrInit / .Save — hides the stream dance; defines the no-optimizer case out of existence (returns a fresh Adam). - TrainWindow struct — Add(ce, huber, acc) + MeanAndReset(). The campaigns stay SEPARATE (their Evaluate and data-generation are different algorithms — merging them would be a shallow/leaky base); only the shared plumbing moved. Net-load + PolicyGrowth wiring stay per-campaign (net types differ). CubeEfficient's own progress blob (samples+round) is untouched. Behaviour-preserving (pure code movement — identical arithmetic + stream I/O, no RNG/logic change). 320 fast tests green — incl. RushHourCampaign_Checkpoints_AndResumes, which saves policy-adam via the new helper and resumes it. RushHour eval-only smoke is byte-identical to pre-B4. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../CubeEfficientCampaign.cs | 32 ++-------- .../CubeImitationCampaign.cs | 32 ++-------- .../RushHourImitationCampaign.cs | 32 ++-------- .../SupervisedTraining.cs | 64 +++++++++++++++++++ 4 files changed, 79 insertions(+), 81 deletions(-) create mode 100644 tools/MintPlayer.AI.ReinforcementLearning.Lab/SupervisedTraining.cs diff --git a/tools/MintPlayer.AI.ReinforcementLearning.Lab/CubeEfficientCampaign.cs b/tools/MintPlayer.AI.ReinforcementLearning.Lab/CubeEfficientCampaign.cs index f7fe8d5..49a34c4 100644 --- a/tools/MintPlayer.AI.ReinforcementLearning.Lab/CubeEfficientCampaign.cs +++ b/tools/MintPlayer.AI.ReinforcementLearning.Lab/CubeEfficientCampaign.cs @@ -36,8 +36,7 @@ internal sealed class CubeEfficientCampaign(AdaptiveBackend adaptive, ulong seed private CubePolicyNet _net = null!; private Adam _adam = null!; private long _round, _totalSamples; - private double _windowCe, _windowHuber, _windowAcc; - private long _windowCount; + private TrainWindow _window; private double _liveLoss = double.NaN, _liveAcc = double.NaN; // most-recent batch, for the live viewer public string Environment => CubeIds.Environment; @@ -67,17 +66,7 @@ public bool Resume(IModelStore store) resumed = false; } } - using (var adamState = store.TryOpenRead(CubeIds.Environment, PolicyAdamId)) - { - if (adamState is not null) - { - using var reader = new BinaryReader(adamState, Encoding.UTF8, leaveOpen: true); - _adam = AdamCheckpoint.Read(_net.Parameters(), reader); - _adam.LearningRate = learningRate; - Log($"resumed Adam state (lr set to {learningRate:E1})"); - } - else _adam = new Adam(_net.Parameters(), learningRate); - } + _adam = AdamState.LoadOrInit(store, CubeIds.Environment, PolicyAdamId, _net.Parameters(), learningRate, Log); // Persisted progress: cumulative samples (displayed count) + the round counter (so the scramble RNG // continues its stream on resume rather than regenerating the same scrambles). using (var progress = store.TryOpenRead(CubeIds.Environment, PolicyProgressId)) @@ -115,10 +104,7 @@ public long TrainChunk() for (int offset = 0; offset + BatchSize <= samples.Count; offset += BatchSize) { var (ce, huber, acc) = CubePolicyTraining.TrainStep(_net, _adam, samples, offset, BatchSize); - _windowCe += ce; - _windowHuber += huber; - _windowAcc += acc; - _windowCount++; + _window.Add(ce, huber, acc); _totalSamples += BatchSize; _liveLoss = ce + huber; _liveAcc = acc; @@ -130,11 +116,7 @@ public long TrainChunk() public CampaignEval Evaluate() { - double ce = _windowCount > 0 ? _windowCe / _windowCount : 0; - double acc = _windowCount > 0 ? _windowAcc / _windowCount : 0; - double huber = _windowCount > 0 ? _windowHuber / _windowCount : 0; - _windowCe = _windowHuber = _windowAcc = 0; - _windowCount = 0; + var (ce, huber, acc) = _window.MeanAndReset(); var metrics = new List { @@ -188,11 +170,7 @@ public CampaignEval Evaluate() public void Checkpoint(IModelStore store) { store.Save(CubeIds.Environment, PolicyId, s => _net.Save(s)); - store.Save(CubeIds.Environment, PolicyAdamId, s => - { - using var writer = new BinaryWriter(s, Encoding.UTF8, leaveOpen: true); - AdamCheckpoint.Write(_adam, writer); - }); + AdamState.Save(store, CubeIds.Environment, PolicyAdamId, _adam); store.Save(CubeIds.Environment, PolicyProgressId, s => { using var writer = new BinaryWriter(s, Encoding.UTF8, leaveOpen: true); diff --git a/tools/MintPlayer.AI.ReinforcementLearning.Lab/CubeImitationCampaign.cs b/tools/MintPlayer.AI.ReinforcementLearning.Lab/CubeImitationCampaign.cs index 8527dfd..8678b9c 100644 --- a/tools/MintPlayer.AI.ReinforcementLearning.Lab/CubeImitationCampaign.cs +++ b/tools/MintPlayer.AI.ReinforcementLearning.Lab/CubeImitationCampaign.cs @@ -28,8 +28,7 @@ internal sealed class CubeImitationCampaign(ulong seed, float learningRate, int private CubePolicyNet _net = null!; private Adam _adam = null!; private long _round, _totalSamples, _totalSolves; - private double _windowCe, _windowHuber, _windowAcc; - private long _windowCount; + private TrainWindow _window; private double _liveLoss = double.NaN, _liveAcc = double.NaN; // most-recent batch, for the live viewer public string Environment => CubeIds.Environment; @@ -55,17 +54,7 @@ public bool Resume(IModelStore store) resumed = false; } } - using (var adamState = store.TryOpenRead(CubeIds.Environment, _ids.PolicyAdam)) - { - if (adamState is not null) - { - using var reader = new BinaryReader(adamState, Encoding.UTF8, leaveOpen: true); - _adam = AdamCheckpoint.Read(_net.Parameters(), reader); - _adam.LearningRate = learningRate; - Log($"resumed Adam state (lr set to {learningRate:E1})"); - } - else _adam = new Adam(_net.Parameters(), learningRate); - } + _adam = AdamState.LoadOrInit(store, CubeIds.Environment, _ids.PolicyAdam, _net.Parameters(), learningRate, Log); Log("warming the Kociemba tables…"); CubeSolver.WarmUp(); return resumed; @@ -99,10 +88,7 @@ public long TrainChunk() for (int offset = 0; offset + BatchSize <= samples.Count; offset += BatchSize) { var (ce, huber, acc) = CubePolicyTraining.TrainStep(_net, _adam, samples, offset, BatchSize); - _windowCe += ce; - _windowHuber += huber; - _windowAcc += acc; - _windowCount++; + _window.Add(ce, huber, acc); _totalSamples += BatchSize; _liveLoss = ce + huber; _liveAcc = acc; @@ -114,11 +100,7 @@ public long TrainChunk() public CampaignEval Evaluate() { - double ce = _windowCount > 0 ? _windowCe / _windowCount : 0; - double acc = _windowCount > 0 ? _windowAcc / _windowCount : 0; - double huber = _windowCount > 0 ? _windowHuber / _windowCount : 0; - _windowCe = _windowHuber = _windowAcc = 0; - _windowCount = 0; + var (ce, huber, acc) = _window.MeanAndReset(); var metrics = new List { @@ -145,11 +127,7 @@ public CampaignEval Evaluate() public void Checkpoint(IModelStore store) { store.Save(CubeIds.Environment, _ids.Policy, s => _net.Save(s)); - store.Save(CubeIds.Environment, _ids.PolicyAdam, s => - { - using var writer = new BinaryWriter(s, Encoding.UTF8, leaveOpen: true); - AdamCheckpoint.Write(_adam, writer); - }); + AdamState.Save(store, CubeIds.Environment, _ids.PolicyAdam, _adam); } /// `--eval-only`: per-depth eval report + the pre-registered M16 gate. No training, no checkpoint. diff --git a/tools/MintPlayer.AI.ReinforcementLearning.Lab/RushHourImitationCampaign.cs b/tools/MintPlayer.AI.ReinforcementLearning.Lab/RushHourImitationCampaign.cs index 25c3d22..8ac1f6a 100644 --- a/tools/MintPlayer.AI.ReinforcementLearning.Lab/RushHourImitationCampaign.cs +++ b/tools/MintPlayer.AI.ReinforcementLearning.Lab/RushHourImitationCampaign.cs @@ -28,8 +28,7 @@ internal sealed class RushHourImitationCampaign(ulong seed, float learningRate, private Adam _adam = null!; private long _totalSamples; private int _totalConfigs; - private double _windowCe, _windowHuber, _windowAcc; - private long _windowCount; + private TrainWindow _window; private long _windowOnPolicy, _windowDrawn; private double _liveLoss = double.NaN, _liveAcc = double.NaN; // most-recent batch, for the live viewer @@ -84,17 +83,7 @@ public bool Resume(IModelStore store) } // Restore Adam's moment estimates when continuing a campaign — without them, resumed // training spends its first minutes re-estimating gradient statistics from zero. - using (var adamState = store.TryOpenRead("rushhour", "policy-adam")) - { - if (adamState is not null) - { - using var reader = new BinaryReader(adamState, Encoding.UTF8, leaveOpen: true); - _adam = AdamCheckpoint.Read(_net.Parameters(), reader); - _adam.LearningRate = learningRate; // CLI overrides the stored schedule position - Log($"resumed Adam state (lr set to {learningRate:E1})"); - } - else _adam = new Adam(_net.Parameters(), learningRate); - } + _adam = AdamState.LoadOrInit(store, "rushhour", "policy-adam", _net.Parameters(), learningRate, Log); return resumed; } @@ -118,10 +107,7 @@ public long TrainChunk() for (int offset = 0; offset + BatchSize <= samples.Count; offset += BatchSize) { var (ce, huber, acc) = TrainStep(samples, offset, BatchSize); - _windowCe += ce; - _windowHuber += huber; - _windowAcc += acc; - _windowCount++; + _window.Add(ce, huber, acc); _totalSamples += BatchSize; _liveLoss = ce + huber; _liveAcc = acc; @@ -133,11 +119,7 @@ public long TrainChunk() public CampaignEval Evaluate() { - double ce = _windowCount > 0 ? _windowCe / _windowCount : 0; - double huber = _windowCount > 0 ? _windowHuber / _windowCount : 0; - double acc = _windowCount > 0 ? _windowAcc / _windowCount : 0; - _windowCe = _windowHuber = _windowAcc = 0; - _windowCount = 0; + var (ce, huber, acc) = _window.MeanAndReset(); if (_windowDrawn > 0) Log($"[mix] on-policy share this window: {_windowOnPolicy / (double)_windowDrawn:P1}"); _windowOnPolicy = _windowDrawn = 0; @@ -183,11 +165,7 @@ public CampaignEval Evaluate() public void Checkpoint(IModelStore store) { store.Save("rushhour", "policy", s => _net.Save(s)); - store.Save("rushhour", "policy-adam", s => - { - using var writer = new BinaryWriter(s, Encoding.UTF8, leaveOpen: true); - AdamCheckpoint.Write(_adam, writer); - }); + AdamState.Save(store, "rushhour", "policy-adam", _adam); } public void Dispose() { } diff --git a/tools/MintPlayer.AI.ReinforcementLearning.Lab/SupervisedTraining.cs b/tools/MintPlayer.AI.ReinforcementLearning.Lab/SupervisedTraining.cs new file mode 100644 index 0000000..724dbf5 --- /dev/null +++ b/tools/MintPlayer.AI.ReinforcementLearning.Lab/SupervisedTraining.cs @@ -0,0 +1,64 @@ +using System.Text; +using MintPlayer.AI.ReinforcementLearning.Core.Checkpoints; +using MintPlayer.AI.ReinforcementLearning.Core.Nn; +using MintPlayer.AI.ReinforcementLearning.Core.Numerics; + +/// +/// A rolling mean of the three per-batch supervised-training losses (cross-entropy, Huber, accuracy) accumulated +/// across the rounds between two evaluations. Every imitation/policy campaign kept the same four fields and the same +/// guarded divide-and-reset; this hides that bookkeeping behind and . +/// +internal struct TrainWindow +{ + private double _ce, _huber, _acc; + private long _count; + + public void Add(double ce, double huber, double acc) + { + _ce += ce; + _huber += huber; + _acc += acc; + _count++; + } + + /// The mean of each metric since the last reset (0 when no batch ran), then clears the window. + public (double Ce, double Huber, double Acc) MeanAndReset() + { + var mean = _count > 0 ? (_ce / _count, _huber / _count, _acc / _count) : (0d, 0d, 0d); + _ce = _huber = _acc = 0; + _count = 0; + return mean; + } +} + +/// +/// Load/save an optimizer's moment estimates alongside a supervised net in the model store. The +/// moments are keyed to the net's parameter tensors, so both sides pass net.Parameters(). Hides the +/// / + UTF-8 + leaveOpen dance the imitation/policy +/// campaigns each copied, and defines the "no stored optimizer yet" case out of existence (a fresh Adam is +/// returned) so the caller never branches on it. +/// +internal static class AdamState +{ + /// Restore Adam's moments from if present (re-pinning the CLI learning rate over + /// the stored schedule position, and logging it), else return a fresh optimizer over . + public static Adam LoadOrInit(IModelStore store, string environmentId, string id, + IEnumerable parameters, float learningRate, Action log) + { + using var stream = store.TryOpenRead(environmentId, id); + if (stream is null) return new Adam(parameters, learningRate); + using var reader = new BinaryReader(stream, Encoding.UTF8, leaveOpen: true); + var adam = AdamCheckpoint.Read(parameters, reader); + adam.LearningRate = learningRate; // CLI overrides the stored schedule position + log($"resumed Adam state (lr set to {learningRate:E1})"); + return adam; + } + + /// Persist Adam's moments under . + public static void Save(IModelStore store, string environmentId, string id, Adam adam) + => store.Save(environmentId, id, s => + { + using var writer = new BinaryWriter(s, Encoding.UTF8, leaveOpen: true); + AdamCheckpoint.Write(adam, writer); + }); +} From 4a0e23766fc4257ca4b57342fb7d90774409471e Mon Sep 17 00:00:00 2001 From: PieterjanDeClippel Date: Sun, 12 Jul 2026 20:20:43 +0200 Subject: [PATCH 07/12] =?UTF-8?q?M38=20B5:=20AtomicFile.Write=20=E2=80=94?= =?UTF-8?q?=20one=20temp-then-rename=20helper?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit GalleryStore.Add and RushHourDeckStore.Write both hand-rolled the same "write to .tmp, then File.Move overwrite" so a concurrent reader never sees a half-written JSON file. Extracted into AtomicFile.Write(path, contents) (also creates the directory). Both call sites collapse to one line. Behaviour-preserving. Build 0 errors; 320 fast tests green (gallery + deck API/store tests exercise the write path). Note: the frontend watch-scaffolding dedup (wake-lock re-acquire / director-loop, PRD B5 P7/P8) is deliberately left for now — the wake lock is already a deep shared service (ScreenWakeLock owns its own visibility re-acquire), the residual per-component wiring is ~3 lines, and this repo's rules forbid running the Angular build/test locally, so those changes can't be verified here. Not worth a blind, behaviour-touching edit for a cosmetic win. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/RLDemo.Web/Services/AtomicFile.cs | 17 +++++++++++++++++ src/RLDemo.Web/Services/GalleryStore.cs | 6 +----- src/RLDemo.Web/Services/RushHourDeckStore.cs | 7 +------ 3 files changed, 19 insertions(+), 11 deletions(-) create mode 100644 src/RLDemo.Web/Services/AtomicFile.cs diff --git a/src/RLDemo.Web/Services/AtomicFile.cs b/src/RLDemo.Web/Services/AtomicFile.cs new file mode 100644 index 0000000..b588219 --- /dev/null +++ b/src/RLDemo.Web/Services/AtomicFile.cs @@ -0,0 +1,17 @@ +namespace RLDemo.Web.Services; + +/// +/// Write a file so a concurrent reader never observes a half-written one: write to a sibling .tmp then +/// atomically rename it over the target (creating the directory if needed). The JSON stores (gallery, deck) use it +/// because a request may read the file while another request rewrites it. +/// +internal static class AtomicFile +{ + public static void Write(string path, string contents) + { + Directory.CreateDirectory(Path.GetDirectoryName(path)!); + string temp = path + ".tmp"; + File.WriteAllText(temp, contents); + File.Move(temp, path, overwrite: true); + } +} diff --git a/src/RLDemo.Web/Services/GalleryStore.cs b/src/RLDemo.Web/Services/GalleryStore.cs index 33a5e37..4886e8a 100644 --- a/src/RLDemo.Web/Services/GalleryStore.cs +++ b/src/RLDemo.Web/Services/GalleryStore.cs @@ -33,11 +33,7 @@ public GalleryEntry Add(string game, string summary, object request, object resp Request: JsonSerializer.SerializeToElement(request, Options), Response: JsonSerializer.SerializeToElement(response, Options)); - Directory.CreateDirectory(RootDirectory); - string path = PathOf(entry.Id); - string temp = path + ".tmp"; - File.WriteAllText(temp, JsonSerializer.Serialize(entry, Options)); - File.Move(temp, path, overwrite: true); + AtomicFile.Write(PathOf(entry.Id), JsonSerializer.Serialize(entry, Options)); return entry; } diff --git a/src/RLDemo.Web/Services/RushHourDeckStore.cs b/src/RLDemo.Web/Services/RushHourDeckStore.cs index 654c5e0..99161cb 100644 --- a/src/RLDemo.Web/Services/RushHourDeckStore.cs +++ b/src/RLDemo.Web/Services/RushHourDeckStore.cs @@ -76,10 +76,5 @@ public bool Delete(string id) } private void Write(List levels) - { - Directory.CreateDirectory(Path.GetDirectoryName(FilePath)!); - string temp = FilePath + ".tmp"; - File.WriteAllText(temp, JsonSerializer.Serialize(new RushHourDeck(1, levels), Json)); - File.Move(temp, FilePath, overwrite: true); - } + => AtomicFile.Write(FilePath, JsonSerializer.Serialize(new RushHourDeck(1, levels), Json)); } From 2d6d4fce1220422238e045f649325663b45a1813 Mon Sep 17 00:00:00 2001 From: PieterjanDeClippel Date: Sun, 12 Jul 2026 20:21:21 +0200 Subject: [PATCH 08/12] =?UTF-8?q?docs(M38):=20mark=20B0=E2=80=93B5=20lande?= =?UTF-8?q?d;=20record=20deferred=20B3=20CliArgs=20+=20B5=20frontend?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/prd/BOILERPLATE_REDUCTION_PRD.md | 2 +- docs/prd/PLAN.md | 13 ++++++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/docs/prd/BOILERPLATE_REDUCTION_PRD.md b/docs/prd/BOILERPLATE_REDUCTION_PRD.md index e29f118..91a6359 100644 --- a/docs/prd/BOILERPLATE_REDUCTION_PRD.md +++ b/docs/prd/BOILERPLATE_REDUCTION_PRD.md @@ -1,6 +1,6 @@ # Reduce per-game boilerplate — campaigns, web services, frontend — PRD -**Status:** Planned · 2026-07-12 · branch TBD (off `master`) · **supersedes the stale [PR #27](https://github.com/MintPlayer/MintPlayer.AI/pull/27)** +**Status:** B0–B5 landed · 2026-07-12 · branch `m38-reduce-boilerplate-plan`, [PR #31](https://github.com/MintPlayer/MintPlayer.AI/pull/31) · **supersedes the stale [PR #27](https://github.com/MintPlayer/MintPlayer.AI/pull/27)** · (deferred: B3 `CliArgs`, B5 frontend P7/P8 — see PLAN M38) **Owner:** Pieterjan **Milestone:** [PLAN.md](PLAN.md) M38 · **Depends on:** nothing new — a behaviour-preserving refactor across the Lab training harness (§8 of [../ARCHITECTURE.md](../ARCHITECTURE.md)), the web model-service layer (§7), and the Angular playground (§10). diff --git a/docs/prd/PLAN.md b/docs/prd/PLAN.md index cd491be..c87469d 100644 --- a/docs/prd/PLAN.md +++ b/docs/prd/PLAN.md @@ -1286,7 +1286,18 @@ note the identity diagonals in the just-deepened layers' heatmaps), both with ** 320 fast tests green (incl. Net2Net forward-equality for DuelingQNet + PolicyValueNet, and a v1-checkpoint-load test). Screenshots: `docs/screenshots/m36-network-grows.png` (DQN), `m37-policy-net-grows.png` (policy). -## M38 — Reduce per-game boilerplate (campaigns · web services · frontend) *(planned 2026-07-12; supersedes stale PR #27; see `BOILERPLATE_REDUCTION_PRD.md`)* 🔜 +## M38 — Reduce per-game boilerplate (campaigns · web services · frontend) *(2026-07-12; branch `m38-reduce-boilerplate-plan`, PR #31; supersedes stale PR #27; see `BOILERPLATE_REDUCTION_PRD.md`)* ✅ + +**Status (2026-07-12):** B0–B5 landed on `m38-reduce-boilerplate-plan` (PR #31), each behaviour-preserving with its +own commit; solution builds 0/0 and 320 fast tests stay green after every step. **B2 is SHA256-bitwise-verified** +(fresh seed-1 `snake`/`fruitcake` produce byte-identical deployable + resume checkpoints vs the pre-refactor build). +Two sub-items are **deliberately deferred**: the B3 `CliArgs`/`CommonLabArgs` parse-loop rewrite (the culture +inconsistency it targets is theoretical for integer CLI args, and rewriting all six parse loops has real regression +risk with no automated gate — low reward, higher risk), and the B5 frontend watch-scaffolding dedup (P7/P8 — the +wake lock is already a deep shared service, the residual per-component wiring is ~3 lines, and this repo forbids +running the Angular build/test locally so the change can't be verified here). `LabHost.Run` (B3) still removed the +bulk of the Lab boilerplate (the ~15-line bootstrap tail × 6). + **Problem.** Adding each game left near-identical copy-paste in three layers, and several copies have already **drifted** (one has a bug its siblings don't). PR #27 took a first cut on 2026-07-10 but is now stale/conflicting: From e42aef74fecbe1932c77584bd34ac46caa838bf2 Mon Sep 17 00:00:00 2001 From: PieterjanDeClippel Date: Sun, 12 Jul 2026 20:46:01 +0200 Subject: [PATCH 09/12] =?UTF-8?q?M38=20B3=20(cont.):=20CliArgs=20=E2=80=94?= =?UTF-8?q?=20one=20flag=20reader;=20fix=20locale-dependent=20parsing?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The six game Labs each hand-rolled a forward `for` loop that repeated, per flag, the `i+1 --- .../CliArgsTests.cs | 82 +++++++++++++++++ .../CliArgs.cs | 33 +++++++ .../CubeLab.cs | 30 ++----- .../CubePolicyLab.cs | 38 +++----- .../FruitCakeLab.cs | 78 ++++++----------- .../RushHourLab.cs | 27 ++---- .../SnakeLab.cs | 87 +++++++------------ 7 files changed, 201 insertions(+), 174 deletions(-) create mode 100644 tests/MintPlayer.AI.ReinforcementLearning.Tests/CliArgsTests.cs create mode 100644 tools/MintPlayer.AI.ReinforcementLearning.Lab/CliArgs.cs diff --git a/tests/MintPlayer.AI.ReinforcementLearning.Tests/CliArgsTests.cs b/tests/MintPlayer.AI.ReinforcementLearning.Tests/CliArgsTests.cs new file mode 100644 index 0000000..d82f36c --- /dev/null +++ b/tests/MintPlayer.AI.ReinforcementLearning.Tests/CliArgsTests.cs @@ -0,0 +1,82 @@ +extern alias Lab; // the Lab exe's internal CLI helper (global namespace, aliased like the campaigns) + +using System.Globalization; +using CliArgs = Lab::CliArgs; + +namespace MintPlayer.AI.ReinforcementLearning.Tests; + +/// +/// Unit tests for the Lab's CliArgs flag reader (M38 B3): the parsing semantics every game Lab now shares — +/// defaults, typed parsing, last-occurrence-wins, missing-value tolerance, and (the bug B3 fixes) culture-invariant +/// numeric parsing. +/// +public class CliArgsTests +{ + [Fact] + public void Missing_flags_return_their_defaults() + { + var a = new CliArgs([]); + Assert.Equal(9.0, a.Dbl("--hours", 9)); + Assert.Equal("data", a.Str("--data", "data")); + Assert.Equal(1UL, a.ULong("--seed", 1)); + Assert.Equal(2048, a.Int("--grow-every", 2048)); + Assert.False(a.Has("--grow")); + Assert.Equal([128, 128], a.Ints("--hidden", [128, 128])); + } + + [Fact] + public void Parses_the_value_after_each_flag() + { + var a = new CliArgs(["--hours", "2.5", "--seed", "42", "--steps", "100000", "--lr", "5e-4", + "--data", "./d", "--grow", "--hidden", "64,64,64"]); + Assert.Equal(2.5, a.Dbl("--hours", 1)); + Assert.Equal(42UL, a.ULong("--seed", 1)); + Assert.Equal(100000L, a.Long("--steps", 0)); + Assert.Equal(5e-4f, a.Flt("--lr", 1e-3f)); + Assert.Equal("./d", a.Str("--data", "data")); + Assert.True(a.Has("--grow")); + Assert.Equal([64, 64, 64], a.Ints("--hidden", [1])); + } + + [Fact] + public void Last_occurrence_of_a_flag_wins() + { + // Matches the old forward-loop behaviour (each match reassigned the variable). + var a = new CliArgs(["--seed", "1", "--seed", "7"]); + Assert.Equal(7UL, a.ULong("--seed", 0)); + } + + [Fact] + public void A_flag_with_no_following_value_falls_back_to_the_default() + { + var a = new CliArgs(["--lr"]); // trailing flag, no value token + Assert.Equal(3e-4f, a.Flt("--lr", 3e-4f)); + } + + [Fact] + public void An_exact_flag_name_is_not_confused_with_a_longer_one() + { + var a = new CliArgs(["--grow-every", "5000"]); // must NOT register "--grow" + Assert.False(a.Has("--grow")); + Assert.Equal(5000, a.Int("--grow-every", 0)); + } + + [Fact] + public void Numeric_parsing_is_culture_invariant() + { + // The bug B3 fixes: on a comma-decimal machine locale, a dot-decimal CLI value like "0.997" must still + // parse to 0.997 — not 997 (dot read as a thousands group) — because CliArgs pins InvariantCulture. + var original = CultureInfo.CurrentCulture; + try + { + CultureInfo.CurrentCulture = new CultureInfo("nl-BE"); // ',' decimal, '.' grouping + var a = new CliArgs(["--lr", "0.0005", "--gamma", "0.997"]); + Assert.Equal(0.0005f, a.Flt("--lr", 1f)); + Assert.Equal(0.997, a.Dbl("--gamma", 0)); + } + finally + { + CultureInfo.CurrentCulture = original; + } + } +} diff --git a/tools/MintPlayer.AI.ReinforcementLearning.Lab/CliArgs.cs b/tools/MintPlayer.AI.ReinforcementLearning.Lab/CliArgs.cs new file mode 100644 index 0000000..0c3270f --- /dev/null +++ b/tools/MintPlayer.AI.ReinforcementLearning.Lab/CliArgs.cs @@ -0,0 +1,33 @@ +using System.Globalization; + +/// +/// A tiny reader over a Lab's string[] args: each accessor finds a --flag and parses the token after +/// it, hiding the three things every hand-rolled flag loop repeated — the i + 1 < args.Length bounds +/// guard, stepping past the consumed value, and (the part that was applied inconsistently) parsing with +/// so a comma-decimal machine locale can't reinterpret 5e-4 or a +/// thousands-grouped integer. A flag's LAST occurrence wins, matching the old forward-loop-overwrites behaviour; a +/// missing flag — or one with no following value — yields the supplied default. Boolean switches use . +/// +internal readonly struct CliArgs(string[] args) +{ + /// True when the switch is present anywhere (e.g. --grow, --eval-only). + public bool Has(string flag) => Array.IndexOf(args, flag) >= 0; + + /// The token following the flag's LAST occurrence, or null if the flag is absent / has no value. + private string? Value(string flag) + { + int i = Array.LastIndexOf(args, flag); + return i >= 0 && i + 1 < args.Length ? args[i + 1] : null; + } + + public string Str(string flag, string @default) => Value(flag) ?? @default; + public double Dbl(string flag, double @default) => Value(flag) is { } v ? double.Parse(v, CultureInfo.InvariantCulture) : @default; + public float Flt(string flag, float @default) => Value(flag) is { } v ? float.Parse(v, CultureInfo.InvariantCulture) : @default; + public int Int(string flag, int @default) => Value(flag) is { } v ? int.Parse(v, CultureInfo.InvariantCulture) : @default; + public long Long(string flag, long @default) => Value(flag) is { } v ? long.Parse(v, CultureInfo.InvariantCulture) : @default; + public ulong ULong(string flag, ulong @default) => Value(flag) is { } v ? ulong.Parse(v, CultureInfo.InvariantCulture) : @default; + + /// A comma-separated int list (e.g. --hidden 256,256), or the default when the flag is absent. + public int[] Ints(string flag, int[] @default) + => Value(flag) is { } v ? [.. v.Split(',').Select(s => int.Parse(s, CultureInfo.InvariantCulture))] : @default; +} diff --git a/tools/MintPlayer.AI.ReinforcementLearning.Lab/CubeLab.cs b/tools/MintPlayer.AI.ReinforcementLearning.Lab/CubeLab.cs index 15e08c7..34be9cf 100644 --- a/tools/MintPlayer.AI.ReinforcementLearning.Lab/CubeLab.cs +++ b/tools/MintPlayer.AI.ReinforcementLearning.Lab/CubeLab.cs @@ -1,5 +1,3 @@ -using System.Globalization; - /// /// `--game cube` entry point: parses the campaign flags and runs the /// (PLAN M16) on the shared (PLAN M25). All console + CSV IO lives in @@ -9,25 +7,15 @@ internal static class CubeLab { public static void Run(string[] args) { - double hours = 9; - string dataDir = "data"; - ulong seed = 1; - float learningRate = 3e-4f; - bool evalOnly = false; - int width = 512; - bool grow = false; // --grow : progressively grow the net wider+deeper mid-training (Net2Net) - int growEvery = 4096; // --grow-every : samples between growth steps (with --grow) - for (int i = 0; i < args.Length; i++) - { - if (args[i] == "--hours" && i + 1 < args.Length) hours = double.Parse(args[++i], CultureInfo.InvariantCulture); - else if (args[i] == "--data" && i + 1 < args.Length) dataDir = args[++i]; - else if (args[i] == "--seed" && i + 1 < args.Length) seed = ulong.Parse(args[++i]); - else if (args[i] == "--lr" && i + 1 < args.Length) learningRate = float.Parse(args[++i], CultureInfo.InvariantCulture); - else if (args[i] == "--width" && i + 1 < args.Length) width = int.Parse(args[++i]); - else if (args[i] == "--eval-only") evalOnly = true; - else if (args[i] == "--grow") grow = true; - else if (args[i] == "--grow-every" && i + 1 < args.Length) growEvery = int.Parse(args[++i]); - } + var a = new CliArgs(args); + double hours = a.Dbl("--hours", 9); + string dataDir = a.Str("--data", "data"); + ulong seed = a.ULong("--seed", 1); + float learningRate = a.Flt("--lr", 3e-4f); + int width = a.Int("--width", 512); + bool evalOnly = a.Has("--eval-only"); + bool grow = a.Has("--grow"); // progressively grow the net wider+deeper mid-training (Net2Net) + int growEvery = a.Int("--grow-every", 4096); // samples between growth steps (with --grow) LabHost.Run(args, dataDir, hours, evalOnly, useGpu: false, _ => new CubeImitationCampaign(seed, learningRate, width, grow, growEvery), diff --git a/tools/MintPlayer.AI.ReinforcementLearning.Lab/CubePolicyLab.cs b/tools/MintPlayer.AI.ReinforcementLearning.Lab/CubePolicyLab.cs index b01eb1c..c51fa09 100644 --- a/tools/MintPlayer.AI.ReinforcementLearning.Lab/CubePolicyLab.cs +++ b/tools/MintPlayer.AI.ReinforcementLearning.Lab/CubePolicyLab.cs @@ -1,4 +1,3 @@ -using System.Globalization; using Microsoft.Extensions.DependencyInjection; using MintPlayer.AI.ReinforcementLearning.Ilgpu; @@ -11,31 +10,18 @@ internal static class CubePolicyLab { public static void Run(string[] args) { - double hours = 24; - string dataDir = "data"; - ulong seed = 1; - float learningRate = 3e-4f; - int width = 512; - int maxScramble = 30; - int beamWidth = 2_000; - int evalEpisodes = 20; - bool evalOnly = false; - bool grow = false; // --grow : progressively grow the net wider+deeper mid-training (Net2Net) - int growEvery = 50_000; // --grow-every : samples between growth steps (with --grow) - for (int i = 0; i < args.Length; i++) - { - if (args[i] == "--hours" && i + 1 < args.Length) hours = double.Parse(args[++i], CultureInfo.InvariantCulture); - else if (args[i] == "--data" && i + 1 < args.Length) dataDir = args[++i]; - else if (args[i] == "--seed" && i + 1 < args.Length) seed = ulong.Parse(args[++i]); - else if (args[i] == "--lr" && i + 1 < args.Length) learningRate = float.Parse(args[++i], CultureInfo.InvariantCulture); - else if (args[i] == "--width" && i + 1 < args.Length) width = int.Parse(args[++i]); - else if (args[i] == "--max-scramble" && i + 1 < args.Length) maxScramble = int.Parse(args[++i]); - else if (args[i] == "--beam" && i + 1 < args.Length) beamWidth = int.Parse(args[++i]); - else if (args[i] == "--episodes" && i + 1 < args.Length) evalEpisodes = int.Parse(args[++i]); - else if (args[i] == "--eval-only") evalOnly = true; - else if (args[i] == "--grow") grow = true; - else if (args[i] == "--grow-every" && i + 1 < args.Length) growEvery = int.Parse(args[++i]); - } + var a = new CliArgs(args); + double hours = a.Dbl("--hours", 24); + string dataDir = a.Str("--data", "data"); + ulong seed = a.ULong("--seed", 1); + float learningRate = a.Flt("--lr", 3e-4f); + int width = a.Int("--width", 512); + int maxScramble = a.Int("--max-scramble", 30); + int beamWidth = a.Int("--beam", 2_000); + int evalEpisodes = a.Int("--episodes", 20); + bool evalOnly = a.Has("--eval-only"); + bool grow = a.Has("--grow"); // progressively grow the net wider+deeper mid-training (Net2Net) + int growEvery = a.Int("--grow-every", 50_000); // samples between growth steps (with --grow) // GPU: the cube nets are large enough to win on GPU, so the campaign runs on the AdaptiveBackend // (useGpu: true → LabHost registers it and this build pulls it from the container). diff --git a/tools/MintPlayer.AI.ReinforcementLearning.Lab/FruitCakeLab.cs b/tools/MintPlayer.AI.ReinforcementLearning.Lab/FruitCakeLab.cs index 8085b39..a99f6fb 100644 --- a/tools/MintPlayer.AI.ReinforcementLearning.Lab/FruitCakeLab.cs +++ b/tools/MintPlayer.AI.ReinforcementLearning.Lab/FruitCakeLab.cs @@ -1,5 +1,3 @@ -using System.Globalization; - /// /// `--game fruitcake` entry point: parses the campaign flags and runs the score-maximizing /// on the shared . CPU-only (the small 41→14 @@ -10,57 +8,31 @@ internal static class FruitCakeLab { public static void Run(string[] args) { - double hours = 1; - string dataDir = "data"; - ulong seed = 1; - int chunkSteps = 2_000; // drops per chunk (each drop = simulate-to-rest; far costlier than a grid step) - long targetSteps = 0; // 0 = time-bounded only (score-maximizing); pass --steps N for a hard drop cap - int evalEpisodes = 10; - float learningRate = 5e-4f; - float explore = 1.0f; // ε-start; pass a low value (e.g. 0.2) to refine a warm-started net - int[] hidden = [256, 256]; // --hidden : trunk widths for the Dueling Q-net - double gamma = 0.99; // --gamma : discount; high for the long drop horizon (PRD bundle uses 0.997) - int nStep = 1; // --nstep : n-step return horizon (1 = single-step DQN) - bool shape = false; // --shape : enable reward shaping (tier-reached bonus + potential-based adjacency/height) - bool evalOnly = false; - bool noisy = false; // --noisy : NoisyNets exploration (learned σ) instead of ε-greedy - bool ab = false; // --ab : head-to-head eval of --data's net vs --baseline's net (no training) - string baselineDir = ""; // --baseline : the net to compare --data's net against - int abEpisodes = 200; // --ab-episodes : paired greedy games per net (averages out eval noise) - bool searchEval = false; // --search-eval : F1 forward-model search vs plain net greedy, on --data's net - int depth = 2; // --depth : search lookahead (1 or 2) - int topK = 5; // --topk : depth-2 first-ply expansion width - int topK2 = 3; // --topk2 : deeper-ply expansion width (depth 3) - string leaf = "net"; // --leaf : search leaf value (net | height | tierpot | blend) - bool grow = false; // --grow : progressively grow the net wider+deeper mid-training (Net2Net demo) - int growEvery = 2000; // --grow-every : drops between growth steps (with --grow) - for (int i = 0; i < args.Length; i++) - { - if (args[i] == "--hours" && i + 1 < args.Length) hours = double.Parse(args[++i], CultureInfo.InvariantCulture); - else if (args[i] == "--data" && i + 1 < args.Length) dataDir = args[++i]; - else if (args[i] == "--seed" && i + 1 < args.Length) seed = ulong.Parse(args[++i]); - else if (args[i] == "--chunk-steps" && i + 1 < args.Length) chunkSteps = int.Parse(args[++i]); - else if (args[i] == "--steps" && i + 1 < args.Length) targetSteps = long.Parse(args[++i], CultureInfo.InvariantCulture); - else if (args[i] == "--episodes" && i + 1 < args.Length) evalEpisodes = int.Parse(args[++i]); - else if (args[i] == "--lr" && i + 1 < args.Length) learningRate = float.Parse(args[++i], CultureInfo.InvariantCulture); - else if (args[i] == "--explore" && i + 1 < args.Length) explore = float.Parse(args[++i], CultureInfo.InvariantCulture); - else if (args[i] == "--hidden" && i + 1 < args.Length) hidden = args[++i].Split(',').Select(int.Parse).ToArray(); - else if (args[i] == "--gamma" && i + 1 < args.Length) gamma = double.Parse(args[++i], CultureInfo.InvariantCulture); - else if (args[i] == "--nstep" && i + 1 < args.Length) nStep = int.Parse(args[++i]); - else if (args[i] == "--shape") shape = true; - else if (args[i] == "--eval-only") evalOnly = true; - else if (args[i] == "--noisy") noisy = true; - else if (args[i] == "--ab") ab = true; - else if (args[i] == "--baseline" && i + 1 < args.Length) baselineDir = args[++i]; - else if (args[i] == "--ab-episodes" && i + 1 < args.Length) abEpisodes = int.Parse(args[++i]); - else if (args[i] == "--search-eval") searchEval = true; - else if (args[i] == "--depth" && i + 1 < args.Length) depth = int.Parse(args[++i]); - else if (args[i] == "--topk" && i + 1 < args.Length) topK = int.Parse(args[++i]); - else if (args[i] == "--topk2" && i + 1 < args.Length) topK2 = int.Parse(args[++i]); - else if (args[i] == "--leaf" && i + 1 < args.Length) leaf = args[++i]; - else if (args[i] == "--grow") grow = true; - else if (args[i] == "--grow-every" && i + 1 < args.Length) growEvery = int.Parse(args[++i]); - } + var a = new CliArgs(args); + double hours = a.Dbl("--hours", 1); + string dataDir = a.Str("--data", "data"); + ulong seed = a.ULong("--seed", 1); + int chunkSteps = a.Int("--chunk-steps", 2_000); // drops per chunk (each drop = simulate-to-rest; far costlier than a grid step) + long targetSteps = a.Long("--steps", 0); // 0 = time-bounded only (score-maximizing); a hard drop cap otherwise + int evalEpisodes = a.Int("--episodes", 10); + float learningRate = a.Flt("--lr", 5e-4f); + float explore = a.Flt("--explore", 1.0f); // ε-start; pass a low value (e.g. 0.2) to refine a warm-started net + int[] hidden = a.Ints("--hidden", [256, 256]); // trunk widths for the Dueling Q-net + double gamma = a.Dbl("--gamma", 0.99); // discount; high for the long drop horizon (PRD bundle uses 0.997) + int nStep = a.Int("--nstep", 1); // n-step return horizon (1 = single-step DQN) + bool shape = a.Has("--shape"); // enable reward shaping (tier-reached bonus + potential-based adjacency/height) + bool evalOnly = a.Has("--eval-only"); + bool noisy = a.Has("--noisy"); // NoisyNets exploration (learned σ) instead of ε-greedy + bool ab = a.Has("--ab"); // head-to-head eval of --data's net vs --baseline's net (no training) + string baselineDir = a.Str("--baseline", ""); // the net to compare --data's net against + int abEpisodes = a.Int("--ab-episodes", 200); // paired greedy games per net (averages out eval noise) + bool searchEval = a.Has("--search-eval"); // F1 forward-model search vs plain net greedy, on --data's net + int depth = a.Int("--depth", 2); // search lookahead (1 or 2) + int topK = a.Int("--topk", 5); // depth-2 first-ply expansion width + int topK2 = a.Int("--topk2", 3); // deeper-ply expansion width (depth 3) + string leaf = a.Str("--leaf", "net"); // search leaf value (net | height | tierpot | blend) + bool grow = a.Has("--grow"); // progressively grow the net wider+deeper mid-training (Net2Net demo) + int growEvery = a.Int("--grow-every", 2000); // drops between growth steps (with --grow) // A growing run starts from the tiny first stage and adds capacity mid-training (Net2Wider/DeeperNet). if (grow) hidden = DqnGrowth.Start; diff --git a/tools/MintPlayer.AI.ReinforcementLearning.Lab/RushHourLab.cs b/tools/MintPlayer.AI.ReinforcementLearning.Lab/RushHourLab.cs index b30ac41..a5094f3 100644 --- a/tools/MintPlayer.AI.ReinforcementLearning.Lab/RushHourLab.cs +++ b/tools/MintPlayer.AI.ReinforcementLearning.Lab/RushHourLab.cs @@ -1,5 +1,3 @@ -using System.Globalization; - /// /// The Lab's default `--game rushhour` entry point: parses the campaign flags and runs the /// (PLAN M16) on the shared (PLAN M25). The @@ -9,23 +7,14 @@ internal static class RushHourLab { public static void Run(string[] args) { - double hours = 9; - string dataDir = "data"; - ulong seed = 1; - float learningRate = 3e-4f; - bool evalOnly = false; - bool grow = false; // --grow : progressively grow the net wider+deeper mid-training (Net2Net) - int growEvery = 2048; // --grow-every : samples between growth steps (with --grow) - for (int i = 0; i < args.Length; i++) - { - if (args[i] == "--hours" && i + 1 < args.Length) hours = double.Parse(args[++i], CultureInfo.InvariantCulture); - else if (args[i] == "--data" && i + 1 < args.Length) dataDir = args[++i]; - else if (args[i] == "--seed" && i + 1 < args.Length) seed = ulong.Parse(args[++i]); - else if (args[i] == "--lr" && i + 1 < args.Length) learningRate = float.Parse(args[++i], CultureInfo.InvariantCulture); - else if (args[i] == "--eval-only") evalOnly = true; - else if (args[i] == "--grow") grow = true; - else if (args[i] == "--grow-every" && i + 1 < args.Length) growEvery = int.Parse(args[++i]); - } + var a = new CliArgs(args); + double hours = a.Dbl("--hours", 9); + string dataDir = a.Str("--data", "data"); + ulong seed = a.ULong("--seed", 1); + float learningRate = a.Flt("--lr", 3e-4f); + bool evalOnly = a.Has("--eval-only"); + bool grow = a.Has("--grow"); // progressively grow the net wider+deeper mid-training (Net2Net) + int growEvery = a.Int("--grow-every", 2048); // samples between growth steps (with --grow) LabHost.Run(args, dataDir, hours, evalOnly, useGpu: false, _ => new RushHourImitationCampaign(seed, learningRate, grow, growEvery), diff --git a/tools/MintPlayer.AI.ReinforcementLearning.Lab/SnakeLab.cs b/tools/MintPlayer.AI.ReinforcementLearning.Lab/SnakeLab.cs index 3e6e624..9f5e495 100644 --- a/tools/MintPlayer.AI.ReinforcementLearning.Lab/SnakeLab.cs +++ b/tools/MintPlayer.AI.ReinforcementLearning.Lab/SnakeLab.cs @@ -1,5 +1,4 @@ using System.Diagnostics; -using System.Globalization; using MintPlayer.AI.ReinforcementLearning.Environments.Snake; /// @@ -12,63 +11,41 @@ internal static class SnakeLab { public static void Run(string[] args) { - double hours = 1; - string dataDir = "data"; - ulong seed = 1; - int trainGrid = 6; - int evalGrid = 12; - int chunkSteps = 5_000; - long targetSteps = 100_000; // the proven M22 budget (curve plateaus ~30k); --steps 0 = time-bounded only - int evalEpisodes = 20; - float learningRate = 5e-4f; - float explore = 1.0f; // ε-start; pass a low value (e.g. 0.2) to refine a warm-started net rather than re-randomize it - int[] hidden = [128, 128]; // --hidden 256,256 : trunk widths for the Dueling Q-net - double gamma = 0.99; // --gamma : discount; higher = longer planning horizon (needed for long-snake routing) - float stepPenalty = -0.01f; // --step-penalty : per-step reward; ~0 removes the efficiency pressure that encourages safe starvation - bool safeMask = false; // --safe-mask : forbid moves that flood-fill into a region too small for the body (anti-self-trap shield) - bool evalOnly = false; - bool grow = false; // --grow : progressively grow the net wider+deeper mid-training (Net2Net demo) - int growEvery = 5000; // --grow-every : steps between growth steps (with --grow) + var a = new CliArgs(args); + double hours = a.Dbl("--hours", 1); + string dataDir = a.Str("--data", "data"); + ulong seed = a.ULong("--seed", 1); + int trainGrid = a.Int("--train-grid", 6); + int evalGrid = a.Int("--eval-grid", 12); + int chunkSteps = a.Int("--chunk-steps", 5_000); + long targetSteps = a.Long("--steps", 100_000); // the proven M22 budget (curve plateaus ~30k); --steps 0 = time-bounded only + int evalEpisodes = a.Int("--episodes", 20); + float learningRate = a.Flt("--lr", 5e-4f); + float explore = a.Flt("--explore", 1.0f); // ε-start; pass a low value (e.g. 0.2) to refine a warm-started net + int[] hidden = a.Ints("--hidden", [128, 128]); // trunk widths for the Dueling Q-net + double gamma = a.Dbl("--gamma", 0.99); // discount; higher = longer planning horizon (long-snake routing) + float stepPenalty = a.Flt("--step-penalty", -0.01f); // per-step reward; ~0 removes the safe-starvation pressure + bool safeMask = a.Has("--safe-mask"); // forbid moves that flood-fill into too-small a region (anti-self-trap) + bool evalOnly = a.Has("--eval-only"); + bool grow = a.Has("--grow"); // progressively grow the net wider+deeper mid-training (Net2Net demo) + int growEvery = a.Int("--grow-every", 5000); // steps between growth steps (with --grow) - // --search : skip training and evaluate the net-guided look-ahead planner (M34) instead of greedy Q. The - // net is only a leaf tiebreak, so the config defaults reproduce PR #11's shipped depth-20/beam-32 sweep. - bool search = false; - string netPath = Path.Combine("src", "RLDemo.Web", "wwwroot", "models", "snake-net.ckpt"); + // --search : skip training and evaluate the net-guided look-ahead planner (M34) instead of greedy Q. The net + // is only a leaf tiebreak, so the config defaults reproduce PR #11's shipped depth-20/beam-32 sweep. + bool search = a.Has("--search"); + string netPath = a.Str("--net", Path.Combine("src", "RLDemo.Web", "wwwroot", "models", "snake-net.ckpt")); var cfg = new SnakeSearchConfig(); - for (int i = 0; i < args.Length; i++) + cfg = cfg with { - if (args[i] == "--search") search = true; - else if (args[i] == "--net" && i + 1 < args.Length) netPath = args[++i]; - else if (args[i] == "--depth" && i + 1 < args.Length) cfg = cfg with { MaxDepth = int.Parse(args[++i]) }; - else if (args[i] == "--beam" && i + 1 < args.Length) cfg = cfg with { BeamWidth = int.Parse(args[++i]) }; - else if (args[i] == "--w-food" && i + 1 < args.Length) cfg = cfg with { FoodWeight = double.Parse(args[++i], CultureInfo.InvariantCulture) }; - else if (args[i] == "--w-trap" && i + 1 < args.Length) cfg = cfg with { TrapPenalty = double.Parse(args[++i], CultureInfo.InvariantCulture) }; - else if (args[i] == "--w-net" && i + 1 < args.Length) cfg = cfg with { NetWeight = double.Parse(args[++i], CultureInfo.InvariantCulture) }; - else if (args[i] == "--w-space" && i + 1 < args.Length) cfg = cfg with { SpaceWeight = double.Parse(args[++i], CultureInfo.InvariantCulture) }; - else if (args[i] == "--w-dist" && i + 1 < args.Length) cfg = cfg with { FoodDistWeight = double.Parse(args[++i], CultureInfo.InvariantCulture) }; - else if (args[i] == "--w-ratio" && i + 1 < args.Length) cfg = cfg with { SpaceRatioWeight = double.Parse(args[++i], CultureInfo.InvariantCulture) }; - } - - for (int i = 0; i < args.Length; i++) - { - if (args[i] == "--hours" && i + 1 < args.Length) hours = double.Parse(args[++i], CultureInfo.InvariantCulture); - else if (args[i] == "--data" && i + 1 < args.Length) dataDir = args[++i]; - else if (args[i] == "--seed" && i + 1 < args.Length) seed = ulong.Parse(args[++i]); - else if (args[i] == "--train-grid" && i + 1 < args.Length) trainGrid = int.Parse(args[++i]); - else if (args[i] == "--eval-grid" && i + 1 < args.Length) evalGrid = int.Parse(args[++i]); - else if (args[i] == "--chunk-steps" && i + 1 < args.Length) chunkSteps = int.Parse(args[++i]); - else if (args[i] == "--steps" && i + 1 < args.Length) targetSteps = long.Parse(args[++i], CultureInfo.InvariantCulture); - else if (args[i] == "--episodes" && i + 1 < args.Length) evalEpisodes = int.Parse(args[++i]); - else if (args[i] == "--lr" && i + 1 < args.Length) learningRate = float.Parse(args[++i], CultureInfo.InvariantCulture); - else if (args[i] == "--explore" && i + 1 < args.Length) explore = float.Parse(args[++i], CultureInfo.InvariantCulture); - else if (args[i] == "--hidden" && i + 1 < args.Length) hidden = args[++i].Split(',').Select(int.Parse).ToArray(); - else if (args[i] == "--gamma" && i + 1 < args.Length) gamma = double.Parse(args[++i], CultureInfo.InvariantCulture); - else if (args[i] == "--step-penalty" && i + 1 < args.Length) stepPenalty = float.Parse(args[++i], CultureInfo.InvariantCulture); - else if (args[i] == "--safe-mask") safeMask = true; - else if (args[i] == "--eval-only") evalOnly = true; - else if (args[i] == "--grow") grow = true; - else if (args[i] == "--grow-every" && i + 1 < args.Length) growEvery = int.Parse(args[++i]); - } + MaxDepth = a.Int("--depth", cfg.MaxDepth), + BeamWidth = a.Int("--beam", cfg.BeamWidth), + FoodWeight = a.Dbl("--w-food", cfg.FoodWeight), + TrapPenalty = a.Dbl("--w-trap", cfg.TrapPenalty), + NetWeight = a.Dbl("--w-net", cfg.NetWeight), + SpaceWeight = a.Dbl("--w-space", cfg.SpaceWeight), + FoodDistWeight = a.Dbl("--w-dist", cfg.FoodDistWeight), + SpaceRatioWeight = a.Dbl("--w-ratio", cfg.SpaceRatioWeight), + }; // A growing run starts from the tiny first stage and adds capacity mid-training (Net2Wider/DeeperNet). if (grow) hidden = DqnGrowth.Start; From 4e351d342e9e880a8246c5966e29096f5ad2b90f Mon Sep 17 00:00:00 2001 From: PieterjanDeClippel Date: Sun, 12 Jul 2026 20:51:02 +0200 Subject: [PATCH 10/12] M38 B5 (cont.): remove the redundant per-component wake-lock visibility handler MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit snake / mountaincar / fruit-cake each carried a byte-identical `onVisibilityChange()` (+ a `(document:visibilitychange)` host binding) that re-acquired the screen wake lock on foreground. But `ScreenWakeLock` ALREADY does this itself: `acquire()` registers its own `visibilitychange` listener that re-requests while desired, and `release()` removes it. So the component handlers were a redundant no-op (acquire() early-returns when already desired). Removed all three — the re-acquire concern lives solely in the deep service (information hiding), and the triplicated method is gone. Verified live (ran the host, Playwright): the Angular bundle compiles clean; Snake "Watch AI" plays (food 17→20), dispatching visibilitychange hidden→visible throws nothing and the AI keeps ticking, and the console has zero errors/warnings across the flow. (Left alone: the director watch-loop — its shared part is a 3-line setInterval wrapper around game-specific render/status, i.e. a shallow abstraction not worth extracting.) Co-Authored-By: Claude Opus 4.8 (1M context) --- src/RLDemo.Web/ClientApp/src/app/fruit-cake/fruit-cake.ts | 7 ------- .../ClientApp/src/app/mountaincar/mountaincar.ts | 7 ------- src/RLDemo.Web/ClientApp/src/app/snake/snake.ts | 8 +------- 3 files changed, 1 insertion(+), 21 deletions(-) diff --git a/src/RLDemo.Web/ClientApp/src/app/fruit-cake/fruit-cake.ts b/src/RLDemo.Web/ClientApp/src/app/fruit-cake/fruit-cake.ts index 6eb01b2..48953a1 100644 --- a/src/RLDemo.Web/ClientApp/src/app/fruit-cake/fruit-cake.ts +++ b/src/RLDemo.Web/ClientApp/src/app/fruit-cake/fruit-cake.ts @@ -22,7 +22,6 @@ import { ScreenWakeLock } from '../screen-wake-lock'; host: { '(document:fullscreenchange)': 'onFullscreenChange()', '(document:webkitfullscreenchange)': 'onFullscreenChange()', - '(document:visibilitychange)': 'onVisibilityChange()', }, }) export class FruitCake implements AfterViewInit { @@ -82,12 +81,6 @@ export class FruitCake implements AfterViewInit { } } - // A backgrounded tab is throttled/frozen by the OS, which also drops the screen wake-lock. On return to the - // foreground still in watch mode, re-acquire it; the director just resumes on the next animation frame. - protected onVisibilityChange(): void { - if (document.visibilityState === 'visible' && this.mode() === 'watch') void this.wakeLock.acquire(); - } - private readonly frame = (nowMs: number): void => { const dt = this.lastMs ? Math.min(0.25, (nowMs - this.lastMs) / 1000) : 0; // clamp to avoid a spiral after a stall this.lastMs = nowMs; diff --git a/src/RLDemo.Web/ClientApp/src/app/mountaincar/mountaincar.ts b/src/RLDemo.Web/ClientApp/src/app/mountaincar/mountaincar.ts index 60eebc9..784ae5c 100644 --- a/src/RLDemo.Web/ClientApp/src/app/mountaincar/mountaincar.ts +++ b/src/RLDemo.Web/ClientApp/src/app/mountaincar/mountaincar.ts @@ -15,7 +15,6 @@ import { ScreenWakeLock } from '../screen-wake-lock'; host: { '(window:keydown)': 'onKeyDown($event)', '(window:keyup)': 'onKeyUp($event)', - '(document:visibilitychange)': 'onVisibilityChange()', }, }) export class MountainCar { @@ -88,12 +87,6 @@ export class MountainCar { if (this.mode() !== 'idle') this.mode.set('idle'); } - // A backgrounded tab is throttled by the OS and drops the wake-lock; on returning to the foreground still in - // watch mode, re-acquire it (the director keeps ticking — no socket to reopen). - protected onVisibilityChange(): void { - if (document.visibilityState === 'visible' && this.mode() === 'watch') void this.wakeLock.acquire(); - } - private clearTimer(): void { if (this.timer !== null) { clearInterval(this.timer); this.timer = null; } } diff --git a/src/RLDemo.Web/ClientApp/src/app/snake/snake.ts b/src/RLDemo.Web/ClientApp/src/app/snake/snake.ts index 93886cd..fd9305c 100644 --- a/src/RLDemo.Web/ClientApp/src/app/snake/snake.ts +++ b/src/RLDemo.Web/ClientApp/src/app/snake/snake.ts @@ -18,7 +18,7 @@ const HUMAN_TICK_MS = 150; selector: 'app-snake', templateUrl: './snake.html', styleUrl: './snake.scss', - host: { '(window:keydown)': 'onKey($event)', '(document:visibilitychange)': 'onVisibilityChange()' }, + host: { '(window:keydown)': 'onKey($event)' }, }) export class Snake { private readonly wakeLock = inject(ScreenWakeLock); @@ -95,12 +95,6 @@ export class Snake { if (this.mode() !== 'idle') this.mode.set('idle'); } - // A backgrounded tab is throttled by the OS and drops the wake-lock; on returning to the foreground still in - // watch mode, re-acquire it (the director just keeps ticking — no socket to reopen). - protected onVisibilityChange(): void { - if (document.visibilityState === 'visible' && this.mode() === 'watch') void this.wakeLock.acquire(); - } - private render(body: number[], food: number, eaten: number): void { this.foodEaten.set(eaten); this.renderer?.push(body, food, eaten); From 3b2ac6df1977af21d42c89050d62c603b191540f Mon Sep 17 00:00:00 2001 From: PieterjanDeClippel Date: Sun, 12 Jul 2026 20:51:45 +0200 Subject: [PATCH 11/12] docs(M38): mark B3 CliArgs + B5 P7 complete and live-verified Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/prd/BOILERPLATE_REDUCTION_PRD.md | 2 +- docs/prd/PLAN.md | 22 +++++++++++++--------- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/docs/prd/BOILERPLATE_REDUCTION_PRD.md b/docs/prd/BOILERPLATE_REDUCTION_PRD.md index 91a6359..4dba343 100644 --- a/docs/prd/BOILERPLATE_REDUCTION_PRD.md +++ b/docs/prd/BOILERPLATE_REDUCTION_PRD.md @@ -1,6 +1,6 @@ # Reduce per-game boilerplate — campaigns, web services, frontend — PRD -**Status:** B0–B5 landed · 2026-07-12 · branch `m38-reduce-boilerplate-plan`, [PR #31](https://github.com/MintPlayer/MintPlayer.AI/pull/31) · **supersedes the stale [PR #27](https://github.com/MintPlayer/MintPlayer.AI/pull/27)** · (deferred: B3 `CliArgs`, B5 frontend P7/P8 — see PLAN M38) +**Status:** B0–B5 complete (incl. B3 `CliArgs` + B5 P7) · 2026-07-12 · branch `m38-reduce-boilerplate-plan`, [PR #31](https://github.com/MintPlayer/MintPlayer.AI/pull/31) · **supersedes the stale [PR #27](https://github.com/MintPlayer/MintPlayer.AI/pull/27)** · verified live (web host + Playwright) — see PLAN M38. (Only B5 P8, a shallow director-loop wrapper, intentionally left.) **Owner:** Pieterjan **Milestone:** [PLAN.md](PLAN.md) M38 · **Depends on:** nothing new — a behaviour-preserving refactor across the Lab training harness (§8 of [../ARCHITECTURE.md](../ARCHITECTURE.md)), the web model-service layer (§7), and the Angular playground (§10). diff --git a/docs/prd/PLAN.md b/docs/prd/PLAN.md index c87469d..d0bb485 100644 --- a/docs/prd/PLAN.md +++ b/docs/prd/PLAN.md @@ -1288,15 +1288,19 @@ Screenshots: `docs/screenshots/m36-network-grows.png` (DQN), `m37-policy-net-gro ## M38 — Reduce per-game boilerplate (campaigns · web services · frontend) *(2026-07-12; branch `m38-reduce-boilerplate-plan`, PR #31; supersedes stale PR #27; see `BOILERPLATE_REDUCTION_PRD.md`)* ✅ -**Status (2026-07-12):** B0–B5 landed on `m38-reduce-boilerplate-plan` (PR #31), each behaviour-preserving with its -own commit; solution builds 0/0 and 320 fast tests stay green after every step. **B2 is SHA256-bitwise-verified** -(fresh seed-1 `snake`/`fruitcake` produce byte-identical deployable + resume checkpoints vs the pre-refactor build). -Two sub-items are **deliberately deferred**: the B3 `CliArgs`/`CommonLabArgs` parse-loop rewrite (the culture -inconsistency it targets is theoretical for integer CLI args, and rewriting all six parse loops has real regression -risk with no automated gate — low reward, higher risk), and the B5 frontend watch-scaffolding dedup (P7/P8 — the -wake lock is already a deep shared service, the residual per-component wiring is ~3 lines, and this repo forbids -running the Angular build/test locally so the change can't be verified here). `LabHost.Run` (B3) still removed the -bulk of the Lab boilerplate (the ~15-line bootstrap tail × 6). +**Status (2026-07-12):** B0–B5 complete on `m38-reduce-boilerplate-plan` (PR #31), each behaviour-preserving with its +own commit; solution builds 0/0 and the fast suite (now **326** tests) stays green after every step. **B2 is +SHA256-bitwise-verified** (fresh seed-1 `snake`/`fruitcake` produce byte-identical deployable + resume checkpoints vs +the pre-refactor build), and the whole branch was independently re-reviewed line-by-line (no behavioural divergence) +and **exercised live**: the web host loaded the real shipped checkpoints (`StartupCheckpoint`/`RefreshingCheckpoint`, +incl. the Cube `onReload` GPU-resident rebuild on a real RTX 3060), the Lab games ran through `LabHost` (CPU + GPU), +and the Snake page's Watch-AI was Playwright-verified (plays, survives a visibility toggle, zero console errors). +**B3 `CliArgs`** landed too (typed, bounds-safe, culture-invariant flag reads across five labs — the int/long/ulong +locale inconsistency fixed; `CubeDaviLab`'s bespoke config-precedence block left as-is; 6 new unit tests incl. a +comma-decimal culture test). **B5 P7** landed as a *removal*: the byte-identical per-component `onVisibilityChange` +was redundant with `ScreenWakeLock`'s own re-acquire, so it's deleted from all three components. The one item +**intentionally not done** is B5 P8 (a shared director watch-loop) — its shared part is a 3-line `setInterval` +wrapper around game-specific render/status, a shallow abstraction not worth extracting. **Problem.** Adding each game left near-identical copy-paste in three layers, and several copies have already From 60f0427485cc30712cb3fc8ccda970718e873729 Mon Sep 17 00:00:00 2001 From: Pieterjan Date: Mon, 13 Jul 2026 16:31:47 +0200 Subject: [PATCH 12/12] =?UTF-8?q?M39:=20self-play=20training=20(Connect-4?= =?UTF-8?q?=20rails=20=E2=86=92=20perft-verified=20chess)=20(#32)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * docs(M39): PRD + plan for self-play training (Connect-4 → chess) A three-agent investigation established that ~70% of the outer loop already exists (PolicyValueNet, the soft-CE+value train step, ITrainingCampaign/ CampaignRunner, model store + checkpoint format, the M38 AdamState/TrainWindow/ PolicyGrowth plumbing, action masking, RNG streams, --viz, the A/B harness). Adds docs/prd/CHESS_SELFPLAY_PRD.md and an M39 milestone in PLAN.md: AlphaZero-style self-play (MCTS + the two-headed net) on ONE new deep seam, IZeroSumGame in Core/Planning (a sibling to IDeterministicModel). Phased to de-risk the novel machinery on Connect-4 first (cheap CPU convergence + a negamax oracle), then land chess as the seam's second consumer (perft-gated legal movegen). Honest scope: MLP-only net → legal, steadily- improving play, not engine strength; self-play is CPU-bound. Docs only. Stacked on the M38 branch (reuses its Lab plumbing). Co-Authored-By: Claude Opus 4.8 (1M context) * M39.1: self-play rails (IZeroSumGame + MCTS + SelfPlayCampaign) on Connect-4 The reusable self-play stack, proven on Connect-4 before chess's rules surface (PLAN M39.1). New and self-contained: - Core/Planning/IZeroSumGame — the two-player zero-sum game seam (side-to-move + win/loss/draw + per-state legal moves), a sibling to IDeterministicModel that MCTS and the self-play trainer both consume. - Core/Planning/Mcts.cs — AlphaZero PUCT: select (Q+U) → expand-with-priors → net-leaf eval → value backup negated every ply (zero-sum), Dirichlet root noise, returns the root visit-count π. Composes over the seam + an Evaluate delegate exactly as ValueGuidedSearch composes over IDeterministicModel. - Environments/Connect4/{Connect4Game : IZeroSumGame, Connect4Solver} — the cheap first consumer + a depth-limited negamax test oracle. - Lab/PolicyValueTraining.TrainStep — soft-CE(π) + MSE(tanh value, z), the AlphaZero loss (generalized from CubePolicyTraining to raw arrays). - Lab/SelfPlayCampaign : ITrainingCampaign — plays K MCTS self-play games/chunk → (obs, π, z) rolling window → trains PolicyValueNet; Evaluate = win-rate vs a random-legal opponent; reuses AdamState + TrainWindow + telemetry + the model store/checkpoint format. --game connect4 dispatch. REUSED unchanged: PolicyValueNet (the AZ net), Adam, the autograd ops, ITrainingCampaign/CampaignRunner/AIHost, IModelStore + checkpoint format, AdamState/TrainWindow, SeedSequence streams, --viz telemetry. (Growth via PolicyGrowth deferred — it needs an IGrowableTrunkNet wrapper.) Gate met: 5 fast Connect-4/MCTS tests green (incl. MCTS finding a forced win purely by search, agreeing with negamax) + the Slow self-play resume-roundtrip contract; 331 fast tests total green. A 4-minute `--game connect4` run from random init reaches 100% vs a random opponent with a falling policy loss (1.37→1.22) — the pipeline learns end-to-end. (A discriminating frozen-baseline arena lands with chess in M39.2.) Co-Authored-By: Claude Opus 4.8 (1M context) * M39.2: chess as the self-play stack's second consumer (perft-verified) Chess plugs into the M39.1 rails — SelfPlayCampaign, MCTS, PolicyValueNet, and the training step are REUSED UNCHANGED. Only chess-specific code is new: - Environments/Chess/ChessBoard.cs (ChessRules + ChessState) — full legal move generation (castling through/into check, en passant, promotion, pins/checks) + make-move + attack detection + terminal detection (checkmate, stalemate, 50-move, insufficient material). Correctness-first mailbox engine. - ChessFen.cs — FEN parse/render for test positions. - ChessMoveEncoding.cs — the AlphaZero 64×73 = 4672 action space (queen/knight/ underpromotion planes); encode + decode (queen-promo inferred on apply). - ChessGame : IZeroSumGame — 18-plane observation, legal-moves→ indices, apply-by-index. - Lab/ChessLab.cs + --game chess dispatch. Gates met: - PERFT (the hard movegen gate): 25/25 published node counts match, incl. startpos depth 5 = 4,865,609 and Kiwipete depth 4 = 4,085,603 (3s Release). - Move encoding: encode→decode→Apply round-trips every legal move on positions rich in castling/en-passant/all promotion flavours, no index collisions; Result detects fool's-mate and stalemate. - End-to-end: a 2-minute `--game chess` run from random init plays legal chess and reaches 62.5% vs a random-legal opponent (16 sims/move), reusing the rails. - 355 fast tests green (+24 chess). Threefold-repetition, perspective canonicalization, batched-leaf MCTS, and the anti-exploitation league/opening-diversity levers (see the PRD robustness note added here) are M39.3. Honest scope: MLP-over-flattened-planes → legal, improving play, not engine strength. Co-Authored-By: Claude Opus 4.8 (1M context) * M39.3 (robustness slice): opponent-diversity self-play The direct code answer to "two self-trained AIs co-adapt into a narrow style, so a novel/weak human move disorients the net and it loses to a beginner" (the exploitability failure — real even at superhuman level, cf. the KataGo cyclic-group exploit). SelfPlayCampaign gains --opponent-random : that fraction of games are learner(MCTS)-vs-random-legal instead of pure self-play, so the net trains on the off-distribution positions a blunder/unexpected move reaches and learns to punish them. Only the learner's positions are recorded, with a constant learner-perspective outcome z (distinct from self-play's alternating z). Default 0 → behaviour unchanged. Wired into --game connect4 and --game chess. (Primary robustness is still search-from-the-actual-position; Dirichlet noise + temperature already broaden self-play. League/opening-diversity/adversarial fine-tune remain M39.3 future work — see the PRD robustness note. NoisyNets is NOT the tool here — it perturbs the policy, not position coverage.) Behaviour-preserving at the default; 356 fast tests green + the new learner-vs-random contract test (frac 1.0). PLAN M39 marked M39.1+M39.2 shipped. Co-Authored-By: Claude Opus 4.8 (1M context) * M39: chess --demo — play/print one AI game (net+MCTS vs random) as FENs A small dev helper to WATCH the self-taught net play: `--game chess --demo` loads the trained checkpoint and plays the net (White, MCTS) vs a random-legal opponent, printing the FEN after each ply. Used to capture a game for a browser replay; not part of training. Co-Authored-By: Claude Opus 4.8 (1M context) * docs(M40): plan — play the chess AI in the browser via Polyglot single-source The FruitCake pattern for chess: write the inference path once in a .pg, transpile to C# (training) + TypeScript (browser); the browser downloads and parses the .ckpt and runs net+MCTS client-side — zero server inference. Feasibility VERIFIED by transpiling a probe: Polyglot supports Math.exp/tanh/ log/sqrt + bitwise ops (transcendentals aren't bit-exact C#/JS, but chess inference doesn't need that). Adds docs/prd/CHESS_WEB_POLYGLOT_PRD.md (incl. a §8 reference appendix: files to port, the exact .ckpt byte-format for the TS parser, store ids, commands) + an M40 milestone in PLAN.md. Phased: M40.1 single-source the engine (re-validate perft) → M40.2 net-forward + MCTS in the .pg + a TS ckpt parser → M40.3 Angular chess page (client-side play). Docs only. Co-Authored-By: Claude Opus 4.8 (1M context) * M40.1 — single-source the chess engine via Polyglot (perft 25/25 on generated core) Port the perft-verified chess engine (ChessBoard: ChessState+ChessRules, and ChessMoveEncoding) into one Polyglot source, chess_solver.pg, transpiled to C# (obj/, build-time) — the same source the browser client will run in M40.2/M40.3. - chess_solver.pg: internal Pg-prefixed core (PgChessState + PgChessMove). Board is List[64], promotion/castling are i32, perft returns i32 (fits every tested depth; widened to long in C#). Rays are bounded for+continue (no while); delta tables are List<(i32,i32)> iterated with tuple destructuring — every construct is one proven by fruitcake_solver.pg. - ChessState / ChessRules / ChessMoveEncoding are now thin C# facades over the generated core; PieceType, ChessMove, and all public signatures are unchanged, so ChessGame, ChessFen, SelfPlayCampaign, and both chess test files are untouched. - ChessGame's seam (LegalMoves/Apply/Result) delegates to the core's legalMoveIndices/applyIndex/result, single-sourcing the index logic training and the browser both use. Perft recurses entirely in-core (no per-node marshalling). Gate: perft 25/25 on the GENERATED engine (incl. startpos d5 = 4,865,609, Kiwipete d4 = 4,085,603, ~10s), encoding round-trip + terminal detection green, SelfPlay chess contract green, full fast suite 355/355 (no FruitCake/Snake/MountainCar regression from the shared re-transpile). PLAN M40.1 marked shipped. Co-Authored-By: Claude Opus 4.8 (1M context) * build(polyglot): bump MSBuild transpiler 0.3.1 → 0.5.3 + fix multi-.pg incremental bug 0.5.3 bundles win-x64/linux-x64/linux-arm64/osx-x64/osx-arm64 (adds macOS, so dev there no longer needs a manual $(PolyglotTool)). CI (ubuntu) already worked on 0.3.1 — it too bundled the linux binaries. The version bump does NOT fix the multi-.pg stale-prelude codegen bug (verified: an incremental touch of one .pg still produces CS0101/CS0260 under 0.5.3). Root cause is in the package .targets, not the CLI: PolyglotTranspile uses per-file Inputs/Outputs, so MSBuild's partial-incremental build invokes the CLI with only the changed .pg — but the CLI must see the whole set in one invocation to emit a single shared prelude + `partial class PolyglotProgram`. A single-file run re-emits an inline, non-partial prelude that clashes with the other solvers' generated files. Fix it at our project layer: a _PolyglotForceFullRetranspile target wipes the generated polyglot dir whenever any .pg is newer than the shared prelude, forcing PolyglotTranspile to regenerate the full set in one CLI call. Its single static Output (not a per-file transform) prevents MSBuild from partially batching it. Verified: the incremental touch that reliably broke now rebuilds clean in both Release and Debug; no-op rebuilds stay up-to-date; full suite 362/362, perft 25/25. Durable fix belongs upstream in the .targets (tracked for a MintPlayer.Polyglot PR). Co-Authored-By: Claude Opus 4.8 (1M context) * docs(polyglot): sharpen the multi-.pg incremental-bug handoff + verified .targets fix Update POLYGLOT_TOPLEVEL_RECORD_BUG.md (the existing Snake-M34 handoff for this bug) with what M40.1 pinned down: - Root cause is the MSBuild .targets, not the CLI — bumping 0.3.1 → 0.5.3 does NOT fix it (verified). PolyglotTranspile's per-file Inputs/Outputs let MSBuild's partial incremental build invoke the CLI with only the CHANGED .pg; the CLI then emits that unit standalone (inline prelude, non-partial PolyglotProgram) and it collides with the other solvers' generated files. The stale prelude is a symptom, not the cause. - Flags that the earlier "clean the --out dir" idea is insufficient and harmful on its own (would delete the other solvers' .cs under a subset invocation). - Gives the recommended upstream fix: a single stamp Output (un-batchable) + RemoveDir, so any staleness re-runs the FULL set in one CLI call. - Documents the consumer-side _PolyglotForceFullRetranspile mitigation already shipped in the Environments csproj, and points the csproj comment at this handoff. Co-Authored-By: Claude Opus 4.8 (1M context) * docs(polyglot): verified drop-in .targets fix for the multi-.pg incremental bug Direct writes to C:\Repos\MintPlayer.Polyglot are hook-blocked, so package the fix as a ready-to-apply handoff instead of a direct PR: - Add docs/prd/polyglot-pilot/MintPlayer.Polyglot.MSBuild.targets.FIXED — the upstream build/MintPlayer.Polyglot.MSBuild.targets with the minimal fix applied (stamp Outputs + RemoveDir + Touch + FileWrites stamp, plus the rationale comment). A session inside the Polyglot repo copies it over the source file, branches, and PRs. - Strengthen the handoff doc: the patch is VERIFIED, not proposed. Built a 2-.pg repro harness that imports the SOURCE .targets directly (no NuGet-cache mutation): the unpatched target fails the touch-one-.pg rebuild (a.cs standalone → CS0101/CS0260/ CS8863); the patched target makes all four states green (recover, incremental touch, no-op up-to-date skip, clean build), with incrementality preserved via the stamp. Co-Authored-By: Claude Opus 4.8 (1M context) * build(polyglot): 0.5.3 → 0.6.0, drop local incremental-bug workaround (fixed upstream) MintPlayer.Polyglot.MSBuild 0.6.0 (PR #26) ships the stamp-Outputs + RemoveDir fix for the multi-.pg incremental-transpile bug that was diagnosed and verified here during M40.1. With it upstream, the temporary local _PolyglotForceFullRetranspile target and the MintPlayer.Polyglot.MSBuild.targets.FIXED drop-in are no longer needed — both removed. Verified on 0.6.0 with NO local workaround: clean build green; the incremental single-.pg touch that used to break (CS0101/CS0260) now rebuilds clean; no-op rebuild up-to-date; full suite 362/362, chess perft 25/25. Docs reconciled: CLAUDE.md's "known build failure" note marked fixed in 0.6.0; PLAN M40.1 and the polyglot-pilot handoff marked resolved (kept for root-cause history). Co-Authored-By: Claude Opus 4.8 (1M context) * M40.2 — single-source the chess inference math (net forward + MCTS) + TS .ckpt parser Add the browser inference path to chess_solver.pg (transpiled to C# + committed TS): - writeObservation(): the 18-plane × 64 = 1152 observation, matching ChessGame.WriteObservation exactly (verified by a parity test over startpos / Kiwipete / an en-passant position). - PgPolicyValueNet.forward(): ReLU trunk (flat-array weights) → policy logits + linear value, mirroring Core/Nn/PolicyValueNet on one row (same shape as fruitcake's PgDuelingNet). - PgChessMcts: inference-only AlphaZero PUCT (masked-softmax priors, sqrt exploration, value negated per ply, NO Dirichlet) — the browser/serving search, single-sourced so C# and TS share it. - Math cleanup: use std.math Math.abs/Math.max (type-preserving on i32), keep isign (no std sign). C# side (Environments): - ChessNetParityTests (the M40.2 gate): Save a PolicyValueNet through its real .ckpt path, parse the bytes into the generated PgPolicyValueNet exactly as chess-net.ts will, and assert forward (logits + value) agrees within an f32 tolerance on the start position. Plus writeObservation parity and an MCTS runtime smoke (valid legal-move distribution summing to 1; chooseMove legal). 358/358 fast tests green. Browser side (RLDemo.Web ClientApp): - chess/chess_solver.ts — committed TS twin (regenerated from the .pg; compiles under the app's tsconfig). - chess/chess-net.ts — the ONE non-single-sourced piece: parses the "selfplay-pv" .ckpt (magic RLNC, trunk widths, per-layer W/b in Parameters() order; inputSize=1152/actions=4672 supplied) into the generated PgPolicyValueNet. Mirrors fruitcake-net.ts. Note: the generated TS is loosely typed in spots (emitter drops local List annotations; renders List as `T | null[]`) — compiles under the app's non-strict tsconfig and is runtime-correct, but the emitter gaps will be handed off to MintPlayer.Polyglot. M40.3 (the interactive board) is next. Co-Authored-By: Claude Opus 4.8 (1M context) * docs(M40.2): mark shipped; note Polyglot TS-emitter issue #27 (verified fix filed) Co-Authored-By: Claude Opus 4.8 (1M context) * build(polyglot): 0.6.0 → 0.7.0 — TS emitter fix (issue #27/PR #28); chess_solver.ts now strictly typed 0.7.0 ships the TypeScript-emitter fix diagnosed + verified during M40.2 (typed local List declarations; parenthesized List). Regenerated the committed chess_solver.ts: - `let d: [number, number][] = []` (was `let d = []`) - `let moves: PgChessMove[] = []` - `children: (PgMctsNode | null)[]` (was `PgMctsNode | null[]`) Verified strict-tsc-clean (strict + noImplicitAny) on chess_solver.ts + chess-net.ts — the M40.2 "loosely typed generated TS" caveat is resolved. Full build green, 365/365 tests (incl. deep chess perft) — no regression across the four solvers from the re-transpile. PLAN M40.2 note updated. Co-Authored-By: Claude Opus 4.8 (1M context) * M40.3 — interactive chess page (play the AI / watch AI-vs-AI), client-side The browser plays chess against the self-taught net with zero server inference — the engine, net forward, and PUCT search are the transpiled chess_solver.ts single source. - chess-director.ts: wraps PgChessState + PgChessMcts + the .ckpt-loaded net. Resolves a human (from,to) click to a legal action index (auto-queen on promotion), applies moves, and plays the AI's move for the side to move (Black in play mode, either side in watch). Falls back to a random legal move if no checkpoint is present. Yields to the UI before the (synchronous) search so the board paints + shows "thinking". - chess.ts: standalone Angular component — an 8x8 board (White at the bottom), click-to-move with legal-target dots + last-move/selected highlights, check/checkmate/stalemate/draw status, and two modes: "Play the AI" and "Watch AI vs AI" (a self-restarting loop). Signals drive reactivity; the watch loop is cancellable on mode change / navigation. - Route /chess + a Home tile. Verified: Angular builds the chess chunk cleanly (51.75 kB); /chess served; no /api/chess endpoint (404); director/net/solver strict-tsc clean; and the transpiled engine+net+MCTS plays a full legal game in Node over the real checkpoint (all moves legal). The shipped weights (wwwroot/models/chess.az.ckpt, LFS) land in a follow-up once training converges. Co-Authored-By: Claude Opus 4.8 (1M context) * M40.3 UX: square rows, orange move highlights, AI-vs-AI speed slider Feedback from trying the page: - Board rows no longer collapse on empty squares — added grid-template-rows: repeat(8, 1fr) so all 8 rows are equal height (square cells regardless of contents). - Move/selection/target highlights recoloured blue → orange (rgba(232,145,42,…)) for much clearer visibility against the board. - Watch mode gets a Speed slider (input type=range) controlling the pause between moves (left = slower … right = faster); the watch loop reads the signal each move so it responds live. Co-Authored-By: Claude Opus 4.8 (1M context) * M40.3 UX: captured-pieces tray (red ✕ over each taken piece) Show material that's been captured off the board, in the controls row (next to the speed slider in watch mode / New game in play mode). ChessDirector.capturedPieces() derives the missing set from the board vs the starting material; the component renders each as its glyph with a red ✕ overlay, updating after every move via the tick signal. Co-Authored-By: Claude Opus 4.8 (1M context) * docs(M40.4): plan chess difficulty — team investigation → PRD §9 + PLAN M40.4 Three read-only agents investigated the checkpoint/training machinery, the web surfaces, and difficulty-composition design. Synthesis (PRD §9, PLAN M40.4): - Difficulty spine = MCTS `sims` on ONE net + a browser-side visit-count TEMPERATURE for human-like variety at the low end. PgChessMcts.search already returns the visit distribution π (chooseMove = argmax), so temperature/sampling live caller-side in chess-director.ts — zero .pg changes, no RNG added to the Polyglot core. - Net-ladder (the owner's idea) → optional "watch it learn" novelty only, not the backbone: a small briefly-trained MLP has no reliably-rankable intermediate checkpoints and each is ~5–6 MB. - Delivery: a committed chess-difficulties.json manifest ({label, ckpt, sims, temp, cpuct}) — shippable now on the single net (tiers differ by sims/temp), upgradeable to multi-checkpoint later with zero code change. - Honest scope: top tier labelled "Full strength", never "Grandmaster". - Capture-ladder + net-vs-net Elo notes for the optional novelty/per-side tiers. Co-Authored-By: Claude Opus 4.8 (1M context) * M40.4a — auto-captured difficulty ladder in the Lab (net-vs-net arena, hands-off) Training is offline-only via the Lab; this makes it produce the web app's difficulty ladder with no manual steps. Enable with `--game chess --ladder` and walk away: as the net improves, new difficulty checkpoints + a manifest are written straight into the web models dir. SelfPlayCampaign (generic): - ArenaVsNet(challenger, champion, games): full net-vs-net games (eval MCTS argmax), diversified by a short randomized opening drawn from a dedicated _arenaRng seeded independently of the self-play/eval streams → training trajectory is unaffected (the arena is inference-only; a --ladder run just spends some eval-time, so a time-bounded run does marginally fewer games). - MaybePromoteDifficulty (after each Checkpoint): promote a new tier when the live net beats the last champion by EITHER a rise in winRate-vs-random ≥ --promote-margin (the discriminating signal while nets are weak — two weak nets draw each other head-to-head, so net-vs-net alone stays ~50% and can't rank them) OR head-to-head arena ≥ --arena-margin (once winRate-vs-random saturates). PromoteTier writes {env}.az.d{K}.ckpt + rewrites {env}-difficulties.json; champion := a frozen (save→reload) snapshot; ladder-resume adopts the highest tier on disk. - ChessLab flags: --ladder, --difficulty-dir (default the web models dir), --promote-margin (0.08), --arena-margin (0.60), --arena-games (20), --difficulty-sims (128), --opening-plies (6); plus --first-eval/--eval-every (configurable cadence, threaded through LabHost). Verified: a forced-promotion run produced an ordered 5-tier ladder (chess.az.d1..d5.ckpt + a 5-entry manifest); the gate correctly declines when neither signal is met; 40/40 chess + self-play tests pass. PRD §9.6 / PLAN M40.4a. Co-Authored-By: Claude Opus 4.8 (1M context) * M40.4b — chess difficulty selector + adopt @mintplayer/ng-bootstrap (dark theme preserved) Web difficulty picker (reads the Lab-written manifest) + first adoption of the @mintplayer/ng-bootstrap component library across the app. - chess-net.ts: loadDifficulties() parses /models/chess-difficulties.json ({label, ckpt, sims, temperature, cpuct, winRateVsRandom}), with a single-net fallback. - chess-director.ts: difficulties[] + current + setDifficulty(d) (loads the tier's ckpt, cached by url so sims-only switches don't refetch); aiStep now samples ∝ π^(1/T) when a tier's temperature > 0 (else argmax). Defaults to the strongest tier. - chess.ts: difficulty (shown when >1 tier), mode tabs as , New game as a themed bootstrap button (BsButtonTypeDirective), speed slider as . - Adoption: `npm i @mintplayer/ng-bootstrap` (auto-installs peers; aligned the app to a consistent Angular 22.0.6). angular.json loads @mintplayer/ng-bootstrap/bootstrap.scss. Dark-blue theme PRESERVED by re-pointing Bootstrap 5.3 CSS variables (--bs-primary #6ea8fe, --bs-body-bg #14171f, …) in styles.scss; the app's global button rule scoped to :not(.btn) so Bootstrap buttons keep their themed styling. (Bootstrap doesn't import card/forms, so Home's .card is untouched.) CLAUDE.md: hardened the rule that the .NET host serving Angular is already running and user-owned — never start/stop/kill/restart it; for new npm deps, ask the user to restart. Verified: Angular builds clean (chess chunk 64.33 kB, no TS/SCSS errors); /chess + the JSON manifest serve; difficulty tiers populate from the running --ladder training. Co-Authored-By: Claude Opus 4.8 (1M context) * web: Chess nav item + roll ng-bootstrap themed buttons across all game pages - app.html: add the Chess nav menu link (after FruitCake). - Convert each game page's ACTION buttons to Bootstrap themed buttons via BsButtonTypeDirective (` -public sealed class PolicyValueNet +public sealed class PolicyValueNet : IPolicyValueNet { private readonly Linear[] _trunk; private readonly Linear _policyHead, _valueHead; @@ -40,6 +40,8 @@ public PolicyValueNet(int inputSize, int[] hidden, int actions, Xoshiro256StarSt /// The shared-trunk hidden widths (drives growth schedules). public int[] Trunk => [.. _trunk.Select(l => l.Weight.Cols)]; + public string Describe() => $"trunk [{string.Join(",", Trunk)}]"; + /// Batched forward pass (autograd-recorded): raw policy logits [B, actions] + value [B, 1]. public (Tensor Logits, Tensor Value) Forward(Tensor observations) { diff --git a/src/MintPlayer.AI.ReinforcementLearning.Core/Numerics/TensorConv.cs b/src/MintPlayer.AI.ReinforcementLearning.Core/Numerics/TensorConv.cs new file mode 100644 index 0000000..c8579e5 --- /dev/null +++ b/src/MintPlayer.AI.ReinforcementLearning.Core/Numerics/TensorConv.cs @@ -0,0 +1,165 @@ +namespace MintPlayer.AI.ReinforcementLearning.Core.Numerics; + +// 2-D convolution as an autograd op (PLAN M42.1). Implemented the standard im2col way: gather each output +// position's receptive field into a row of a [M, inC·kH·kW] matrix, then a single GEMM against the weight +// matrix produces all output channels at once. The heavy multiply therefore runs through Backend.Current +// (GPU-routable), and the reverse pass is just the two transposed GEMMs the backend already provides plus a +// col2im scatter — so a spatial conv net needs NO new backend/kernel, only this data-movement wrapper. +// +// Layout convention: an NCHW image is carried as a rank-2 tensor [N, C·H·W] (row = image, col = c*H*W + h*W + w), +// so the op never produces a rank-4 tensor and never trips the rank-2 assertions the other ops enforce. The +// weight is [inC·kH·kW, outC] and the bias is [outC]. +public sealed partial class Tensor +{ + /// + /// 2-D convolution over an NCHW input carried as [N, inC·inH·inW], producing [N, outC·outH·outW] where + /// outH = (inH + 2·pad − kH)/stride + 1 (outW likewise). is [inC·kH·kW, outC]; + /// is [outC]. Autograd-recorded: the backward pass yields dInput (col2im of dCols), + /// dWeight (colsᵀ·dOut) and dBias (Σ dOut). + /// + public Tensor Conv2D(Tensor weight, Tensor bias, + int inC, int inH, int inW, int outC, int kH, int kW, int stride, int pad) + { + CheckRank2(this); + int n = Rows; + int kk = inC * kH * kW; + if (Cols != inC * inH * inW) + throw new ArgumentException($"Conv2D input cols {Cols} != inC·inH·inW = {inC * inH * inW}."); + if (weight.Rows != kk || weight.Cols != outC) + throw new ArgumentException($"Conv2D weight must be [{kk},{outC}], got [{weight.Rows},{weight.Cols}]."); + if (bias.Length != outC) + throw new ArgumentException($"Conv2D bias length {bias.Length} != outC {outC}."); + if (stride < 1 || pad < 0) throw new ArgumentException("Conv2D needs stride ≥ 1 and pad ≥ 0."); + + int outH = (inH + 2 * pad - kH) / stride + 1; + int outW = (inW + 2 * pad - kW) / stride + 1; + if (outH <= 0 || outW <= 0) + throw new ArgumentException("Conv2D output size is non-positive; check kernel/stride/pad."); + int m = n * outH * outW; + + // im2col → [M, kk], then GEMM against weight [kk, outC] → [M, outC]. + var cols = new float[m * kk]; + Im2Col(Data, cols, n, inC, inH, inW, kH, kW, stride, pad, outH, outW); + var outMat = new float[m * outC]; + Backend.Current.Gemm(cols, weight.Data, outMat, m, kk, outC); + + // Permute [M=n·oH·oW, outC] → [N, outC·oH·oW] and add the per-channel bias. + var data = new float[n * outC * outH * outW]; + ScatterMOutCToNCHW(outMat, bias.Data, data, n, outC, outH, outW); + + return MakeResult(data, [n, outC * outH * outW], [this, weight, bias], result => () => + { + // dOut [N, outC·oH·oW] → dOutMat [M, outC]. + var dOutMat = new float[m * outC]; + GatherNCHWToMOutC(result.Grad!, dOutMat, n, outC, outH, outW); + + if (weight.NeedsGrad) + { + weight.EnsureGrad(); + Backend.Current.GemmTransposeA(cols, dOutMat, weight.Grad!, m, kk, outC); // dW[kk,outC] += colsᵀ·dOut + } + if (bias.NeedsGrad) + { + bias.EnsureGrad(); + var db = bias.Grad!; + for (int i = 0; i < m; i++) + for (int oc = 0; oc < outC; oc++) db[oc] += dOutMat[i * outC + oc]; + } + if (NeedsGrad) + { + EnsureGrad(); + var dCols = new float[m * kk]; + Backend.Current.GemmTransposeB(dOutMat, weight.Data, dCols, m, kk, outC); // dCols[M,kk] += dOut·Wᵀ + Col2Im(dCols, Grad!, n, inC, inH, inW, kH, kW, stride, pad, outH, outW); // scatter-add into dInput + } + }); + } + + // Gather each output position's receptive field into a matrix row: cols[m, (c·kH+kh)·kW+kw] = x[n,c,ih,iw] + // (0 where the padded window falls outside the image). m = (n·oH+oh)·oW+ow. + private static void Im2Col(ReadOnlySpan x, Span cols, + int n, int inC, int inH, int inW, int kH, int kW, int stride, int pad, int outH, int outW) + { + int kk = inC * kH * kW; + for (int img = 0; img < n; img++) + for (int oh = 0; oh < outH; oh++) + for (int ow = 0; ow < outW; ow++) + { + int row = ((img * outH) + oh) * outW + ow; + int colBase = row * kk; + for (int c = 0; c < inC; c++) + for (int kh = 0; kh < kH; kh++) + { + int ih = oh * stride - pad + kh; + for (int kw = 0; kw < kW; kw++) + { + int iw = ow * stride - pad + kw; + int kcol = (c * kH + kh) * kW + kw; + cols[colBase + kcol] = (ih >= 0 && ih < inH && iw >= 0 && iw < inW) + ? x[((img * inC + c) * inH + ih) * inW + iw] + : 0f; + } + } + } + } + + // The transpose of Im2Col: scatter-ADD each matrix row back to the input positions it was gathered from + // (overlapping windows accumulate). Adds into dInput (does not overwrite). + private static void Col2Im(ReadOnlySpan cols, Span dInput, + int n, int inC, int inH, int inW, int kH, int kW, int stride, int pad, int outH, int outW) + { + int kk = inC * kH * kW; + for (int img = 0; img < n; img++) + for (int oh = 0; oh < outH; oh++) + for (int ow = 0; ow < outW; ow++) + { + int row = ((img * outH) + oh) * outW + ow; + int colBase = row * kk; + for (int c = 0; c < inC; c++) + for (int kh = 0; kh < kH; kh++) + { + int ih = oh * stride - pad + kh; + if (ih < 0 || ih >= inH) continue; + for (int kw = 0; kw < kW; kw++) + { + int iw = ow * stride - pad + kw; + if (iw < 0 || iw >= inW) continue; + int kcol = (c * kH + kh) * kW + kw; + dInput[((img * inC + c) * inH + ih) * inW + iw] += cols[colBase + kcol]; + } + } + } + } + + // [M=n·oH·oW, outC] → [N, outC·oH·oW] (+ per-channel bias). out[n,oc,oh,ow] = mat[(n·oH+oh)·oW+ow, oc] + bias[oc]. + private static void ScatterMOutCToNCHW(ReadOnlySpan mat, ReadOnlySpan bias, Span outp, + int n, int outC, int outH, int outW) + { + int hw = outH * outW; + for (int img = 0; img < n; img++) + for (int oh = 0; oh < outH; oh++) + for (int ow = 0; ow < outW; ow++) + { + int row = ((img * outH) + oh) * outW + ow; + int sp = oh * outW + ow; + for (int oc = 0; oc < outC; oc++) + outp[(img * outC + oc) * hw + sp] = mat[row * outC + oc] + bias[oc]; + } + } + + // The inverse index map: [N, outC·oH·oW] → [M, outC]. mat[(n·oH+oh)·oW+ow, oc] = grad[n,oc,oh,ow]. + private static void GatherNCHWToMOutC(ReadOnlySpan nchw, Span mat, + int n, int outC, int outH, int outW) + { + int hw = outH * outW; + for (int img = 0; img < n; img++) + for (int oh = 0; oh < outH; oh++) + for (int ow = 0; ow < outW; ow++) + { + int row = ((img * outH) + oh) * outW + ow; + int sp = oh * outW + ow; + for (int oc = 0; oc < outC; oc++) + mat[row * outC + oc] = nchw[(img * outC + oc) * hw + sp]; + } + } +} diff --git a/src/MintPlayer.AI.ReinforcementLearning.Core/Planning/IMaterialScore.cs b/src/MintPlayer.AI.ReinforcementLearning.Core/Planning/IMaterialScore.cs new file mode 100644 index 0000000..ca02640 --- /dev/null +++ b/src/MintPlayer.AI.ReinforcementLearning.Core/Planning/IMaterialScore.cs @@ -0,0 +1,17 @@ +namespace MintPlayer.AI.ReinforcementLearning.Core.Planning; + +/// +/// An OPTIONAL capability an may also implement to expose a dense, per-position +/// material/score signal (from the side-to-move's perspective). Self-play uses it to shape the value target so +/// a still-weak net gets a gradient on every capture — not just the sparse win/loss/draw outcome, which for a net +/// that can't yet force mate is ~always a draw (→ 0) and teaches nothing. It also gives the difficulty ladder a +/// non-saturating strength metric (two "drawing" nets are separable by average material). Games with no material +/// notion simply don't implement it, and self-play falls back to the pure outcome. +/// +/// The game state type. +public interface IMaterialScore +{ + /// The side-to-move's material advantage (own material − opponent's), in the game's natural units + /// (chess: pawns). Positive = the side to move is ahead. + float MaterialAdvantage(TState state); +} diff --git a/src/MintPlayer.AI.ReinforcementLearning.Core/Planning/IZeroSumGame.cs b/src/MintPlayer.AI.ReinforcementLearning.Core/Planning/IZeroSumGame.cs new file mode 100644 index 0000000..c6aabc0 --- /dev/null +++ b/src/MintPlayer.AI.ReinforcementLearning.Core/Planning/IZeroSumGame.cs @@ -0,0 +1,56 @@ +namespace MintPlayer.AI.ReinforcementLearning.Core.Planning; + +/// A terminal game result, stated RELATIVE TO THE SIDE TO MOVE in the queried state. +public enum GameResult +{ + /// Not a terminal position. + Ongoing, + /// The side to move has already won (rare — most games are won on the mover's own move). + Win, + /// The side to move has lost (e.g. it is checkmated / the opponent just completed a line). + Loss, + /// A draw (stalemate, repetition, full board, …). + Draw, +} + +/// +/// A perfect-information, two-player, zero-sum, deterministic game — the shared shape that MCTS +/// () and a self-play trainer both consume. It is deliberately distinct from +/// IEnvironment (which has a single-agent reward/episode loop) and from +/// (whose single-goal IsGoal and always-valid ActionCount can't express side-to-move, +/// win/loss/draw, or per-state legal moves). +/// +/// Conventions the implementer MUST honour: returns a FRESH successor and leaves the input +/// state untouched (search reuses one state across many candidate moves), and applying a move FLIPS the side +/// to move. and are always from the perspective of the side to +/// move in the given state, so the network sees one canonical "me vs. them" view regardless of whose turn it is. +/// +/// +/// The immutable-from-the-caller's-view game state (a position). +public interface IZeroSumGame +{ + /// The fixed action-index space — equals the policy head width of the net that plays this game. + /// A given state's legal subset is ; every legal move is an index in [0, PolicySize). + int PolicySize { get; } + + /// The length of the observation writes (the net's input width). + int ObservationSize { get; } + + /// The start position. is for games with a randomized setup (chess/Connect-4 ignore it). + TState Root(ulong? seed = null); + + /// The legal action indices in (a subset of [0, PolicySize)). Never empty for a + /// non-terminal state. + IReadOnlyList LegalMoves(TState state); + + /// The successor after playing — a fresh state, with the side to move flipped. + /// is not mutated. must be one of . + TState Apply(TState state, int move); + + /// The terminal result for the side to move in , or . + GameResult Result(TState state); + + /// Writes the side-to-move-relative observation of into + /// (length ). + void WriteObservation(TState state, Span destination); +} diff --git a/src/MintPlayer.AI.ReinforcementLearning.Core/Planning/Mcts.cs b/src/MintPlayer.AI.ReinforcementLearning.Core/Planning/Mcts.cs new file mode 100644 index 0000000..f85f39f --- /dev/null +++ b/src/MintPlayer.AI.ReinforcementLearning.Core/Planning/Mcts.cs @@ -0,0 +1,178 @@ +using MintPlayer.AI.ReinforcementLearning.Core.Random; + +namespace MintPlayer.AI.ReinforcementLearning.Core.Planning; + +/// +/// AlphaZero-style PUCT Monte-Carlo Tree Search over an , guided by a neural +/// leaf evaluator (policy priors + a value in [-1,1]). It composes over the game seam + an +/// delegate exactly as composes over + a +/// cost-to-go delegate, so Core carries no game or NN dependency. +/// +/// Each simulation walks the tree by maximizing Q + U (mean action value + a prior-weighted exploration +/// bonus), expands one new leaf, evaluates it with the net, and backs the value up the path negating it every +/// ply (zero-sum: a position good for the child's mover is bad for the parent's). At the root, Dirichlet noise is +/// mixed into the priors so self-play keeps exploring. returns the root's visit-count +/// distribution — the policy target π for training, from which the caller samples (early game) or takes the argmax. +/// +/// +public static class Mcts +{ + /// Tree simulations per move (more = stronger, slower). + /// PUCT exploration constant. + /// Concentration of the root-noise Dirichlet (chess ≈ 0.3; smaller = spikier). + /// Fraction of the root prior replaced by Dirichlet noise (0 disables). + public sealed record Config(int Simulations = 200, float Cpuct = 1.25f, float DirichletAlpha = 0.3f, float RootNoiseFrac = 0.25f); + + /// Evaluates a leaf: a probability distribution over PolicySize (illegal moves ≈ 0; the search + /// renormalizes over the legal set) and the position value in [-1,1] from the side-to-move's perspective. Backed + /// by a PolicyValueNet.Forward under NoGrad, softmaxed over legal moves + tanh value. + public delegate (float[] Priors, float Value) Evaluate(TState state); + + private sealed class Node + { + public required int[] Moves; // legal action indices in this state + public float[] P = []; // prior per move (aligned to Moves) + public int[] N = []; // visit count per move + public float[] W = []; // summed value per move, from THIS node's mover perspective + public Node?[] Children = []; + public bool Expanded; + public bool Terminal; + public float TerminalValue; // set when Terminal: +1 win / -1 loss / 0 draw, mover-relative + } + + /// Runs .Simulations from and returns the root + /// visit-count distribution over game.PolicySize (sums to 1 over the legal moves; 0 elsewhere). Falls back + /// to the raw priors only if no simulation recorded a visit (Simulations == 0). + public static float[] Search(IZeroSumGame game, TState rootState, + Evaluate evaluate, Config config, Xoshiro256StarStar rng) + { + var root = NewNode(game, rootState); + if (!root.Terminal) + { + ExpandLeaf(root, evaluate, rootState); + AddRootNoise(root, config, rng); + for (int sim = 0; sim < config.Simulations; sim++) + Simulate(root, game, rootState, evaluate, config); + } + + var pi = new float[game.PolicySize]; + int total = 0; + for (int i = 0; i < root.Moves.Length; i++) total += root.N[i]; + if (total == 0) + { + for (int i = 0; i < root.Moves.Length; i++) pi[root.Moves[i]] = root.P[i]; + return pi; + } + for (int i = 0; i < root.Moves.Length; i++) pi[root.Moves[i]] = root.N[i] / (float)total; + return pi; + } + + // Returns the value of 'state' from the perspective of its side to move. + private static float Simulate(Node node, IZeroSumGame game, TState state, + Evaluate evaluate, Config config) + { + if (node.Terminal) return node.TerminalValue; + if (!node.Expanded) return ExpandLeaf(node, evaluate, state); + + int edge = SelectChild(node, config); + var childState = game.Apply(state, node.Moves[edge]); + node.Children[edge] ??= NewNode(game, childState); + float value = -Simulate(node.Children[edge]!, game, childState, evaluate, config); // flip: child mover's value + node.N[edge]++; + node.W[edge] += value; + return value; + } + + private static int SelectChild(Node node, Config config) + { + int sumN = 0; + for (int i = 0; i < node.N.Length; i++) sumN += node.N[i]; + float sqrtSum = MathF.Sqrt(sumN); + + int best = 0; + float bestScore = float.NegativeInfinity; + for (int i = 0; i < node.Moves.Length; i++) + { + float q = node.N[i] > 0 ? node.W[i] / node.N[i] : 0f; + float u = config.Cpuct * node.P[i] * sqrtSum / (1 + node.N[i]); + float score = q + u; + if (score > bestScore) { bestScore = score; best = i; } + } + return best; + } + + private static Node NewNode(IZeroSumGame game, TState state) + { + var result = game.Result(state); + if (result != GameResult.Ongoing) + return new Node { Moves = [], Terminal = true, TerminalValue = result switch { GameResult.Win => 1f, GameResult.Loss => -1f, _ => 0f } }; + return new Node { Moves = [.. game.LegalMoves(state)] }; + } + + // Evaluate the leaf, seed per-move priors (masked to legal + renormalized), and return the net's value estimate. + private static float ExpandLeaf(Node node, Evaluate evaluate, TState state) + { + var (priors, value) = evaluate(state); + int k = node.Moves.Length; + node.P = new float[k]; + node.N = new int[k]; + node.W = new float[k]; + node.Children = new Node?[k]; + + float sum = 0f; + for (int i = 0; i < k; i++) { node.P[i] = MathF.Max(priors[node.Moves[i]], 0f); sum += node.P[i]; } + if (sum > 0f) { for (int i = 0; i < k; i++) node.P[i] /= sum; } + else { for (int i = 0; i < k; i++) node.P[i] = 1f / k; } // net gave no mass to legal moves → uniform + + node.Expanded = true; + return value; + } + + private static void AddRootNoise(Node root, Config config, Xoshiro256StarStar rng) + { + if (config.RootNoiseFrac <= 0f || root.Moves.Length == 0) return; + var noise = SampleDirichlet(root.Moves.Length, config.DirichletAlpha, rng); + float frac = config.RootNoiseFrac; + for (int i = 0; i < root.P.Length; i++) root.P[i] = (1f - frac) * root.P[i] + frac * noise[i]; + } + + // ── Dirichlet sampling (Gamma via Marsaglia–Tsang; normals via Box–Muller), on the game's own RNG stream ── + private static float[] SampleDirichlet(int k, double alpha, Xoshiro256StarStar rng) + { + var g = new double[k]; + double sum = 0; + for (int i = 0; i < k; i++) { g[i] = SampleGamma(alpha, rng); sum += g[i]; } + var d = new float[k]; + if (sum <= 0) { for (int i = 0; i < k; i++) d[i] = 1f / k; return d; } + for (int i = 0; i < k; i++) d[i] = (float)(g[i] / sum); + return d; + } + + private static double SampleGamma(double alpha, Xoshiro256StarStar rng) + { + if (alpha < 1.0) + { + double u = rng.NextDouble(); + return SampleGamma(alpha + 1.0, rng) * Math.Pow(u <= 0 ? double.Epsilon : u, 1.0 / alpha); + } + double d = alpha - 1.0 / 3.0; + double c = 1.0 / Math.Sqrt(9.0 * d); + while (true) + { + double x = NextNormal(rng); + double v = 1 + c * x; + if (v <= 0) continue; + v = v * v * v; + double u2 = rng.NextDouble(); + if (u2 < 1 - 0.0331 * x * x * x * x) return d * v; + if (Math.Log(u2 <= 0 ? double.Epsilon : u2) < 0.5 * x * x + d * (1 - v + Math.Log(v))) return d * v; + } + } + + private static double NextNormal(Xoshiro256StarStar rng) + { + double u1 = rng.NextDouble(); + double u2 = rng.NextDouble(); + return Math.Sqrt(-2.0 * Math.Log(u1 <= 0 ? double.Epsilon : u1)) * Math.Cos(2.0 * Math.PI * u2); + } +} diff --git a/src/MintPlayer.AI.ReinforcementLearning.Core/Training/DeterministicParallel.cs b/src/MintPlayer.AI.ReinforcementLearning.Core/Training/DeterministicParallel.cs new file mode 100644 index 0000000..3cdd34d --- /dev/null +++ b/src/MintPlayer.AI.ReinforcementLearning.Core/Training/DeterministicParallel.cs @@ -0,0 +1,81 @@ +using MintPlayer.AI.ReinforcementLearning.Core.Random; + +namespace MintPlayer.AI.ReinforcementLearning.Core.Training; + +/// +/// Generates a batch of independent items — one per index in [0, count) — with a determinism guarantee: +/// the result array is bitwise-identical whether produced in parallel or sequentially, and at any degree of +/// parallelism. This is the reusable form of the data-generation loop the cube campaigns hand-roll and that +/// self-play needs: episodes/games/labeled-samples are embarrassingly parallel, but a seeded run must stay +/// reproducible regardless of how many cores happen to run it. +/// +/// It holds the same way Core's other parallel primitives do (: per-unit RNG; +/// the backend GEMM: disjoint output rows) — each item's randomness is a pure function of its global index +/// (never execution order), and each item writes only its own result slot, so there is no shared mutable state and +/// no reduction. The caller supplies a + a stream index so generation stays on its own +/// RNG stream, disjoint from the trainer's other streams (init, eval, …), exactly as the rest of the codebase fans +/// seeds out. +/// +/// +public static class DeterministicParallel +{ + /// + /// Runs for each local index in [0, count), each with its OWN RNG derived + /// from (, , + localIndex), + /// and returns the results in ascending index order. Because a given global index always derives the same RNG + /// and writes the same slot, the output does not depend on or on the worker count — + /// callers get free multi-core scaling with zero effect on a seeded run's outcome. + /// + /// Number of items to generate (0 ⇒ empty array). + /// The run's master seed fan-out; the per-item base seed is seeds.Derive(stream). + /// RNG stream index for this generation phase (see ), keeping it + /// disjoint from other streams. + /// Global index of the first item — advance it across calls (e.g. total games so far) + /// so successive batches draw non-overlapping, non-repeating RNGs. + /// Produces item i from its derived RNG. MUST be pure w.r.t. that RNG and any + /// read-only captured state (a shared read-only model snapshot is fine); it must not touch shared mutable state, + /// or the determinism guarantee is lost. + /// Spread work across cores (bitwise-identical to the sequential path). + /// Optional cap on the degree of parallelism (defaults to the runtime's choice). + public static TItem[] Generate( + int count, SeedSequence seeds, int stream, long baseIndex, + Func makeItem, bool parallel, int? maxDop = null) + { + ArgumentNullException.ThrowIfNull(seeds); + return Generate(count, seeds.Derive(stream), baseIndex, makeItem, parallel, maxDop); + } + + /// + /// As above, but for callers that manage their own base seed rather than fanning out a + /// — e.g. a data-gen loop keyed on a per-round seed. Item i's RNG is + /// Xoshiro( + ( + i)·φ), so passing + /// (baseSeed: roundSeed, baseIndex: 1) reproduces the common roundSeed + φ·(worker+1) per-worker + /// seeding exactly. + /// + public static TItem[] Generate( + int count, ulong baseSeed, long baseIndex, + Func makeItem, bool parallel, int? maxDop = null) + { + ArgumentOutOfRangeException.ThrowIfNegative(count); + ArgumentNullException.ThrowIfNull(makeItem); + + var results = new TItem[count]; + if (parallel && count > 1) + { + var options = new ParallelOptions(); + if (maxDop is int dop) options.MaxDegreeOfParallelism = dop; + Parallel.For(0, count, options, i => results[i] = makeItem(i, DeriveRng(baseSeed, baseIndex + i))); + } + else + { + for (int i = 0; i < count; i++) + results[i] = makeItem(i, DeriveRng(baseSeed, baseIndex + i)); + } + return results; + } + + /// Per-item RNG: the golden-ratio index stride the codebase uses everywhere (VectorEnv.Reset), + /// whose xoshiro seeding runs each seed through SplitMix64 — so adjacent indices give decorrelated streams. + private static Xoshiro256StarStar DeriveRng(ulong baseSeed, long globalIndex) + => new(unchecked(baseSeed + (ulong)globalIndex * 0x9E3779B97F4A7C15UL)); +} diff --git a/src/MintPlayer.AI.ReinforcementLearning.Environments/Chess/ChessBoard.cs b/src/MintPlayer.AI.ReinforcementLearning.Environments/Chess/ChessBoard.cs new file mode 100644 index 0000000..3f899fd --- /dev/null +++ b/src/MintPlayer.AI.ReinforcementLearning.Environments/Chess/ChessBoard.cs @@ -0,0 +1,96 @@ +namespace MintPlayer.AI.ReinforcementLearning.Environments.Chess; + +/// Piece type (colour-agnostic). = empty square. +public enum PieceType : byte { None = 0, Pawn = 1, Knight = 2, Bishop = 3, Rook = 4, Queen = 5, King = 6 } + +/// A move: from/to squares (0..63, = rank*8 + file, a1 = 0), plus the promotion piece for a pawn reaching +/// the last rank ( otherwise). Castling, en passant, and double-push are inferred from +/// the board when the move is made, so they need no extra flag here. +public readonly record struct ChessMove(byte From, byte To, PieceType Promotion = PieceType.None); + +/// +/// A chess position — the public, ergonomic view (mailbox , side to move, castling rights, +/// en-passant target, halfmove clock). This is a thin facade over the single-source engine transpiled from +/// polyglot/chess_solver.pg (PgChessState) — the same source the browser client runs. The board is +/// immutable: every move produces a fresh state. Squares are 0..63 (file = sq % 8, rank = sq / 8); White moves +/// toward higher ranks. Cells: 0 = empty, +1..+6 = White P,N,B,R,Q,K, −1..−6 = Black (sign = colour). +/// Threefold-repetition is intentionally not modelled (see PLAN M39); the 50-move rule and the self-play +/// ply cap bound looping games. +/// +public sealed class ChessState +{ + internal readonly PgChessState Core; + private sbyte[]? _squares; + + public const byte CastleWK = 1, CastleWQ = 2, CastleBK = 4, CastleBQ = 8; + + internal ChessState(PgChessState core) => Core = core; + + public ChessState(sbyte[] squares, bool whiteToMove, byte castling, sbyte enPassant, byte halfmoveClock) + { + var cells = new List(64); + for (int i = 0; i < 64; i++) cells.Add(squares[i]); + Core = new PgChessState(cells, whiteToMove, castling, enPassant, halfmoveClock); + } + + /// The 64-cell mailbox (a cached snapshot of the core board — read-only). + public sbyte[] Squares + { + get + { + if (_squares is null) + { + _squares = new sbyte[64]; + for (int i = 0; i < 64; i++) _squares[i] = (sbyte)Core.squares[i]; + } + return _squares; + } + } + + public bool WhiteToMove => Core.whiteToMove; + public byte Castling => (byte)Core.castling; // bit 0 = White O-O, 1 = White O-O-O, 2 = Black O-O, 3 = Black O-O-O + public sbyte EnPassant => (sbyte)Core.enPassant; + public byte HalfmoveClock => (byte)Core.halfmoveClock; + + public static ChessState StartPosition() => ChessFen.Parse(ChessFen.StartFen); + + // Move ↔ generated-move mapping (promotion piece codes coincide with PieceType's byte values, so the cast is + // a no-op renaming). Shared by ChessRules and ChessMoveEncoding — the two facades over the single source. + internal static PgChessMove ToPg(ChessMove m) => new(m.From, m.To, (int)m.Promotion); + internal static ChessMove FromPg(PgChessMove m) => new((byte)m.from, (byte)m.to, (PieceType)m.promotion); +} + +/// +/// The chess rules — a public facade over the single-source engine (chess_solver.pg → PgChessState): +/// legal move generation (castling, en passant, promotion, pins/checks), make-move, attack detection, and terminal +/// detection (checkmate, stalemate, 50-move, insufficient material). Correctness is pinned by perft, which +/// recurses entirely inside the generated core (no per-node marshalling). +/// +public static class ChessRules +{ + public static bool IsSquareAttacked(ChessState s, int square, bool byWhite) => s.Core.isSquareAttacked(square, byWhite); + + public static int KingSquare(ChessState s, bool white) => s.Core.kingSquare(white); + + public static bool InCheck(ChessState s, bool white) => s.Core.inCheck(white); + + /// All fully-legal moves for the side to move (own king not left in check). + public static List LegalMoves(ChessState s) + { + var core = s.Core.legalMoves(); + var moves = new List(core.Count); + foreach (var m in core) moves.Add(ChessState.FromPg(m)); + return moves; + } + + /// The position after (a fresh state; is unchanged). + public static ChessState MakeMove(ChessState s, ChessMove move) => new(s.Core.makeMove(ChessState.ToPg(move))); + + public static bool IsFiftyMove(ChessState s) => s.Core.isFiftyMove(); + + public static bool IsInsufficientMaterial(ChessState s) => s.Core.isInsufficientMaterial(); + + /// Perft: the number of leaf nodes at — the movegen correctness oracle. The + /// count is produced entirely inside the generated engine (i32 there; widened to long here). + public static long Perft(ChessState s, int depth) => s.Core.perft(depth); +} diff --git a/src/MintPlayer.AI.ReinforcementLearning.Environments/Chess/ChessFen.cs b/src/MintPlayer.AI.ReinforcementLearning.Environments/Chess/ChessFen.cs new file mode 100644 index 0000000..23599ba --- /dev/null +++ b/src/MintPlayer.AI.ReinforcementLearning.Environments/Chess/ChessFen.cs @@ -0,0 +1,101 @@ +using System.Text; + +namespace MintPlayer.AI.ReinforcementLearning.Environments.Chess; + +/// +/// Forsyth–Edwards Notation (FEN) ↔ . Used to set up standard test positions (perft) and to +/// render a position for debugging. Only the fields the engine tracks are parsed (board, side, castling, en passant, +/// halfmove clock); the fullmove number is ignored on read and written as 1. +/// +public static class ChessFen +{ + public const string StartFen = "rnbqkbnr/pppppppp/8/8/8/8/PPPPPPPP/RNBQKBNR w KQkq - 0 1"; + + public static ChessState Parse(string fen) + { + string[] parts = fen.Split(' '); + var squares = new sbyte[64]; + + string[] ranks = parts[0].Split('/'); // rank 8 first + for (int i = 0; i < 8; i++) + { + int rank = 7 - i; // ranks[0] is the 8th rank (rank index 7) + int file = 0; + foreach (char c in ranks[i]) + { + if (char.IsDigit(c)) { file += c - '0'; continue; } + squares[rank * 8 + file] = PieceOf(c); + file++; + } + } + + bool whiteToMove = parts[1] == "w"; + byte castling = 0; + if (parts[2] != "-") + foreach (char c in parts[2]) + castling |= c switch + { + 'K' => ChessState.CastleWK, + 'Q' => ChessState.CastleWQ, + 'k' => ChessState.CastleBK, + 'q' => ChessState.CastleBQ, + _ => (byte)0, + }; + + sbyte enPassant = parts[3] == "-" ? (sbyte)-1 : (sbyte)SquareOf(parts[3]); + byte halfmove = parts.Length > 4 && byte.TryParse(parts[4], out byte hm) ? hm : (byte)0; + return new ChessState(squares, whiteToMove, castling, enPassant, halfmove); + } + + public static string ToFen(ChessState s) + { + var sb = new StringBuilder(); + for (int rank = 7; rank >= 0; rank--) + { + int empty = 0; + for (int file = 0; file < 8; file++) + { + sbyte piece = s.Squares[rank * 8 + file]; + if (piece == 0) { empty++; continue; } + if (empty > 0) { sb.Append(empty); empty = 0; } + sb.Append(CharOf(piece)); + } + if (empty > 0) sb.Append(empty); + if (rank > 0) sb.Append('/'); + } + sb.Append(s.WhiteToMove ? " w " : " b "); + string rights = string.Concat( + (s.Castling & ChessState.CastleWK) != 0 ? "K" : "", + (s.Castling & ChessState.CastleWQ) != 0 ? "Q" : "", + (s.Castling & ChessState.CastleBK) != 0 ? "k" : "", + (s.Castling & ChessState.CastleBQ) != 0 ? "q" : ""); + sb.Append(rights.Length == 0 ? "-" : rights); + sb.Append(' ').Append(s.EnPassant < 0 ? "-" : SquareName(s.EnPassant)); + sb.Append(' ').Append(s.HalfmoveClock).Append(" 1"); + return sb.ToString(); + } + + private static sbyte PieceOf(char c) + { + sbyte type = char.ToLowerInvariant(c) switch + { + 'p' => 1, 'n' => 2, 'b' => 3, 'r' => 4, 'q' => 5, 'k' => 6, + _ => throw new FormatException($"Bad FEN piece '{c}'."), + }; + return char.IsUpper(c) ? type : (sbyte)-type; + } + + private static char CharOf(sbyte piece) + { + char c = (PieceType)Math.Abs(piece) switch + { + PieceType.Pawn => 'p', PieceType.Knight => 'n', PieceType.Bishop => 'b', + PieceType.Rook => 'r', PieceType.Queen => 'q', PieceType.King => 'k', + _ => '?', + }; + return piece > 0 ? char.ToUpperInvariant(c) : c; + } + + private static int SquareOf(string s) => (s[1] - '1') * 8 + (s[0] - 'a'); + private static string SquareName(int sq) => $"{(char)('a' + (sq & 7))}{(char)('1' + (sq >> 3))}"; +} diff --git a/src/MintPlayer.AI.ReinforcementLearning.Environments/Chess/ChessGame.cs b/src/MintPlayer.AI.ReinforcementLearning.Environments/Chess/ChessGame.cs new file mode 100644 index 0000000..2f9da91 --- /dev/null +++ b/src/MintPlayer.AI.ReinforcementLearning.Environments/Chess/ChessGame.cs @@ -0,0 +1,74 @@ +using MintPlayer.AI.ReinforcementLearning.Core.Planning; + +namespace MintPlayer.AI.ReinforcementLearning.Environments.Chess; + +/// +/// Chess as an — the headline consumer of the self-play stack (PLAN M39.2). It +/// adapts the perft-verified engine to the seam: the action space is the 4672-index +/// , and the observation is an 18-plane board stack (12 piece planes + side-to-move + +/// 4 castling-rights + en passant), flattened. MCTS, the two-headed net, and the self-play campaign are all reused +/// unchanged from M39.1. (Board geometry is absolute — the net learns both colours via the side-to-move plane; +/// perspective canonicalization is a later efficiency lever, PLAN M39.3.) +/// +public sealed class ChessGame : IZeroSumGame, IMaterialScore +{ + private const int Planes = 18; + + // Standard relative piece values, indexed by |piece| (none, P, N, B, R, Q, K). The king isn't "captured" + // (reaching it is checkmate — the ±1 game outcome), so it scores 0 and cancels between the two sides. + private static readonly int[] PieceValues = [0, 1, 3, 3, 5, 9, 0]; + + public int PolicySize => ChessMoveEncoding.Size; // 4672 + public int ObservationSize => Planes * 64; // 1152 + + /// The side-to-move's material advantage in pawns (its pieces − the opponent's). Dense reward signal + /// for self-play shaping + the difficulty ladder's strength metric (). + public float MaterialAdvantage(ChessState state) + { + int white = 0, black = 0; + foreach (sbyte p in state.Squares) + { + int v = PieceValues[Math.Abs(p)]; + if (p > 0) white += v; else if (p < 0) black += v; + } + int diff = white - black; + return state.WhiteToMove ? diff : -diff; + } + + public ChessState Root(ulong? seed = null) => ChessState.StartPosition(); + + // The seam methods delegate to the single-source core so training and the browser client share one + // implementation of "legal indices / apply an index / terminal result". + public IReadOnlyList LegalMoves(ChessState state) => state.Core.legalMoveIndices(); + + public ChessState Apply(ChessState state, int move) => new(state.Core.applyIndex(move)); + + public GameResult Result(ChessState state) => state.Core.result() switch + { + 1 => GameResult.Loss, // side to move is checkmated + 2 => GameResult.Draw, // stalemate / 50-move / insufficient material + _ => GameResult.Ongoing, + }; + + public void WriteObservation(ChessState state, Span destination) + { + destination.Clear(); + // Piece planes 0..5 = White P,N,B,R,Q,K ; 6..11 = Black. Plane p, square sq → index p*64 + sq. + for (int sq = 0; sq < 64; sq++) + { + sbyte piece = state.Squares[sq]; + if (piece == 0) continue; + int type = Math.Abs(piece) - 1; // 0..5 + int plane = piece > 0 ? type : 6 + type; // white vs black block + destination[plane * 64 + sq] = 1f; + } + if (state.WhiteToMove) Fill(destination, 12); + if ((state.Castling & ChessState.CastleWK) != 0) Fill(destination, 13); + if ((state.Castling & ChessState.CastleWQ) != 0) Fill(destination, 14); + if ((state.Castling & ChessState.CastleBK) != 0) Fill(destination, 15); + if ((state.Castling & ChessState.CastleBQ) != 0) Fill(destination, 16); + if (state.EnPassant >= 0) destination[17 * 64 + state.EnPassant] = 1f; + } + + private static void Fill(Span dest, int plane) => dest.Slice(plane * 64, 64).Fill(1f); +} diff --git a/src/MintPlayer.AI.ReinforcementLearning.Environments/Chess/ChessMoveEncoding.cs b/src/MintPlayer.AI.ReinforcementLearning.Environments/Chess/ChessMoveEncoding.cs new file mode 100644 index 0000000..5de0116 --- /dev/null +++ b/src/MintPlayer.AI.ReinforcementLearning.Environments/Chess/ChessMoveEncoding.cs @@ -0,0 +1,25 @@ +namespace MintPlayer.AI.ReinforcementLearning.Environments.Chess; + +/// +/// The AlphaZero move encoding: a fixed 64 × 73 = 4672 action space (from-square × move-type plane), which is +/// the policy-head width. The 73 planes are 56 "queen" moves (8 directions × 7 distances — this also carries +/// queen-promotions and every sliding/king/pawn move), 8 knight moves, and 9 underpromotions (Knight/Bishop/Rook × +/// {capture-left, straight, capture-right}). Distinct legal moves always map to distinct indices, so the search can +/// tell them apart; queen-promotions ride the queen planes (decoded promotion is inferred as Queen when a pawn +/// reaches the last rank — see ). +/// A thin facade over the single-source encoder (chess_solver.pg → PgChessState.encode/decode), +/// so the browser client and the training seam share one implementation. +/// +public static class ChessMoveEncoding +{ + public const int PlanesPerSquare = 73; + public const int Size = 64 * PlanesPerSquare; // 4672 + + /// The action index (0..4671) for a legal move. Queen-promotions use the queen planes; N/B/R + /// promotions use the underpromotion planes. + public static int Encode(ChessMove move) => PgChessState.encode(ChessState.ToPg(move)); + + /// Decodes an action index to a move. A queen-promotion decodes with — the + /// caller promotes a pawn reaching the last rank to a Queen by default. + public static ChessMove Decode(int index) => ChessState.FromPg(PgChessState.decode(index)); +} diff --git a/src/MintPlayer.AI.ReinforcementLearning.Environments/Chess/polyglot/chess_solver.pg b/src/MintPlayer.AI.ReinforcementLearning.Environments/Chess/polyglot/chess_solver.pg new file mode 100644 index 0000000..3d27ae3 --- /dev/null +++ b/src/MintPlayer.AI.ReinforcementLearning.Environments/Chess/polyglot/chess_solver.pg @@ -0,0 +1,740 @@ +// chess_solver.pg — single-source port of the perft-verified chess engine +// (ChessBoard.cs: ChessState + ChessRules, and ChessMoveEncoding.cs). ONE source, +// transpiled to C# (obj/, build-time, wrapped by the ChessState/ChessRules facades) +// and — from M40.2 — to a committed TypeScript twin the browser runs client-side. +// +// Types are `Pg`-prefixed and stay internal; the hand-written C# facades +// (ChessBoard.cs, ChessMoveEncoding.cs) adapt this i32/List core to the public +// PieceType/ChessMove/sbyte[] API the rest of the SDK and the perft tests consume. +// Perft (the movegen correctness gate) recurses entirely in this core. +// +// Subset choices (mirror the FruitCake precedent): the mailbox board is a List +// of 64 cells (0 empty, +1..+6 White P,N,B,R,Q,K, −1..−6 Black); promotion is an i32 +// piece code (0 = none, 2..5 = N,B,R,Q); castling is an i32 bitmask (WK=1,WQ=2,BK=4, +// BQ=8). No i64 — perft returns i32 (fits every position/depth the tests probe; +// the C# facade widens to long). Ray/slide loops are bounded `for`+`continue` (no +// `while`); delta tables are List<(i32,i32)> iterated with tuple destructuring — every +// construct here is one proven by fruitcake_solver.pg. + +import { List } from "std.collections" +import { Math } from "std.math" + +// A move: from/to squares (0..63 = rank*8 + file) plus the promotion piece code +// (0 = none) for a pawn reaching the last rank. Castling / en passant / double-push +// are inferred from the board when the move is made, so they need no extra flag. +record PgChessMove(from: i32, to: i32, promotion: i32) + +class PgChessState { + const PromoKnight: i32 = 2 + const PromoBishop: i32 = 3 + const PromoRook: i32 = 4 + const PromoQueen: i32 = 5 + + const CastleWK: i32 = 1 + const CastleWQ: i32 = 2 + const CastleBK: i32 = 4 + const CastleBQ: i32 = 8 + + const Size: i32 = 4672 // 64 × 73 AlphaZero action space + const PlanesPerSquare: i32 = 73 + + var squares: List // length 64 + var whiteToMove: bool + var castling: i32 + var enPassant: i32 // target square a pawn could capture onto, or -1 + var halfmoveClock: i32 + + init(squares: List, whiteToMove: bool, castling: i32, enPassant: i32, halfmoveClock: i32) { + this.squares = squares + this.whiteToMove = whiteToMove + this.castling = castling + this.enPassant = enPassant + this.halfmoveClock = halfmoveClock + } + + // ── small helpers (std.math covers abs/max/min type-preserving on i32; only sign is missing) ─ + static fn fileOf(sq: i32): i32 => sq % 8 + static fn rankOf(sq: i32): i32 => sq / 8 + static fn isign(x: i32): i32 => if x > 0 { 1 } else { if x < 0 { -1 } else { 0 } } + static fn onBoard(f: i32, r: i32): bool => f >= 0 && f < 8 && r >= 0 && r < 8 + static fn isWhite(piece: i32): bool => piece > 0 + static fn pieceType(piece: i32): i32 => if piece < 0 { -piece } else { piece } + // Clear the given bit(s) of a mask without a bitwise-not: drop the parts that ARE in `bits`. + static fn clearBits(mask: i32, bits: i32): i32 => mask - (mask & bits) + + // ── delta tables (built fresh; iterated with tuple destructuring) ───────────────────────── + static fn knightDeltas(): List<(i32, i32)> { + var d: List<(i32, i32)> = List<(i32, i32)>() + d.add((1, 2)); d.add((2, 1)); d.add((2, -1)); d.add((1, -2)) + d.add((-1, -2)); d.add((-2, -1)); d.add((-2, 1)); d.add((-1, 2)) + return d + } + static fn kingDeltas(): List<(i32, i32)> { + var d: List<(i32, i32)> = List<(i32, i32)>() + d.add((1, 0)); d.add((1, 1)); d.add((0, 1)); d.add((-1, 1)) + d.add((-1, 0)); d.add((-1, -1)); d.add((0, -1)); d.add((1, -1)) + return d + } + static fn bishopDirs(): List<(i32, i32)> { + var d: List<(i32, i32)> = List<(i32, i32)>() + d.add((1, 1)); d.add((-1, 1)); d.add((1, -1)); d.add((-1, -1)) + return d + } + static fn rookDirs(): List<(i32, i32)> { + var d: List<(i32, i32)> = List<(i32, i32)>() + d.add((1, 0)); d.add((-1, 0)); d.add((0, 1)); d.add((0, -1)) + return d + } + // Encoding tables (AlphaZero plane order — shared by encode and decode). + static fn encDirs(): List<(i32, i32)> { + var d: List<(i32, i32)> = List<(i32, i32)>() + d.add((0, 1)); d.add((1, 1)); d.add((1, 0)); d.add((1, -1)) + d.add((0, -1)); d.add((-1, -1)); d.add((-1, 0)); d.add((-1, 1)) + return d + } + static fn encKnights(): List<(i32, i32)> { + var d: List<(i32, i32)> = List<(i32, i32)>() + d.add((1, 2)); d.add((2, 1)); d.add((2, -1)); d.add((1, -2)) + d.add((-1, -2)); d.add((-2, -1)); d.add((-2, 1)); d.add((-1, 2)) + return d + } + + fn clone(): PgChessState { + var sq: List = List() + for v in this.squares { sq.add(v) } + return PgChessState(sq, this.whiteToMove, this.castling, this.enPassant, this.halfmoveClock) + } + + // ── attack detection ────────────────────────────────────────────────────────────────────── + fn isSquareAttacked(square: i32, byWhite: bool): bool { + let tf = fileOf(square) + let tr = rankOf(square) + + // Pawns: an attacking pawn sits one rank behind the target in its forward direction. + let pawnDir = if byWhite { 1 } else { -1 } + let pawn = if byWhite { 1 } else { -1 } + for k in 0..2 { + let df = k * 2 - 1 // -1, +1 + let f = tf + df + let r = tr - pawnDir + if onBoard(f, r) && this.squares[r * 8 + f] == pawn { return true } + } + + let knight = if byWhite { 2 } else { -2 } + let kn = knightDeltas() + for (df, dr) in kn { + let f = tf + df + let r = tr + dr + if onBoard(f, r) && this.squares[r * 8 + f] == knight { return true } + } + + let king = if byWhite { 6 } else { -6 } + let kg = kingDeltas() + for (df, dr) in kg { + let f = tf + df + let r = tr + dr + if onBoard(f, r) && this.squares[r * 8 + f] == king { return true } + } + + if this.rayHits(tf, tr, bishopDirs(), byWhite, 3) { return true } // bishops/queens + if this.rayHits(tf, tr, rookDirs(), byWhite, 4) { return true } // rooks/queens + return false + } + + // A sliding piece of type `slider` (or a queen, code 5) of colour `byWhite` attacks (tf,tr) + // along one of `dirs`. Bounded to 7 steps; `blocked` stops the walk at the first obstruction. + fn rayHits(tf: i32, tr: i32, dirs: List<(i32, i32)>, byWhite: bool, slider: i32): bool { + for (df, dr) in dirs { + var found = false + var blocked = false + for step in 1..8 { + if blocked { continue } + let f = tf + df * step + let r = tr + dr * step + if !onBoard(f, r) { blocked = true; continue } + let piece = this.squares[r * 8 + f] + if piece != 0 { + if isWhite(piece) == byWhite { + let t = pieceType(piece) + if t == slider || t == 5 { found = true } + } + blocked = true + } + } + if found { return true } + } + return false + } + + fn kingSquare(white: bool): i32 { + let king = if white { 6 } else { -6 } + for sq in 0..64 { if this.squares[sq] == king { return sq } } + return -1 + } + + fn inCheck(white: bool): bool => this.isSquareAttacked(this.kingSquare(white), !white) + + // ── move generation ───────────────────────────────────────────────────────────────────── + fn legalMoves(): List { + var legal: List = List() + let pseudo = this.pseudoLegal() + for m in pseudo { + let next = this.makeMove(m) + if !next.inCheck(this.whiteToMove) { legal.add(m) } // the side that just moved must be safe + } + return legal + } + + fn pseudoLegal(): List { + var moves: List = List() + let white = this.whiteToMove + for sq in 0..64 { + let piece = this.squares[sq] + if piece == 0 || isWhite(piece) != white { continue } + let f = fileOf(sq) + let r = rankOf(sq) + let t = pieceType(piece) + if t == 1 { this.pawnMoves(sq, f, r, white, moves) } + else if t == 2 { this.stepMoves(sq, f, r, white, knightDeltas(), moves) } + else if t == 6 { this.stepMoves(sq, f, r, white, kingDeltas(), moves); this.castleMoves(white, moves) } + else if t == 3 { this.slideMoves(sq, f, r, white, bishopDirs(), moves) } + else if t == 4 { this.slideMoves(sq, f, r, white, rookDirs(), moves) } + else if t == 5 { + this.slideMoves(sq, f, r, white, bishopDirs(), moves) + this.slideMoves(sq, f, r, white, rookDirs(), moves) + } + } + return moves + } + + fn stepMoves(sq: i32, f: i32, r: i32, white: bool, deltas: List<(i32, i32)>, moves: List) { + for (df, dr) in deltas { + let nf = f + df + let nr = r + dr + if !onBoard(nf, nr) { continue } + let to = nr * 8 + nf + let target = this.squares[to] + if target == 0 || isWhite(target) != white { moves.add(PgChessMove(sq, to, 0)) } + } + } + + fn slideMoves(sq: i32, f: i32, r: i32, white: bool, dirs: List<(i32, i32)>, moves: List) { + for (df, dr) in dirs { + var blocked = false + for step in 1..8 { + if blocked { continue } + let nf = f + df * step + let nr = r + dr * step + if !onBoard(nf, nr) { blocked = true; continue } + let to = nr * 8 + nf + let target = this.squares[to] + if target == 0 { moves.add(PgChessMove(sq, to, 0)) } + else { + if isWhite(target) != white { moves.add(PgChessMove(sq, to, 0)) } + blocked = true + } + } + } + } + + fn pawnMoves(sq: i32, f: i32, r: i32, white: bool, moves: List) { + let dir = if white { 1 } else { -1 } + let startRank = if white { 1 } else { 6 } + let lastRank = if white { 7 } else { 0 } + + let one = (r + dir) * 8 + f + if this.squares[one] == 0 { + this.addPawn(sq, one, r + dir == lastRank, moves) + if r == startRank { + let two = (r + 2 * dir) * 8 + f + if this.squares[two] == 0 { moves.add(PgChessMove(sq, two, 0)) } + } + } + + for k in 0..2 { + let df = k * 2 - 1 // -1, +1 + let nf = f + df + let nr = r + dir + if !onBoard(nf, nr) { continue } + let to = nr * 8 + nf + let target = this.squares[to] + if target != 0 && isWhite(target) != white { this.addPawn(sq, to, nr == lastRank, moves) } + else if to == this.enPassant { moves.add(PgChessMove(sq, to, 0)) } // en passant onto the empty ep square + } + } + + fn addPawn(from: i32, to: i32, promotion: bool, moves: List) { + if promotion { + moves.add(PgChessMove(from, to, PromoQueen)) + moves.add(PgChessMove(from, to, PromoRook)) + moves.add(PgChessMove(from, to, PromoBishop)) + moves.add(PgChessMove(from, to, PromoKnight)) + } else { + moves.add(PgChessMove(from, to, 0)) + } + } + + fn castleMoves(white: bool, moves: List) { + let rank = if white { 0 } else { 7 } + let kingSq = rank * 8 + 4 + let ownKing = if white { 6 } else { -6 } + if this.squares[kingSq] != ownKing { return } + if this.isSquareAttacked(kingSq, !white) { return } // can't castle out of check + + let kSide = if white { CastleWK } else { CastleBK } + let qSide = if white { CastleWQ } else { CastleBQ } + + // King-side: f,g empty; king doesn't pass through/into attack. + let kEmpty = this.squares[rank * 8 + 5] == 0 && this.squares[rank * 8 + 6] == 0 + let kSafe = !this.isSquareAttacked(rank * 8 + 5, !white) && !this.isSquareAttacked(rank * 8 + 6, !white) + if (this.castling & kSide) != 0 && kEmpty && kSafe { moves.add(PgChessMove(kingSq, rank * 8 + 6, 0)) } + + // Queen-side: b,c,d empty; king passes over d,c (b need not be safe, only empty). + let qEmpty = this.squares[rank * 8 + 1] == 0 && this.squares[rank * 8 + 2] == 0 && this.squares[rank * 8 + 3] == 0 + let qSafe = !this.isSquareAttacked(rank * 8 + 3, !white) && !this.isSquareAttacked(rank * 8 + 2, !white) + if (this.castling & qSide) != 0 && qEmpty && qSafe { moves.add(PgChessMove(kingSq, rank * 8 + 2, 0)) } + } + + // ── make move ─────────────────────────────────────────────────────────────────────────── + // The position after `move` (a fresh state; this one is unchanged). Castling / en passant / + // promotion / double-push are inferred from the board. + fn makeMove(move: PgChessMove): PgChessState { + var b: List = List() + for v in this.squares { b.add(v) } + let white = this.whiteToMove + let piece = b[move.from] + let t = pieceType(piece) + var capture = b[move.to] != 0 + var newEp = -1 + var castling = this.castling + + b[move.from] = 0 + + // En passant capture: the taken pawn sits beside the destination, not on it. + if t == 1 && move.to == this.enPassant && this.enPassant >= 0 { + let capturedSq = move.to + (if white { -8 } else { 8 }) + b[capturedSq] = 0 + capture = true + } + + // Place the piece (promotion swaps in the chosen piece). + if move.promotion != 0 { + b[move.to] = (if white { 1 } else { -1 }) * move.promotion + } else { + b[move.to] = piece + } + + // Double push sets the en-passant target square. + if t == 1 && Math.abs(rankOf(move.to) - rankOf(move.from)) == 2 { + newEp = (move.from + move.to) / 2 + } + + // Castling: move the rook too. + if t == 6 && Math.abs(fileOf(move.to) - fileOf(move.from)) == 2 { + let rank = rankOf(move.from) + if fileOf(move.to) == 6 { b[rank * 8 + 5] = b[rank * 8 + 7]; b[rank * 8 + 7] = 0 } // king-side + else { b[rank * 8 + 3] = b[rank * 8 + 0]; b[rank * 8 + 0] = 0 } // queen-side + } + + // Update castling rights: a king move clears both of its sides; a rook move/capture on a + // home corner clears that right. + if t == 6 { + if white { castling = clearBits(castling, CastleWK + CastleWQ) } + else { castling = clearBits(castling, CastleBK + CastleBQ) } + } + castling = clearBits(castling, cornerRight(move.from)) + castling = clearBits(castling, cornerRight(move.to)) // a rook captured on its home square loses that right too + + let halfmove = if t == 1 || capture { 0 } else { this.halfmoveClock + 1 } + return PgChessState(b, !white, castling, newEp, halfmove) + } + + // The castling-right bit anchored at a rook/king home corner (0 = none). + static fn cornerRight(sq: i32): i32 { + if sq == 0 { return CastleWQ } // a1 rook + if sq == 7 { return CastleWK } // h1 rook + if sq == 56 { return CastleBQ } // a8 rook + if sq == 63 { return CastleBK } // h8 rook + return 0 + } + + // ── terminal detection ────────────────────────────────────────────────────────────────── + fn isFiftyMove(): bool => this.halfmoveClock >= 100 + + fn isInsufficientMaterial(): bool { + var knights = 0 + var bishops = 0 + for p in this.squares { + let t = pieceType(p) + if t == 0 || t == 6 { continue } + else if t == 2 { knights += 1 } + else if t == 3 { bishops += 1 } + else { return false } // a pawn/rook/queen = enough material + } + return knights + bishops <= 1 // K vs K, K+N vs K, K+B vs K (the common draws) + } + + // Terminal result FROM THE SIDE TO MOVE's view: 0 = ongoing, 1 = loss (checkmated), 2 = draw. + fn result(): i32 { + if this.legalMoves().count == 0 { + return if this.inCheck(this.whiteToMove) { 1 } else { 2 } // mate vs stalemate + } + if this.isFiftyMove() || this.isInsufficientMaterial() { return 2 } + return 0 + } + + // Perft: the number of leaf nodes at `depth` — the movegen correctness oracle. i32 suffices + // for every position/depth the tests probe (max ~4.9M); the C# facade widens to long. + fn perft(depth: i32): i32 { + if depth == 0 { return 1 } + let moves = this.legalMoves() + if depth == 1 { return moves.count } + var nodes = 0 + for m in moves { nodes += this.makeMove(m).perft(depth - 1) } + return nodes + } + + // ── seam helpers (index-level API the C# facade / browser consume) ──────────────────────── + fn legalMoveIndices(): List { + var out: List = List() + let moves = this.legalMoves() + for m in moves { out.add(encode(m)) } + return out + } + + // The 18-plane × 64 = 1152 observation the net consumes (single-sourced; the C# facade casts f64→float + // for the float32 net, the browser feeds it directly). Planes: [0–5] White P,N,B,R,Q,K, [6–11] Black, + // [12] side-to-move (all-ones iff white to move), [13–16] castling WK/WQ/BK/BQ (each all-ones iff the + // right is held), [17] en-passant target square (one-hot). Must match ChessGame.WriteObservation exactly. + fn writeObservation(): List { + var obs: List = List() + for i in 0..1152 { obs.add(0.0) } + + for sq in 0..64 { + let piece = this.squares[sq] + if piece == 0 { continue } + let t = pieceType(piece) - 1 // 0..5 + let plane = if piece > 0 { t } else { 6 + t } // white vs black block + obs[plane * 64 + sq] = 1.0 + } + if this.whiteToMove { this.fillPlane(obs, 12) } + if (this.castling & CastleWK) != 0 { this.fillPlane(obs, 13) } + if (this.castling & CastleWQ) != 0 { this.fillPlane(obs, 14) } + if (this.castling & CastleBK) != 0 { this.fillPlane(obs, 15) } + if (this.castling & CastleBQ) != 0 { this.fillPlane(obs, 16) } + if this.enPassant >= 0 { obs[17 * 64 + this.enPassant] = 1.0 } + return obs + } + + fn fillPlane(obs: List, plane: i32) { + for sq in 0..64 { obs[plane * 64 + sq] = 1.0 } + } + + // Apply an encoded action index (a queen-promotion rides the queen planes and decodes with + // promotion 0 → promote a pawn reaching the last rank to a Queen by default). + fn applyIndex(index: i32): PgChessState { + let m = decode(index) + var chosen = m + if m.promotion == 0 && pieceType(this.squares[m.from]) == 1 { + let toRank = rankOf(m.to) + if toRank == 0 || toRank == 7 { chosen = PgChessMove(m.from, m.to, PromoQueen) } + } + return this.makeMove(chosen) + } + + // ── AlphaZero 4672 move encoding (64 × 73 = from-square × plane) ────────────────────────── + // 56 queen planes (8 dir × 7 dist — also carry queen-promotions and king/pawn moves) + 8 + // knight planes + 9 underpromotion planes (N/B/R × {capture-left, straight, capture-right}). + static fn encode(move: PgChessMove): i32 { + let df = fileOf(move.to) - fileOf(move.from) + let dr = rankOf(move.to) - rankOf(move.from) + var plane = 0 + if move.promotion == PromoKnight || move.promotion == PromoBishop || move.promotion == PromoRook { + let pieceIdx = if move.promotion == PromoKnight { 0 } else { if move.promotion == PromoBishop { 1 } else { 2 } } + plane = 64 + pieceIdx * 3 + (df + 1) // df ∈ {-1,0,1} → {capture-left, straight, capture-right} + } else if isKnightMove(df, dr) { + plane = 56 + knightIndex(df, dr) + } else { + let dir = dirIndex(isign(df), isign(dr)) + let dist = Math.max(Math.abs(df), Math.abs(dr)) + plane = dir * 7 + (dist - 1) + } + return move.from * PlanesPerSquare + plane + } + + // Decode an action index to a move. A queen-promotion decodes with promotion 0 — the caller + // promotes a pawn reaching the last rank to a Queen by default (see applyIndex). + static fn decode(index: i32): PgChessMove { + let from = index / PlanesPerSquare + let plane = index % PlanesPerSquare + let f = fileOf(from) + let r = rankOf(from) + + if plane >= 64 { + let p = plane - 64 + let promo = if p / 3 == 0 { PromoKnight } else { if p / 3 == 1 { PromoBishop } else { PromoRook } } + let df = (p % 3) - 1 + let dr = if r == 6 { 1 } else { -1 } // a pawn on the 7th rank promotes upward, on the 2nd downward + return PgChessMove(from, (r + dr) * 8 + (f + df), promo) + } + if plane >= 56 { + let target = plane - 56 + var i = 0 + var rdf = 0 + var rdr = 0 + for (df, dr) in encKnights() { + if i == target { rdf = df; rdr = dr } + i += 1 + } + return PgChessMove(from, (r + rdr) * 8 + (f + rdf), 0) + } + let dirTarget = plane / 7 + let dist = plane % 7 + 1 + var j = 0 + var ddf = 0 + var ddr = 0 + for (df, dr) in encDirs() { + if j == dirTarget { ddf = df; ddr = dr } + j += 1 + } + return PgChessMove(from, (r + ddr * dist) * 8 + (f + ddf * dist), 0) + } + + static fn isKnightMove(df: i32, dr: i32): bool { + let a = Math.abs(df) + let b = Math.abs(dr) + return (a == 1 && b == 2) || (a == 2 && b == 1) + } + + static fn knightIndex(df: i32, dr: i32): i32 { + var i = 0 + var found = -1 + for (kdf, kdr) in encKnights() { + if kdf == df && kdr == dr { found = i } + i += 1 + } + return found + } + + static fn dirIndex(sdf: i32, sdr: i32): i32 { + var i = 0 + var found = -1 + for (ddf, ddr) in encDirs() { + if ddf == sdf && ddr == sdr { found = i } + i += 1 + } + return found + } +} + +// ── Two-headed policy/value net inference (single-source forward pass) ──────────────────────── +// Mirrors Core/Nn/PolicyValueNet: a ReLU trunk → policy logits (one per action, linear) + a scalar +// value (linear). INFERENCE ONLY — training stays on the SDK autograd/GEMM. Weights are float32 in +// the .ckpt; the per-platform parser (chess-net.ts / the C# parity loader) loads them into these f64 +// flat arrays. Trunk weights/biases are concatenated in layer order (each row-major [in, out]) with +// per-layer offsets from `hidden` — a single List avoids nested-generic (List>) params, +// which the transpiler mishandles (same shape as fruitcake's PgDuelingNet). + +// forward() output: raw policy logits (length actions) + the raw scalar value (linear — the leaf +// evaluator applies tanh, matching PolicyValueNet's linear value head + SelfPlayCampaign.Evaluate). +record PgNetOut(logits: List, value: f64) + +class PgPolicyValueNet { + var inputSize: i32 + var actions: i32 + var hidden: List // trunk hidden widths (from the checkpoint) + var trunkWFlat: List // trunk weight matrices concatenated in layer order (each row-major [in, out]) + var trunkBFlat: List // trunk biases concatenated in layer order + var policyW: List // [lastHidden * actions] + var policyB: List // [actions] + var valueW: List // [lastHidden * 1] + var valueB: List // [1] + + init(inputSize: i32, actions: i32, hidden: List, trunkWFlat: List, trunkBFlat: List, policyW: List, policyB: List, valueW: List, valueB: List) { + this.inputSize = inputSize + this.actions = actions + this.hidden = hidden + this.trunkWFlat = trunkWFlat + this.trunkBFlat = trunkBFlat + this.policyW = policyW + this.policyB = policyB + this.valueW = valueW + this.valueB = valueB + } + + // obs (length inputSize) → logits (length actions) + scalar value. Matches PolicyValueNet.Forward on one row. + fn forward(obs: List): PgNetOut { + var x = obs + var prev = this.inputSize + var wOff = 0 + var bOff = 0 + for l in 0..this.hidden.count { + let h = this.hidden[l] + x = relu(linear(x, this.trunkWFlat, wOff, this.trunkBFlat, bOff, prev, h)) + wOff += prev * h + bOff += h + prev = h + } + let logits = linear(x, this.policyW, 0, this.policyB, 0, prev, this.actions) + let value = linear(x, this.valueW, 0, this.valueB, 0, prev, 1) + return PgNetOut(logits, value[0]) + } + + // Dense layer: out[o] = b[bOff+o] + Σ_i x[i]·w[wOff + i*outDim + o] (w row-major [inDim, outDim]). + static fn linear(x: List, w: List, wOff: i32, b: List, bOff: i32, inDim: i32, outDim: i32): List { + var out: List = List() + for o in 0..outDim { + var s = b[bOff + o] + for i in 0..inDim { s += x[i] * w[wOff + i * outDim + o] } + out.add(s) + } + return out + } + + static fn relu(x: List): List { + var out: List = List() + for v in x { out.add(Math.max(0.0, v)) } + return out + } +} + +// ── Inference MCTS (AlphaZero PUCT) ─────────────────────────────────────────────────────────── +// Single-source port of Core/Planning/Mcts, minus the self-play-only Dirichlet root noise (this is +// the browser/serving path). Each simulation walks the tree maximizing Q + U, expands one leaf, and +// backs the net value up the path NEGATING it every ply (zero-sum). `search` returns the root +// visit-count distribution over the 4672 action space; `chooseMove` is its argmax. + +// A leaf evaluation: priors aligned to the node's legal-move list (masked-softmax over legal) + the +// tanh value from the side-to-move's perspective. +record PgLeafEval(priors: List, value: f64) + +class PgMctsNode { + var moves: List // legal action indices (empty if terminal) + var p: List // prior per move (aligned to moves) + var n: List // visit count per move + var w: List // summed value per move (this node's mover perspective) + var children: List + var expanded: bool + var terminal: bool + var terminalValue: f64 // set when terminal: -1 loss (side-to-move mated) / 0 draw, mover-relative + + init(moves: List, terminal: bool, terminalValue: f64) { + this.moves = moves + this.terminal = terminal + this.terminalValue = terminalValue + this.p = List() + this.n = List() + this.w = List() + this.children = List() + this.expanded = false + } +} + +class PgChessMcts { + // Run `sims` PUCT simulations from `root` and return the root visit-count distribution over the + // 4672 action space (0 for illegal/unvisited). Falls back to raw priors if nothing was visited. + static fn search(net: PgPolicyValueNet, root: PgChessState, sims: i32, cpuct: f64): List { + let rootNode = newNode(root) + if !rootNode.terminal { + expandLeaf(rootNode, net, root) + for s in 0..sims { simulate(rootNode, net, root, cpuct) } + } + + var pi: List = List() + for i in 0..4672 { pi.add(0.0) } + var total = 0 + for i in 0..rootNode.moves.count { total += rootNode.n[i] } + if total == 0 { + for i in 0..rootNode.moves.count { pi[rootNode.moves[i]] = rootNode.p[i] } + return pi + } + for i in 0..rootNode.moves.count { pi[rootNode.moves[i]] = rootNode.n[i] * 1.0 / total } + return pi + } + + // Best action index (argmax visit count via the returned distribution). + static fn chooseMove(net: PgPolicyValueNet, root: PgChessState, sims: i32, cpuct: f64): i32 { + let pi = search(net, root, sims, cpuct) + var best = -1 + var bestV = -1.0 + for i in 0..pi.count { if pi[i] > bestV { bestV = pi[i]; best = i } } + return best + } + + // Returns the value of `state` from the perspective of its side to move. + static fn simulate(node: PgMctsNode, net: PgPolicyValueNet, state: PgChessState, cpuct: f64): f64 { + if node.terminal { return node.terminalValue } + if !node.expanded { return expandLeaf(node, net, state) } + + let edge = selectChild(node, cpuct) + let childState = state.applyIndex(node.moves[edge]) + if node.children[edge] == null { node.children[edge] = newNode(childState) } + let child = node.children[edge]! + let value = -simulate(child, net, childState, cpuct) // flip: child mover's value + node.n[edge] = node.n[edge] + 1 + node.w[edge] = node.w[edge] + value + return value + } + + static fn selectChild(node: PgMctsNode, cpuct: f64): i32 { + var sumN = 0 + for i in 0..node.n.count { sumN += node.n[i] } + let sqrtSum = Math.sqrt(sumN * 1.0) + var best = 0 + var bestScore = -1.0e30 + for i in 0..node.moves.count { + let q = if node.n[i] > 0 { node.w[i] / node.n[i] } else { 0.0 } + let u = cpuct * node.p[i] * sqrtSum / (1.0 + node.n[i]) + let score = q + u + if score > bestScore { bestScore = score; best = i } + } + return best + } + + static fn newNode(state: PgChessState): PgMctsNode { + let r = state.result() // 0 ongoing, 1 loss (side-to-move mated), 2 draw + if r != 0 { + let tv = if r == 1 { -1.0 } else { 0.0 } + return PgMctsNode(List(), true, tv) + } + return PgMctsNode(state.legalMoveIndices(), false, 0.0) + } + + // Evaluate the leaf, seed per-move priors (masked-softmax over legal), return the net's value estimate. + static fn expandLeaf(node: PgMctsNode, net: PgPolicyValueNet, state: PgChessState): f64 { + let ev = evaluate(net, state, node.moves) + let k = node.moves.count + node.p = List() + node.n = List() + node.w = List() + node.children = List() + for i in 0..k { + node.p.add(ev.priors[i]) + node.n.add(0) + node.w.add(0.0) + node.children.add(null) + } + node.expanded = true + return ev.value + } + + // Masked-softmax priors over `moves` (aligned) + tanh value. Mirrors SelfPlayCampaign.Evaluate. + static fn evaluate(net: PgPolicyValueNet, state: PgChessState, moves: List): PgLeafEval { + let out = net.forward(state.writeObservation()) + var mx = -1.0e30 + for m in moves { if out.logits[m] > mx { mx = out.logits[m] } } + var pr: List = List() + var sum = 0.0 + for m in moves { + let e = Math.exp(out.logits[m] - mx) + pr.add(e) + sum += e + } + if sum > 0.0 { + for i in 0..pr.count { pr[i] = pr[i] / sum } + } else { + for i in 0..pr.count { pr[i] = 1.0 / pr.count } + } + return PgLeafEval(pr, Math.tanh(out.value)) + } +} diff --git a/src/MintPlayer.AI.ReinforcementLearning.Environments/Connect4/Connect4Game.cs b/src/MintPlayer.AI.ReinforcementLearning.Environments/Connect4/Connect4Game.cs new file mode 100644 index 0000000..162f5cc --- /dev/null +++ b/src/MintPlayer.AI.ReinforcementLearning.Environments/Connect4/Connect4Game.cs @@ -0,0 +1,100 @@ +using MintPlayer.AI.ReinforcementLearning.Core.Planning; + +namespace MintPlayer.AI.ReinforcementLearning.Environments.Connect4; + +/// A Connect-4 position: 7×6 cells (0 = empty, 1 / 2 = the two players) plus the side to move. +/// Immutable from the caller's view — always returns a fresh state. +public sealed class Connect4State +{ + public const int Columns = 7; + public const int Rows = 6; + + /// Row-major, row 0 = bottom: cell (row, col) = Cells[row * Columns + col]. + public byte[] Cells { get; } + public int ToMove { get; } // 1 or 2 + + public Connect4State(byte[] cells, int toMove) { Cells = cells; ToMove = toMove; } +} + +/// +/// Connect-4 as an — the cheap first consumer of the self-play stack (tiny rules, +/// converges from random self-play in minutes on CPU, and a shallow negamax gives an exact test oracle). The action +/// index is the column (0..6); the observation is a side-to-move-relative two-plane (mine / theirs) board. +/// +public sealed class Connect4Game : IZeroSumGame +{ + public int PolicySize => Connect4State.Columns; // one action per column + public int ObservationSize => 2 * Connect4State.Columns * Connect4State.Rows; // mine + theirs planes + + public Connect4State Root(ulong? seed = null) => new(new byte[Connect4State.Columns * Connect4State.Rows], toMove: 1); + + public IReadOnlyList LegalMoves(Connect4State state) + { + var moves = new List(Connect4State.Columns); + int topRowBase = (Connect4State.Rows - 1) * Connect4State.Columns; + for (int col = 0; col < Connect4State.Columns; col++) + if (state.Cells[topRowBase + col] == 0) moves.Add(col); // top cell empty → column not full + return moves; + } + + public Connect4State Apply(Connect4State state, int move) + { + var cells = (byte[])state.Cells.Clone(); + for (int row = 0; row < Connect4State.Rows; row++) + { + int idx = row * Connect4State.Columns + move; + if (cells[idx] == 0) { cells[idx] = (byte)state.ToMove; break; } + } + return new Connect4State(cells, toMove: 3 - state.ToMove); + } + + public GameResult Result(Connect4State state) + { + int opponent = 3 - state.ToMove; + if (HasFour(state.Cells, opponent)) return GameResult.Loss; // the player who just moved completed a line + if (HasFour(state.Cells, state.ToMove)) return GameResult.Win; // defensive; unreachable in normal play + return IsFull(state.Cells) ? GameResult.Draw : GameResult.Ongoing; + } + + public void WriteObservation(Connect4State state, Span destination) + { + int n = Connect4State.Columns * Connect4State.Rows; + int mine = state.ToMove, theirs = 3 - state.ToMove; + for (int i = 0; i < n; i++) + { + destination[i] = state.Cells[i] == mine ? 1f : 0f; + destination[n + i] = state.Cells[i] == theirs ? 1f : 0f; + } + } + + private static bool IsFull(byte[] cells) + { + int topRowBase = (Connect4State.Rows - 1) * Connect4State.Columns; + for (int col = 0; col < Connect4State.Columns; col++) + if (cells[topRowBase + col] == 0) return false; + return true; + } + + // True if player p has any four-in-a-row (horizontal, vertical, or either diagonal). + private static bool HasFour(byte[] cells, int p) + { + const int C = Connect4State.Columns, R = Connect4State.Rows; + for (int row = 0; row < R; row++) + for (int col = 0; col < C; col++) + { + if (cells[row * C + col] != p) continue; + if (col + 3 < C && Line(cells, p, row, col, 0, 1)) return true; // → + if (row + 3 < R && Line(cells, p, row, col, 1, 0)) return true; // ↑ + if (row + 3 < R && col + 3 < C && Line(cells, p, row, col, 1, 1)) return true; // ↗ + if (row + 3 < R && col - 3 >= 0 && Line(cells, p, row, col, 1, -1)) return true; // ↖ + } + return false; + } + + private static bool Line(byte[] cells, int p, int row, int col, int dRow, int dCol) + { + for (int step = 1; step < 4; step++) + if (cells[(row + step * dRow) * Connect4State.Columns + (col + step * dCol)] != p) return false; + return true; + } +} diff --git a/src/MintPlayer.AI.ReinforcementLearning.Environments/Connect4/Connect4Solver.cs b/src/MintPlayer.AI.ReinforcementLearning.Environments/Connect4/Connect4Solver.cs new file mode 100644 index 0000000..e0fade2 --- /dev/null +++ b/src/MintPlayer.AI.ReinforcementLearning.Environments/Connect4/Connect4Solver.cs @@ -0,0 +1,48 @@ +using MintPlayer.AI.ReinforcementLearning.Core.Planning; + +namespace MintPlayer.AI.ReinforcementLearning.Environments.Connect4; + +/// +/// A depth-limited negamax solver for Connect-4 — the exact oracle the self-play tests measure MCTS against (the same +/// role Kociemba plays for the cube and BFS for Rush Hour). Scores are from the side-to-move's perspective: +/// +1 = a forced win within the horizon, -1 = a forced loss, 0 = a draw or undecided within the +/// depth. Full-game solving is exponential, so callers pass a modest maxDepth and use it on tactical positions. +/// +public static class Connect4Solver +{ + private static readonly Connect4Game Game = new(); + + /// The best move for the side to move (preferring a forced win, else a draw, avoiding a loss) and its + /// negamax score in {-1, 0, +1} within plies. + public static (int Score, int BestMove) Solve(Connect4State state, int maxDepth) + { + int bestScore = int.MinValue, bestMove = -1; + foreach (int move in Game.LegalMoves(state)) + { + int score = -Negamax(Game.Apply(state, move), maxDepth - 1); + if (score > bestScore) { bestScore = score; bestMove = move; } + } + return (bestScore == int.MinValue ? 0 : bestScore, bestMove); + } + + /// The forced result of for its side to move within plies. + public static int Negamax(Connect4State state, int depth) + { + switch (Game.Result(state)) + { + case GameResult.Loss: return -1; // side to move has already lost + case GameResult.Win: return 1; + case GameResult.Draw: return 0; + } + if (depth <= 0) return 0; // undecided within the horizon → treat as neutral + + int best = -2; // below the -1..+1 range + foreach (int move in Game.LegalMoves(state)) + { + int value = -Negamax(Game.Apply(state, move), depth - 1); + if (value > best) best = value; + if (best == 1) break; // a forced win can't be improved on — prune + } + return best; + } +} diff --git a/src/MintPlayer.AI.ReinforcementLearning.Environments/MintPlayer.AI.ReinforcementLearning.Environments.csproj b/src/MintPlayer.AI.ReinforcementLearning.Environments/MintPlayer.AI.ReinforcementLearning.Environments.csproj index c81fcff..435a3d1 100644 --- a/src/MintPlayer.AI.ReinforcementLearning.Environments/MintPlayer.AI.ReinforcementLearning.Environments.csproj +++ b/src/MintPlayer.AI.ReinforcementLearning.Environments/MintPlayer.AI.ReinforcementLearning.Environments.csproj @@ -4,12 +4,14 @@ - + - +