From 0799674f85b3e21e878e05f1e93ccd0707f7117b Mon Sep 17 00:00:00 2001 From: PieterjanDeClippel Date: Mon, 22 Jun 2026 09:18:54 +0200 Subject: [PATCH 1/5] Snake: switch off the trainer's redundant internal eval (campaign owns eval) The campaign runs its own authoritative 12x12 eval + save-best; the trainer's internal eval ran long shielded episodes and was ~halving throughput. Set the trainer EvalEvery off so the curriculum stages train ~2x faster. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../SnakeDqnCampaign.cs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tools/MintPlayer.AI.ReinforcementLearning.Lab/SnakeDqnCampaign.cs b/tools/MintPlayer.AI.ReinforcementLearning.Lab/SnakeDqnCampaign.cs index 886ec85..1da12ac 100644 --- a/tools/MintPlayer.AI.ReinforcementLearning.Lab/SnakeDqnCampaign.cs +++ b/tools/MintPlayer.AI.ReinforcementLearning.Lab/SnakeDqnCampaign.cs @@ -98,9 +98,10 @@ public long TrainChunk() int from = _state?.StepsCompleted ?? 0; int to = from + chunkSteps; if (targetSteps > 0) to = (int)Math.Min(to, targetSteps); - // EvalEvery == the chunk size so the trainer's own (6×6) eval fires at most once per chunk — the campaign's - // authoritative eval is the 12×12 food in Evaluate(). MaxSteps is ABSOLUTE: resuming raises the ceiling. - var options = BaseOptions with { MaxSteps = to, EvalEvery = Math.Max(1, chunkSteps) }; + // Switch OFF the trainer's internal eval (EvalEvery huge): the campaign runs its own authoritative 12×12 + // eval + save-best in Evaluate(), and with the survival shield the trainer's on-policy eval episodes run + // very long — pure overhead that was ~halving throughput. MaxSteps is ABSOLUTE: resuming raises the ceiling. + var options = BaseOptions with { MaxSteps = to, EvalEvery = int.MaxValue }; // First chunk: resume the full state if present, else warm-start from the deployable net (if any). var result = DqnTrainer.Train(_env, options, _seeds, resume: _state, warmStart: _state is null ? _warmNet : null); _state = result.State; From 6ed755189de0ec66537363bdb52f4ec838662d7c Mon Sep 17 00:00:00 2001 From: PieterjanDeClippel Date: Mon, 22 Jun 2026 10:01:03 +0200 Subject: [PATCH 2/5] Snake (M27): lean ray-cast observation, replacing the egocentric patch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the 9x9 obstacle/food patch (162 of 177 inputs, myopic ±4 cells, mostly zeros) with CodeBullet-style 8-direction ray-cast vision: per ray, food-on-line + 1/dist-to-body + 1/dist-to-wall (24 inputs of long-range awareness a fixed window can't give). Kept flood-fill (anti-trap), food/tail bearing, heading, length. ObservationSize 177 -> 39: leaner + the right info (rays + flood-fill). Size-invariant. Removed the dead patch consts + loop. 8 env tests green. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../Snake/SnakeEnv.cs | 102 +++++++++--------- .../SnakeEnvTests.cs | 24 +++-- 2 files changed, 61 insertions(+), 65 deletions(-) diff --git a/src/MintPlayer.AI.ReinforcementLearning.Environments/Snake/SnakeEnv.cs b/src/MintPlayer.AI.ReinforcementLearning.Environments/Snake/SnakeEnv.cs index 79edbba..44c2bc8 100644 --- a/src/MintPlayer.AI.ReinforcementLearning.Environments/Snake/SnakeEnv.cs +++ b/src/MintPlayer.AI.ReinforcementLearning.Environments/Snake/SnakeEnv.cs @@ -6,11 +6,11 @@ namespace MintPlayer.AI.ReinforcementLearning.Environments.Snake; /// /// Classic Snake on a configurable × grid, as an RL environment -/// (PLAN M22; observation reworked in M27). The observation is egocentric and grid-size-invariant: a -/// × obstacle+food patch centred on the head, plus food -/// direction/distance, heading, length, and a flood-fill of the open space reachable through each move. The -/// flood-fill is what lets a long snake avoid trapping itself — a fixed window alone can't see a coil beyond its -/// radius. Four absolute-direction actions; the single illegal action (the 180° reversal onto the neck) is masked +/// (PLAN M22; observation reworked in M27). The observation is grid-size-invariant: 8-direction ray-cast +/// vision (food-on-ray, 1/distance to body, 1/distance to wall) for long-range awareness, plus a flood-fill of the +/// open space reachable through each move (the anti-self-trap signal — a long snake can't see a coil beyond a fixed +/// window, but it can be told which moves keep space reachable), food & tail bearing + distance, heading, and +/// length. Four absolute-direction actions; the single illegal action (the 180° reversal onto the neck) is masked /// via . Walls and the snake's own body are NOT masked — death stays learnable. /// /// Episodes end on death, a board-full win, a starvation timeout ( steps without food), @@ -21,17 +21,15 @@ public sealed class SnakeEnv : IEnvironment, IActionMaskProvider, { public const int ActionCount = 4; - // Egocentric, grid-size-invariant observation (PLAN M27). A (2R+1)×(2R+1) patch centred on the head — two - // channels (obstacle = wall or non-vacating body; food) so a CNN can read it spatially — followed by scalar - // features: food direction (2) + L1 distance (1), heading one-hot (4), normalized length (1), and the - // flood-fill count of free cells reachable through each of the 4 neighbours (4). The flood-fill is the - // anti-self-trap signal a fixed window can't give: it tells a long snake which moves keep open space reachable. - public const int PatchRadius = 4; - public const int PatchSide = 2 * PatchRadius + 1; // 9 - public const int PatchChannels = 2; // 0 = obstacle, 1 = food - public const int PatchSize = PatchSide * PatchSide * PatchChannels; // 162 - public const int ScalarFeatures = 15; // foodΔ(2)+dist(1)+heading(4)+len(1)+free(4)+tailΔ(2)+tailDist(1) - public const int ObservationSize = PatchSize + ScalarFeatures; // 177 + // Grid-size-invariant observation (PLAN M27). 8-direction ray-cast vision (CodeBullet-style): per ray, whether + // food lies on it, 1/distance to the nearest body segment, and 1/distance to the wall — long-range awareness a + // fixed window can't give. Plus the flood-fill of open space reachable through each of the 4 neighbours (the + // anti-self-trap signal), food & tail bearing + distance, heading one-hot, and normalized length. Distances are + // 1/cells so the whole vector is size-invariant — a net trained on one grid transfers to another. + public const int RayDirections = 8; + public const int RayChannels = 3; // food-on-ray, 1/dist-to-body, 1/dist-to-wall + public const int RayFeatures = RayDirections * RayChannels; // 24 + public const int ObservationSize = RayFeatures + 4 + 3 + 3 + 4 + 1; // 39: rays + flood(4) + food(3) + tail(3) + heading(4) + len(1) public const float FoodReward = 1f; public const float StepPenalty = -0.01f; @@ -46,6 +44,10 @@ public sealed class SnakeEnv : IEnvironment, IActionMaskProvider, // Action = absolute direction; (dRow, dCol) per action index. private static readonly (int Dr, int Dc)[] Deltas = [(-1, 0), (1, 0), (0, -1), (0, 1)]; // Up, Down, Left, Right + // The 8 ray-cast directions for the vision features (4 cardinal + 4 diagonal), clockwise from North. + private static readonly (int Dr, int Dc)[] RayDeltas = + [(-1, 0), (-1, 1), (0, 1), (1, 1), (1, 0), (1, -1), (0, -1), (-1, -1)]; + private Xoshiro256StarStar _rng = new(0); private readonly LinkedList _body = new(); // head = First, tail = Last (cell indices) private readonly HashSet _occupied = []; @@ -221,55 +223,47 @@ private float[] Observation() int tail = _body.Last!.Value; int fr = _food / Size, fc = _food % Size; - // A cell is lethal to enter (an "obstacle") if it is off the board or holds a body segment that will NOT - // vacate this step. The tail cell vacates when the snake doesn't eat, so it is passable. - bool Obstacle(int r, int c) - { - if (r < 0 || r >= Size || c < 0 || c >= Size) return true; - int cell = r * Size + c; - return _occupied.Contains(cell) && cell != tail; - } - var obs = new float[ObservationSize]; + int s = 0; - // ── channels 0..1: a (2R+1)×(2R+1) egocentric patch centred on the head ── - // Layout is channel-major (all obstacle cells, then all food cells), row-major within each channel, so a - // CNN can reshape it to [PatchChannels, PatchSide, PatchSide]. Cells off the board read as obstacle. - const int plane = PatchSide * PatchSide; - for (int dr = -PatchRadius, i = 0; dr <= PatchRadius; dr++) - for (int dc = -PatchRadius; dc <= PatchRadius; dc++, i++) + // ── 8-direction ray-cast vision: per ray, food-on-line, 1/dist to nearest body, 1/dist to wall ── + foreach (var (dr, dc) in RayDeltas) + { + int r = hr + dr, c = hc + dc, dist = 1; + float food = 0f, body = 0f; + while (r >= 0 && r < Size && c >= 0 && c < Size) { - int r = hr + dr, c = hc + dc; - if ((dr != 0 || dc != 0) && Obstacle(r, c)) obs[i] = 1f; // centre is the head itself, not an obstacle - if (r == fr && c == fc) obs[plane + i] = 1f; // food channel + int cell = r * Size + c; + if (food == 0f && cell == _food) food = 1f; + if (body == 0f && _occupied.Contains(cell)) body = 1f / dist; // nearest body segment along the ray + r += dr; c += dc; dist++; } + obs[s++] = food; + obs[s++] = body; + obs[s++] = 1f / dist; // wall: closer wall ⇒ larger + } - // ── scalar features (start after the two patch planes) ── - int s = PatchSize; - obs[s++] = (fc - hc) / (float)Size; // food Δcol (signed, normalized) - obs[s++] = (fr - hr) / (float)Size; // food Δrow (signed, normalized) - obs[s++] = (Math.Abs(fr - hr) + Math.Abs(fc - hc)) / (2f * Size); // L1 distance to food - obs[s++] = _heading == 0 ? 1f : 0f; - obs[s++] = _heading == 1 ? 1f : 0f; - obs[s++] = _heading == 2 ? 1f : 0f; - obs[s++] = _heading == 3 ? 1f : 0f; - obs[s++] = _body.Count / (float)Cells; // normalized length - - // Flood-fill the open space reachable through each neighbour of the head (normalized by board area). This - // is the key anti-trap signal: a move into a region that can only reach a few cells is about to seal the - // snake in, even when the immediate cell looks safe. - for (int a = 0; a < 4; a++) - { - var (dr, dc) = Deltas[a]; + // ── flood-fill of open space reachable through each neighbour (the anti-self-trap signal) ── + foreach (var (dr, dc) in Deltas) obs[s++] = ReachableFreeSpace(hr + dr, hc + dc, tail) / (float)Cells; - } - // Direction + distance to the tail: chasing the tail is the canonical way a long snake stays alive (the - // cell the tail vacates is always safe to follow), so giving the agent the tail's bearing helps it learn it. + // ── food bearing + L1 distance (precise, unlike the binary ray-food) ── + obs[s++] = (fc - hc) / (float)Size; + obs[s++] = (fr - hr) / (float)Size; + obs[s++] = (Math.Abs(fr - hr) + Math.Abs(fc - hc)) / (2f * Size); + + // ── tail bearing + distance: chasing the tail (whose cell always vacates) is the canonical survival move ── int tr = tail / Size, tc = tail % Size; obs[s++] = (tc - hc) / (float)Size; obs[s++] = (tr - hr) / (float)Size; obs[s++] = (Math.Abs(tr - hr) + Math.Abs(tc - hc)) / (2f * Size); + + // ── heading one-hot + normalized length ── + obs[s++] = _heading == 0 ? 1f : 0f; + obs[s++] = _heading == 1 ? 1f : 0f; + obs[s++] = _heading == 2 ? 1f : 0f; + obs[s++] = _heading == 3 ? 1f : 0f; + obs[s++] = _body.Count / (float)Cells; return obs; } diff --git a/tests/MintPlayer.AI.ReinforcementLearning.Tests/SnakeEnvTests.cs b/tests/MintPlayer.AI.ReinforcementLearning.Tests/SnakeEnvTests.cs index 85c8f22..86eda76 100644 --- a/tests/MintPlayer.AI.ReinforcementLearning.Tests/SnakeEnvTests.cs +++ b/tests/MintPlayer.AI.ReinforcementLearning.Tests/SnakeEnvTests.cs @@ -8,29 +8,31 @@ public class SnakeEnvTests private static int Head(SnakeEnv e) => (e.Size / 2) * e.Size + (e.Size / 2); [Fact] - public void Reset_StartsLength3_HeadCentred_EgocentricObservation() + public void Reset_StartsLength3_HeadCentred_RayObservation() { var env = new SnakeEnv(); var (obs, _) = env.Reset(1); - Assert.Equal(SnakeEnv.ObservationSize, obs.Length); // patch (2 planes) + scalars + Assert.Equal(SnakeEnv.ObservationSize, obs.Length); // rays + flood + food + tail + heading + length Assert.Equal(3, env.Length); Assert.Equal(Head(env), env.Body.First()); - // The patch centre (the head's own cell) is never an obstacle. - int side = SnakeEnv.PatchSide, centre = (side / 2) * side + (side / 2); - Assert.Equal(0f, obs[centre]); + // Every ray's wall channel (the 3rd of each triple) is a positive 1/distance — the head always sees walls. + for (int d = 0; d < SnakeEnv.RayDirections; d++) + Assert.InRange(obs[d * SnakeEnv.RayChannels + 2], 1f / env.Size, 1f); - // Heading one-hot lives in the scalar block: exactly one bit set, and it's Right. - var heading = obs[(SnakeEnv.PatchSize + 3)..(SnakeEnv.PatchSize + 7)]; + // Heading one-hot (after rays + flood(4) + food(3) + tail(3)) = exactly one bit, Right (index 3). + int headingStart = SnakeEnv.RayFeatures + 4 + 3 + 3; + var heading = obs[headingStart..(headingStart + 4)]; Assert.Equal(1, heading.Count(v => v == 1f)); - Assert.Equal(1f, obs[SnakeEnv.PatchSize + 6]); // Right + Assert.Equal(1f, heading[3]); // Right - // Food Δ (signed, normalized) matches the food's position relative to the head. + // Food Δ (signed, normalized) sits right after the flood-fill block and matches the food's position. + int foodStart = SnakeEnv.RayFeatures + 4; int hr = Head(env) / env.Size, hc = Head(env) % env.Size; int fr = env.Food / env.Size, fc = env.Food % env.Size; - Assert.Equal((fc - hc) / (float)env.Size, obs[SnakeEnv.PatchSize + 0]); - Assert.Equal((fr - hr) / (float)env.Size, obs[SnakeEnv.PatchSize + 1]); + Assert.Equal((fc - hc) / (float)env.Size, obs[foodStart + 0]); + Assert.Equal((fr - hr) / (float)env.Size, obs[foodStart + 1]); } [Fact] From ad57b6e89014a0f12f77f496e8cef872f1dee3d8 Mon Sep 17 00:00:00 2001 From: PieterjanDeClippel Date: Mon, 22 Jun 2026 10:04:14 +0200 Subject: [PATCH 3/5] =?UTF-8?q?docs:=20M27=20Snake=20=E2=80=94=20full=20le?= =?UTF-8?q?dger,=20hotfix,=20lean=20ray-obs=20pivot,=20resume=20plan?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/PLAN.md | 75 ++++++++++++++++++++++++++++++++++------------------ docs/PRD.md | 8 ++++++ 2 files changed, 58 insertions(+), 25 deletions(-) diff --git a/docs/PLAN.md b/docs/PLAN.md index 67906f3..c206c61 100644 --- a/docs/PLAN.md +++ b/docs/PLAN.md @@ -919,37 +919,62 @@ doesn't train at all, so the campaigns stay `internal` to the Lab (the contract **Net −284 lines.** If a contributor needs to (re)produce a checkpoint, that's a dev-side Lab/Console run + an LFS commit, documented in `ADDING_A_GAME.md`. -## M27 — Snake: spatial observation + anti-self-trap shield *(branch `snake-spatial-obs`, stacked on M25/M26)* ⏳ - -**Problem.** "Improve the Snake AI" (it expects >100 food). The shipped net scored **~21 food on 12×12** and was -stuck there structurally, not for lack of training: (1) the observation was **12 local features** (danger one cell -out + a food compass) — the agent was *blind to its own body*, so a long snake inevitably trapped itself; (2) -`MaxEpisodeSteps = 1000` was a flat truncation that *hard-capped* how much food was even reachable per episode. - -**Fixes (this milestone).** A grid-size-invariant **egocentric observation** — a 9×9 obstacle+food patch centred on -the head + food/tail direction & distance + heading + length + a **flood-fill of reachable open space per move** -(the anti-trap signal a fixed window can't give) — and a **starvation episode limit** (`StarveLimit ≈ 2·cells` with no -food) replacing the flat cap. Plus an opt-in **`safeMask`**: the action mask also forbids moves that flood-fill into -a region too small for the body (a reactive 1-ply shield), used in training and the live web demo. - -**Experiment ledger (food@12, 50-ep eval unless noted; all >> the old 21):** +## M27 — Snake: stronger AI via better inputs (not more training) ⏳ IN PROGRESS + +**Goal.** "Improve the Snake AI" — the user wants it toward **>100 food on the 12×12 demo grid** (max 141). The +original shipped net plateaued at **~21 food**, and that was *structural*, not under-training. + +**Root causes found + fixed (all SHIPPED to master via #9, hotfixed by #10):** +1. **Blind observation** — was 12 local features (danger one cell out + food compass); the agent couldn't see its + own body, so a long snake trapped itself. → reworked the observation (below). +2. **Flat `MaxEpisodeSteps=1000` truncation** hard-capped reachable food. → replaced with a board-scaled + **starvation limit** (`StarveLimit ≈ 2·cells` without food) + a large absolute ceiling. +3. **Self-trapping deaths** capped the score. → opt-in **`safeMask`**: the action mask also forbids moves that + flood-fill into a region smaller than the body (reactive 1-ply shield), used in training + the live demo. +4. Reusable training mechanics added to the harness: `DqnTrainer.warmStart` (continue a deployable net-only + checkpoint), **save-best** (ship the best-eval net, not the noisy last), trainer internal-eval switched off in + the campaign (≈2× throughput), Lab knobs `--hidden/--gamma/--step-penalty/--safe-mask/--explore`. + +**Experiment ledger (food@12 on 12×12, 50-ep eval unless noted; all ≫ the old 21):** | config | food | |---|---| -| egocentric obs, `[128,128]` | 39.9 | +| egocentric 9×9 patch obs, `[128,128]` | 39.9 | | `[256,256]` (capacity) | 39.4 — capacity is *not* the bottleneck | | + tail feature + γ0.995 | 43.1 | | + step-penalty 0 + γ0.997 | 42.6 | | same net + post-hoc shield | 46.6 | -| **shield-*trained*** (γ0.997, pen 0) | **52.3 peak / 50.7 @200-ep** | - -Shipping the shield-trained net (**~50 food, ~2.4× the old 21**): `models/snake.dqn.ckpt` overwritten, -`SnakeController` live demo uses `safeMask`. **In progress:** a small→large **grid curriculum** (7×7 → 9×9 → 12×12, -warm-started) — train space-management where trapping is unavoidable-to-confront, then transfer up; may beat ~50. - -**Honest ceiling.** ~50 is the **pure-learned (DQN + 1-ply shield)** plateau across capacity/feature/horizon/reward -experiments. Reliable **100** (filling ~70% of the board) needs **multi-ply planning/search** — the EfficientCube -pattern (learned net guiding a beam/lookahead, already in this repo) or a Hamiltonian planner — out of scope for -this learning-only milestone, recommended as the follow-up. +| **shield-trained** (γ0.997, pen 0) — **SHIPPED in #9** | **52.3 peak / 50.7 @200-ep** | +| 7×7 curriculum (old patch obs, stopped early) | climbed to 26.5/46 on 7×7, abandoned for the ray pivot | + +**Shipped (master):** `models/snake.dqn.ckpt` = the shield-trained net (**~50 food, ~2.4× the old 21**); web live +demo uses `safeMask`. **#10 hotfix:** the live stream broke ("stream closed") because the persistent `/data` volume +kept the OLD 12-dim net vs the new obs — fixed by (a) seed now OVERWRITES shipped checkpoints (load-only web ⇒ +`models/` is source of truth; this also means model updates finally propagate to the volume) and (b) +`SnakeModelService` rejects a net whose `InputSize != SnakeEnv.ObservationSize` (clean 503, not a crash). Redeploy +needed for prod; or delete `/data/snake.dqn.ckpt` + restart. + +**CURRENT pivot — lean ray-cast observation (branch `snake-ray-obs`, off master + #10; NOT yet trained/shipped).** +Decision (with the user, after studying their cloned `C:\Repos\SnakeFusion` = CodeBullet's GA snake): the patch was +**myopic** (±4 cells). Replaced the 9×9 patch (162 of 177 inputs, mostly zeros) with **8-direction ray-cast vision** +(CodeBullet-style): per ray, food-on-line + 1/dist-to-body + 1/dist-to-wall = 24 inputs of *long-range* awareness. +Kept the flood-fill (4, anti-trap) + food/tail bearing+dist (6) + heading (4) + length (1). **ObservationSize +177 → 39**, size-invariant. This is the user's preferred path: better *inputs* for a genuinely *learned* net (the +user explicitly rejected the Hamiltonian/brute-force route). Env tests green (8/8); not yet trained. + +**NEXT (resume here after compaction):** +1. Rebuild Release (the running build is stale re: the lean obs) and launch a fresh 12×12 lean-obs training run: + `--game snake --train-grid 12 --eval-grid 12 --hidden 256,256 --gamma 0.997 --step-penalty 0 --safe-mask` + (fresh scratch under `…/scratchpad/`). Compare food@12 to the shipped **50.7**. +2. If it beats ~50: optionally layer the 7×7→9×9→12×12 curriculum on the ray obs (warm-start each stage). Then + ship: overwrite `models/snake.dqn.ckpt`, commit the lean `SnakeEnv` + net together (ObservationSize changed, so + the `#10` `InputSize` guard requires a matching net), full suite, PR `snake-ray-obs` → master. +3. Branch map: `master` = #9+#10 (shipped ~50 net). `snake-ray-obs` (current) = master + eval-speedup + lean obs. + `snake-stream-hotfix` already merged (#10). Old `snake-curriculum`/`snake-spatial-obs` deleted/merged. + +**Honest ceiling.** ~50 was the pure-learned (DQN + 1-ply shield + patch) plateau. The ray obs may push higher +(long-range awareness is the missing piece), but a reactive learned policy likely still tops out short of a clean +**100** — that needs **multi-ply planning/search** (the EfficientCube pattern: a learned net guiding a beam/lookahead, +already in this repo). Offered as the follow-up if the ray obs plateaus; the user prefers staying with learning for now. ## Testing strategy (cross-cutting, from research) diff --git a/docs/PRD.md b/docs/PRD.md index 01d9974..529439a 100644 --- a/docs/PRD.md +++ b/docs/PRD.md @@ -557,3 +557,11 @@ via `CampaignRunner`. Investigation showed the web's `EnsureModel` training was committed to `models/` (Git LFS) and seeded at startup, so it never ran in production. So M26 instead made the web **load-only**: training lives solely on the dev side (Lab campaigns / Console) and is committed; the web loads and serves. The campaigns stay `internal` to the Lab (no shared library needed — the web doesn't train). See PLAN M26. + +**Snake AI — stronger via better inputs (M27, in progress).** Goal: push the Snake demo toward >100 food on 12×12 +(was ~21, structurally capped by a blind 12-feature observation + a flat episode cap). Reworked the observation +(grid-size-invariant), added a starvation episode limit + an opt-in flood-fill **self-trap shield**, and shipped a +**~50-food net** (≈2.4×) to master (#9; live-stream hotfix #10). Now pivoting (user-preferred, learning-only) to a +**lean 8-direction ray-cast observation** (CodeBullet-style: food/body/wall per ray + flood-fill) for long-range +awareness, on branch `snake-ray-obs` — not yet trained. Full ledger, branch map, and resume steps in **PLAN M27** +(the authoritative resume point). Reliable 100 would need multi-ply planning/search (EfficientCube pattern) — deferred. From b65a4ab2665875fefbdd660ba8d64c6ad63bb958 Mon Sep 17 00:00:00 2001 From: PieterjanDeClippel Date: Mon, 22 Jun 2026 20:01:32 +0200 Subject: [PATCH 4/5] =?UTF-8?q?Snake=20(M27):=20net-guided=20multi-ply=20l?= =?UTF-8?q?ook-ahead=20=E2=80=94=20food@12=20~50=20=E2=86=92=20~78.6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The lean ray-cast obs trained to ~53 greedy (vs the shipped 50.7), confirming the reactive-DQN plateau is structural. So this takes the user's chosen route — the EfficientCube idea of searching on top of a learned value function, applied to Snake. SnakeSearchAgent (Environments/Snake): Snake is deterministic between food (the food RNG lives in the env's saved state), so look-ahead is exact. It simulates every legal line on a private env clone (SaveState/RestoreState) to ~20 plies, scores each leaf by flood-fill survivability (new SnakeEnv.FreeSpaceAhead, a hard self-trap penalty) + food eaten + the trained net's value + tiebreaks, and plays the best line's first move. The net is the leaf evaluator (net-guided, not brute force). Sweep findings (food@12, 12×12; greedy ≈ 50): depth is the lever (d20 sweet spot; deeper gets misranked under beam pruning), wider beam hurts, space-weight helps then saturates, net weight is marginal (search carries it; forward skipped when weight 0), and the starvation window is not the cap — self-traps beyond the horizon are. Shipped config (net-guided d20, beam 32, space 50): food@12 ≈ 78.6 (3.7× the original 21). - SnakeEnv: + FreeSpaceAhead (look-ahead survivability), + configurable starveLimitCells - SnakeSearchAgent / SnakeSearchOptions; SnakeSearchAgentTests (beats greedy, deterministic, legal) - Lab eval: --search/--depth/--beam/--w-*/--starve - SnakeModelService exposes the net; SnakeController.Live drives the demo via the planner - models/snake.dqn.ckpt -> lean 39-dim net (old 177-dim is incompatible; net + SnakeEnv ship together) Reaching a clean 100 is the open stretch — it needs a tail-reachability survival invariant / Hamiltonian endgame (not yet built). See PLAN M27. Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/PLAN.md | 63 +++++--- docs/PRD.md | 17 +- models/snake.dqn.ckpt | 4 +- .../Snake/SnakeEnv.cs | 32 +++- .../Snake/SnakeSearchAgent.cs | 153 ++++++++++++++++++ src/RLDemo.Web/Controllers/SnakeController.cs | 15 +- src/RLDemo.Web/Services/SnakeModelService.cs | 14 ++ .../SnakeSearchAgentTests.cs | 74 +++++++++ .../SnakeDqnCampaign.cs | 18 ++- .../SnakeLab.cs | 18 ++- 10 files changed, 362 insertions(+), 46 deletions(-) create mode 100644 src/MintPlayer.AI.ReinforcementLearning.Environments/Snake/SnakeSearchAgent.cs create mode 100644 tests/MintPlayer.AI.ReinforcementLearning.Tests/SnakeSearchAgentTests.cs diff --git a/docs/PLAN.md b/docs/PLAN.md index c206c61..ac6babd 100644 --- a/docs/PLAN.md +++ b/docs/PLAN.md @@ -919,7 +919,7 @@ doesn't train at all, so the campaigns stay `internal` to the Lab (the contract **Net −284 lines.** If a contributor needs to (re)produce a checkpoint, that's a dev-side Lab/Console run + an LFS commit, documented in `ADDING_A_GAME.md`. -## M27 — Snake: stronger AI via better inputs (not more training) ⏳ IN PROGRESS +## M27 — Snake: stronger AI via better inputs + net-guided search ⏳ SEARCH SHIPPING (100 stretch open) **Goal.** "Improve the Snake AI" — the user wants it toward **>100 food on the 12×12 demo grid** (max 141). The original shipped net plateaued at **~21 food**, and that was *structural*, not under-training. @@ -953,28 +953,45 @@ kept the OLD 12-dim net vs the new obs — fixed by (a) seed now OVERWRITES ship `SnakeModelService` rejects a net whose `InputSize != SnakeEnv.ObservationSize` (clean 503, not a crash). Redeploy needed for prod; or delete `/data/snake.dqn.ckpt` + restart. -**CURRENT pivot — lean ray-cast observation (branch `snake-ray-obs`, off master + #10; NOT yet trained/shipped).** -Decision (with the user, after studying their cloned `C:\Repos\SnakeFusion` = CodeBullet's GA snake): the patch was -**myopic** (±4 cells). Replaced the 9×9 patch (162 of 177 inputs, mostly zeros) with **8-direction ray-cast vision** -(CodeBullet-style): per ray, food-on-line + 1/dist-to-body + 1/dist-to-wall = 24 inputs of *long-range* awareness. -Kept the flood-fill (4, anti-trap) + food/tail bearing+dist (6) + heading (4) + length (1). **ObservationSize -177 → 39**, size-invariant. This is the user's preferred path: better *inputs* for a genuinely *learned* net (the -user explicitly rejected the Hamiltonian/brute-force route). Env tests green (8/8); not yet trained. - -**NEXT (resume here after compaction):** -1. Rebuild Release (the running build is stale re: the lean obs) and launch a fresh 12×12 lean-obs training run: - `--game snake --train-grid 12 --eval-grid 12 --hidden 256,256 --gamma 0.997 --step-penalty 0 --safe-mask` - (fresh scratch under `…/scratchpad/`). Compare food@12 to the shipped **50.7**. -2. If it beats ~50: optionally layer the 7×7→9×9→12×12 curriculum on the ray obs (warm-start each stage). Then - ship: overwrite `models/snake.dqn.ckpt`, commit the lean `SnakeEnv` + net together (ObservationSize changed, so - the `#10` `InputSize` guard requires a matching net), full suite, PR `snake-ray-obs` → master. -3. Branch map: `master` = #9+#10 (shipped ~50 net). `snake-ray-obs` (current) = master + eval-speedup + lean obs. - `snake-stream-hotfix` already merged (#10). Old `snake-curriculum`/`snake-spatial-obs` deleted/merged. - -**Honest ceiling.** ~50 was the pure-learned (DQN + 1-ply shield + patch) plateau. The ray obs may push higher -(long-range awareness is the missing piece), but a reactive learned policy likely still tops out short of a clean -**100** — that needs **multi-ply planning/search** (the EfficientCube pattern: a learned net guiding a beam/lookahead, -already in this repo). Offered as the follow-up if the ray obs plateaus; the user prefers staying with learning for now. +**Lean ray-cast observation (branch `snake-ray-obs`, off master + #10) — DONE.** The 9×9 patch was **myopic** +(±4 cells, 162 of 177 inputs mostly zeros). Replaced with **8-direction ray-cast vision** (CodeBullet-style, from +the user's cloned `C:\Repos\SnakeFusion`): per ray, food-on-line + 1/dist-to-body + 1/dist-to-wall = 24 inputs of +*long-range* awareness, + flood-fill (4) + food/tail bearing&dist (6) + heading (4) + length (1). **ObservationSize +177 → 39**, size-invariant. Trained fresh 12×12 (`--hidden 256,256 --gamma 0.997 --step-penalty 0 --safe-mask`): +peaked **food@12 ≈ 53 greedy** (vs the shipped 50.7) at ~240k steps, converging far faster than the patch obs. But +~53 confirmed the **reactive-DQN plateau is structural** — better inputs alone don't reach 100. + +**Multi-ply search (the user's chosen path; the EfficientCube pattern applied to Snake) — DONE, shipping.** +Snake's dynamics are deterministic between food (the food RNG is in the env's saved state), so look-ahead is *exact*. +New `SnakeSearchAgent` (Environments/Snake): receding-horizon beam search that simulates every legal line on a +private env clone (`SaveState`/`RestoreState`), scores each leaf by **flood-fill survivability** (the new +`SnakeEnv.FreeSpaceAhead`, a hard self-trap penalty) + food eaten + the trained net's value + tiebreaks, and plays +the first move of the best line. Net-guided (the net is the leaf evaluator — the AlphaZero/EfficientCube idea), so +it honors the user's "really trained AI, not brute force" preference. Lab eval gained `--search --depth --beam +--w-* --starve`; `SnakeModelService` exposes the net; `SnakeController.Live` drives the demo via the planner. + +**Search sweep (food@12 on 12×12; greedy baseline ≈ 50):** +| lever | finding | +|---|---| +| depth | **the lever** — 65 (d6) → 71 (d10) → **84/79 (d20)**; d28/d36 no reliable gain (beam misranks deep lines). d20 = sweet spot | +| beam width | wider **hurts** (d6: 65→59 at b64) — keeps greedy food-grabs that trap later | +| space weight | helps, saturates (d6 65→69; d16 72→79; flat by ~50–100) | +| net weight | **marginal** — net=0 ties net-on (search carries it); net forward skipped when weight 0 | +| starvation window | **not the cap** — 2→4→8 saturates at ~80; the cap is self-traps, not truncation | +| **SHIPPED: net-guided d20, beam 32, space 50** | **food@12 ≈ 78.6** (deployable net, 20-ep) — **greedy 50 → 78.6, old shipped 21 → 78.6 (3.7×)** | + +**Shipped artifacts (branch `snake-ray-obs`, local commit — PR pending user OK):** lean `SnakeEnv` (39-dim obs + +`FreeSpaceAhead` + configurable `starveLimitCells`), `SnakeSearchAgent`/`SnakeSearchOptions`, Lab search flags, +`SnakeModelService.Net`, `SnakeController` net-guided look-ahead, `SnakeSearchAgentTests` (3) + updated env test; +`models/snake.dqn.ckpt` replaced with the lean-obs net (39-dim; the old 177-dim net is incompatible with this +branch, so net + `SnakeEnv` must ship together for the #10 `InputSize` guard). + +**Honest ceiling.** ~50 was the reactive-DQN plateau; net-guided search lifts it to **~78** by guaranteeing the +snake doesn't walk into a box within ~20 plies. The residual gap to a clean **100** is occasional self-traps that +form *beyond* the horizon — the principled fix is a **tail-reachability** survival invariant (guarantee the head can +always reach its own tail → it can follow the tail indefinitely) and/or explicit Hamiltonian-style endgame play. +Offered as the next step; not yet built. Branch map: `master` = #9+#10. `snake-ray-obs` (current) = master + +eval-speedup + lean obs + search. ## Testing strategy (cross-cutting, from research) diff --git a/docs/PRD.md b/docs/PRD.md index 529439a..c1122e4 100644 --- a/docs/PRD.md +++ b/docs/PRD.md @@ -558,10 +558,13 @@ committed to `models/` (Git LFS) and seeded at startup, so it never ran in produ **load-only**: training lives solely on the dev side (Lab campaigns / Console) and is committed; the web loads and serves. The campaigns stay `internal` to the Lab (no shared library needed — the web doesn't train). See PLAN M26. -**Snake AI — stronger via better inputs (M27, in progress).** Goal: push the Snake demo toward >100 food on 12×12 -(was ~21, structurally capped by a blind 12-feature observation + a flat episode cap). Reworked the observation -(grid-size-invariant), added a starvation episode limit + an opt-in flood-fill **self-trap shield**, and shipped a -**~50-food net** (≈2.4×) to master (#9; live-stream hotfix #10). Now pivoting (user-preferred, learning-only) to a -**lean 8-direction ray-cast observation** (CodeBullet-style: food/body/wall per ray + flood-fill) for long-range -awareness, on branch `snake-ray-obs` — not yet trained. Full ledger, branch map, and resume steps in **PLAN M27** -(the authoritative resume point). Reliable 100 would need multi-ply planning/search (EfficientCube pattern) — deferred. +**Snake AI — stronger via better inputs + net-guided search (M27).** Goal: push the Snake demo toward >100 food on +12×12 (was ~21, structurally capped by a blind observation + a flat episode cap). Path: (1) reworked the observation +to a grid-size-invariant **8-direction ray-cast** (CodeBullet-style, long-range), added a starvation episode limit + +an opt-in flood-fill self-trap shield — a learned reactive DQN tops out at **~50 food**; (2) on the user's chosen +route, added **net-guided multi-ply look-ahead** (`SnakeSearchAgent`, the EfficientCube "search on a learned value +function" idea): exact deterministic simulation to ~20 plies, leaf-scored by flood-fill survivability + the trained +net's value, plays the best line. **Result: food@12 ≈ 78.6** (3.7× the old 21, ~1.6× the reactive net). The live +demo (`SnakeController`) now plays via the planner. The residual gap to a clean 100 is self-traps forming beyond the +horizon; the next step is a **tail-reachability** survival invariant / Hamiltonian endgame. Branch `snake-ray-obs` +(lean obs + search; `models/snake.dqn.ckpt` swapped to the 39-dim net). Authoritative detail in **PLAN M27**. diff --git a/models/snake.dqn.ckpt b/models/snake.dqn.ckpt index a496e94..1b02372 100644 --- a/models/snake.dqn.ckpt +++ b/models/snake.dqn.ckpt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:c497b30f8b51556f0691f7423059467f05bf92a47905e96818fed618857e40b8 -size 450650 +oid sha256:9e59bee886b955d4c9059018ff5b3a9e52be6d4d813e84cc662a4b97242e2a43 +size 309338 diff --git a/src/MintPlayer.AI.ReinforcementLearning.Environments/Snake/SnakeEnv.cs b/src/MintPlayer.AI.ReinforcementLearning.Environments/Snake/SnakeEnv.cs index 44c2bc8..d1e250d 100644 --- a/src/MintPlayer.AI.ReinforcementLearning.Environments/Snake/SnakeEnv.cs +++ b/src/MintPlayer.AI.ReinforcementLearning.Environments/Snake/SnakeEnv.cs @@ -37,8 +37,10 @@ public sealed class SnakeEnv : IEnvironment, IActionMaskProvider, // Episode limits scale with the board so high-score games are possible: the snake dies only after this many // steps WITHOUT eating (a starvation timeout, not a flat cap that would hard-limit food); a generous absolute - // ceiling guards a true infinite loop that keeps eating just often enough to never starve. - private int StarveLimit => 2 * Cells; + // ceiling guards a true infinite loop that keeps eating just often enough to never starve. The starvation + // window is a multiple of the board (default 2): training keeps it tight to discourage aimless wandering, but + // a long snake under look-ahead inference legitimately needs long, food-less detours, so inference widens it. + private int StarveLimit => _starveLimitCells * Cells; private int MaxEpisodeSteps => 100 * Cells; // Action = absolute direction; (dRow, dCol) per action index. @@ -59,6 +61,7 @@ private static readonly (int Dr, int Dc)[] RayDeltas = private bool _done = true; private readonly float _stepPenalty; private readonly bool _safeMask; + private readonly int _starveLimitCells; /// /// Per-non-eating-step reward (training only — inference is greedy and ignores reward). Defaults to @@ -70,14 +73,22 @@ private static readonly (int Dr, int Dc)[] RayDeltas = /// small to hold its body — a reactive flood-fill "shield" against self-trapping. Off by default (preserves /// the reversal-only mask). Can be enabled for both training and inference. /// - public SnakeEnv(int size = 12, float stepPenalty = StepPenalty, bool safeMask = false) + /// + /// Starvation window as a multiple of the board area: the episode truncates after this many steps without + /// eating. Default 2 keeps training episodes from wandering aimlessly; raise it for look-ahead inference, where + /// a long snake must legitimately take board-spanning, food-less detours to stay safe. + /// + public SnakeEnv(int size = 12, float stepPenalty = StepPenalty, bool safeMask = false, int starveLimitCells = 2) { if (size < 5) throw new ArgumentOutOfRangeException(nameof(size), "Grid must be at least 5×5 (the length-3 start needs room)."); + if (starveLimitCells < 1) + throw new ArgumentOutOfRangeException(nameof(starveLimitCells), "Starvation window must be at least one board."); Size = size; Cells = size * size; _stepPenalty = stepPenalty; _safeMask = safeMask; + _starveLimitCells = starveLimitCells; ObservationSpace = new BoxSpace(0f, 1f, ObservationSize); ActionSpace = new DiscreteSpace(ActionCount); } @@ -216,6 +227,21 @@ public bool[] CurrentActionMask() public float[] CurrentObservation() => Observation(); + /// + /// The largest open region the head can move into next: max over the four neighbours of the free cells reachable + /// from that neighbour (BFS, tail counts as free). A position where this is below is a + /// guaranteed self-trap — the snake can no longer fit its own body. Exposed for look-ahead search, which scores a + /// simulated leaf position by survivability (the signal a reactive value net is worst at). 0 means already boxed in. + /// + public int FreeSpaceAhead() + { + int head = _body.First!.Value, tail = _body.Last!.Value; + int hr = head / Size, hc = head % Size, best = 0; + foreach (var (dr, dc) in Deltas) + best = Math.Max(best, ReachableFreeSpace(hr + dr, hc + dc, tail)); + return best; + } + private float[] Observation() { int head = _body.First!.Value; diff --git a/src/MintPlayer.AI.ReinforcementLearning.Environments/Snake/SnakeSearchAgent.cs b/src/MintPlayer.AI.ReinforcementLearning.Environments/Snake/SnakeSearchAgent.cs new file mode 100644 index 0000000..0738aa4 --- /dev/null +++ b/src/MintPlayer.AI.ReinforcementLearning.Environments/Snake/SnakeSearchAgent.cs @@ -0,0 +1,153 @@ +using MintPlayer.AI.ReinforcementLearning.Core.Environments; +using MintPlayer.AI.ReinforcementLearning.Core.Nn; +using MintPlayer.AI.ReinforcementLearning.Core.Training; + +namespace MintPlayer.AI.ReinforcementLearning.Environments.Snake; + +/// Tunable weights for . The defaults make eating dominate, treat a +/// self-trap as nearly as bad as death, and use the net value only as a tiebreak for "which safe way to go". +public sealed record SnakeSearchOptions +{ + /// Plies to look ahead. Higher catches traps that form further out, at ~3× cost per extra ply. + public int MaxDepth { get; init; } = 6; + + /// Live (non-terminal) nodes carried to the next ply, best-scoring first. Caps the per-ply node count + /// so deep horizons stay cheap; raise for stronger (slower) play. + public int BeamWidth { get; init; } = 32; + + /// Reward per food eaten along a line — the dominant term; eating safely beats any non-eating line. + public float FoodWeight { get; init; } = 10_000f; + + /// Penalty for a leaf whose reachable space can no longer hold the body (a guaranteed future trap). + /// Smaller than the food reward so the snake still eats when it can do so without boxing itself in. + public float TrapPenalty { get; init; } = 50_000f; + + /// Weight on the trained net's position value (max Q at the leaf) — the learned "which way is good". + public float NetWeight { get; init; } = 500f; + + /// Weight on reachable free space (tiebreak: prefer keeping more room). + public float SpaceWeight { get; init; } = 5f; + + /// Pull toward food when no line eats within the horizon (tiebreak by L1 head→food distance). + public float FoodDistWeight { get; init; } = 1f; +} + +/// +/// Receding-horizon, net-guided look-ahead for Snake (PLAN M27 — the EfficientCube idea of searching on top of a +/// learned value function, applied to Snake). Snake's dynamics are deterministic between food (the food RNG lives in +/// the env's saved state), so the agent can simulate every legal action sequence to a fixed horizon exactly on +/// a private clone of the env. Each simulated leaf is scored by the trained 's value PLUS an +/// exact flood-fill survivability term — the net says "which way is good", the search guarantees "don't walk into a +/// box you can't get out of", which is the failure mode that caps a reactive learned policy. The agent plays the first +/// move of the best line and re-searches next tick. +/// +/// The agent reads the live env's state directly (via ), so it is driven by a +/// parameterless — the observation/mask passed to the usual selector contract are ignored. +/// +/// +public sealed class SnakeSearchAgent +{ + private readonly SnakeEnv _live; + private readonly SnakeEnv _sim; // private clone for branching; safeMask off → reversal-only legality + private readonly GreedyQAgent _value; // net leaf evaluator + private readonly SnakeSearchOptions _opt; + + public SnakeSearchAgent(SnakeEnv live, IValueNet net, SnakeSearchOptions? options = null) + { + _live = live; + _sim = new SnakeEnv(live.Size, safeMask: false); + _value = new GreedyQAgent(net, SnakeEnv.ActionCount); + _opt = options ?? new SnakeSearchOptions(); + } + + /// Plays the first move of the highest-scoring line found within the horizon. + public int Act() + { + var root = _live.SaveState(); + int rootFood = _live.FoodEaten; + + // The current beam: live (non-terminal) states to expand further, paired with the first move that led there. + var beam = new List<(byte[] State, int First)> { (root, -1) }; + double bestScore = double.NegativeInfinity; + int bestFirst = FallbackAction(root); + + for (int depth = 0; depth < _opt.MaxDepth && beam.Count > 0; depth++) + { + var next = new List<(byte[] State, int First, double Score)>(); + foreach (var (state, first) in beam) + { + _sim.RestoreState(state); + var mask = _sim.CurrentActionMask(); // reversal-only (this sim has safeMask off) + for (int a = 0; a < SnakeEnv.ActionCount; a++) + { + if (!mask[a]) continue; + _sim.RestoreState(state); + var step = _sim.Step(a); + int childFirst = first < 0 ? a : first; + double score = LeafScore(step, rootFood, depth + 1); + + if (score > bestScore) { bestScore = score; bestFirst = childFirst; } + + // A still-alive, non-final node can be expanded another ply. + if (!step.Done && depth + 1 < _opt.MaxDepth) + next.Add((_sim.SaveState(), childFirst, score)); + } + } + + // Prune to the best BeamWidth survivors before going deeper. + if (next.Count > _opt.BeamWidth) + { + next.Sort((x, y) => y.Score.CompareTo(x.Score)); + next.RemoveRange(_opt.BeamWidth, next.Count - _opt.BeamWidth); + } + beam = next.ConvertAll(n => (n.State, n.First)); + } + + return bestFirst; + } + + /// + /// Scores the simulated position just reached. A board-full win dominates everything; a + /// death is dominated by everything but ranked so a later death beats a sooner one (buys time for the tail to + /// clear); a survived position is rewarded for food eaten, penalized hard if it can no longer fit its body, and + /// nudged by net value, free space, and proximity to food. + /// + private double LeafScore(StepResult step, int rootFood, int depth) + { + bool boardFull = step.Terminated && _sim.Length == _sim.Cells; + if (boardFull) return 1e9; + if (step.Terminated) return -1e6 + depth * 1_000; // death — prefer to delay the unavoidable + + int foodGained = _sim.FoodEaten - rootFood; + int free = _sim.FreeSpaceAhead(); + double score = foodGained * _opt.FoodWeight; + if (free < _sim.Length) score -= _opt.TrapPenalty; + score += free * _opt.SpaceWeight; + + var obs = step.Observation; + // The forward pass dominates the per-node cost, so skip it entirely when the net term is disabled + // (empirically the net contributes almost nothing here — the flood-fill survival term carries the search). + if (_opt.NetWeight != 0f) + { + var q = _value.QValues(obs); + float bestQ = float.NegativeInfinity; + for (int a = 0; a < q.Length; a++) bestQ = MathF.Max(bestQ, q[a]); + score += bestQ * _opt.NetWeight; + } + + // L1 head→food distance is the last three obs slots' source; recover it from the food-distance feature + // (index RayFeatures+4+2), normalized by 2·Size. Pull toward food only as a final tiebreak. + float foodDist = obs[SnakeEnv.RayFeatures + 4 + 2] * (2f * _sim.Size); + score -= foodDist * _opt.FoodDistWeight; + return score; + } + + private int FallbackAction(byte[] state) + { + _sim.RestoreState(state); + var mask = _sim.CurrentActionMask(); + for (int a = 0; a < SnakeEnv.ActionCount; a++) + if (mask[a]) return a; + return 0; + } +} diff --git a/src/RLDemo.Web/Controllers/SnakeController.cs b/src/RLDemo.Web/Controllers/SnakeController.cs index 679db34..21bfb6f 100644 --- a/src/RLDemo.Web/Controllers/SnakeController.cs +++ b/src/RLDemo.Web/Controllers/SnakeController.cs @@ -13,6 +13,12 @@ public sealed class SnakeController(SnakeModelService model) : ControllerBase { private static int _seedCounter; + // The live demo plays via net-guided multi-ply look-ahead (PLAN M27): the trained net evaluates leaf positions + // while a 20-ply beam search guarantees the snake never walks into a box it can't escape — the failure that caps + // a reactive policy. Depth 20 is the empirical sweet spot on the 12×12 grid (≈84 food vs ≈50 for the raw greedy + // net; deeper gives no reliable gain and costs more per tick). Each move searches in a few ms — well inside the tick. + private static readonly SnakeSearchOptions LiveSearch = new() { MaxDepth = 20, BeamWidth = 32, SpaceWeight = 50f }; + [HttpGet("status")] public StatusResponse Status() { @@ -35,22 +41,21 @@ public async Task Live() Response.StatusCode = StatusCodes.Status400BadRequest; return; } - var agent = model.Agent; - if (agent is null) + var net = model.Net; + if (net is null) { Response.StatusCode = StatusCodes.Status503ServiceUnavailable; // still training — reject the upgrade return; } using var socket = await HttpContext.WebSockets.AcceptWebSocketAsync(); - // Shield on: the deployed net is trained with the anti-self-trap action mask (M27), so the live demo - // must use the same mask the policy expects (and it makes the snake visibly stronger regardless). var env = new SnakeEnv(safeMask: true); + var planner = new SnakeSearchAgent(env, net, LiveSearch); // net-guided look-ahead reads the env state directly ulong seed = (ulong)System.Threading.Interlocked.Increment(ref _seedCounter); // distinct game per connection await EpisodeStreamer.RunAsync( socket, env, - act: (obs, mask) => agent.Act(obs, mask, greedy: true), + act: (_, _) => planner.Act(), // obs/mask ignored — the planner searches from the live env's full state frame: (e, action, step) => new SnakeFrameDto( [.. e.Body], e.Food, action, step.Reward, step.Done, e.FoodEaten, e.Length), tickMs: 120, diff --git a/src/RLDemo.Web/Services/SnakeModelService.cs b/src/RLDemo.Web/Services/SnakeModelService.cs index 1d9e329..4b0eb99 100644 --- a/src/RLDemo.Web/Services/SnakeModelService.cs +++ b/src/RLDemo.Web/Services/SnakeModelService.cs @@ -1,4 +1,5 @@ using MintPlayer.AI.ReinforcementLearning.Core.Checkpoints; +using MintPlayer.AI.ReinforcementLearning.Core.Nn; using MintPlayer.AI.ReinforcementLearning.Core.Training; using MintPlayer.AI.ReinforcementLearning.Environments.Snake; @@ -16,6 +17,7 @@ public sealed class SnakeModelService(IModelStore store, ILoggerThe loaded value network (or null while not ready) — for building a per-connection + /// , the net-guided look-ahead that drives the live demo. + public IValueNet? Net + { + get + { + if (_net is null && Status == ModelStatus.Loading) TryLoadFromStore(); + return _net; + } + } + public bool TryLoadFromStore() { lock (_lock) @@ -48,6 +61,7 @@ public bool TryLoadFromStore() logger.LogError("Snake model is incompatible ({Stored} vs {Expected} obs) — refusing to load it.", net.InputSize, SnakeEnv.ObservationSize); return false; } + _net = net; _agent = new GreedyQAgent(net, SnakeEnv.ActionCount); Status = ModelStatus.Ready; logger.LogInformation("Loaded Snake model from the store."); diff --git a/tests/MintPlayer.AI.ReinforcementLearning.Tests/SnakeSearchAgentTests.cs b/tests/MintPlayer.AI.ReinforcementLearning.Tests/SnakeSearchAgentTests.cs new file mode 100644 index 0000000..4474402 --- /dev/null +++ b/tests/MintPlayer.AI.ReinforcementLearning.Tests/SnakeSearchAgentTests.cs @@ -0,0 +1,74 @@ +using MintPlayer.AI.ReinforcementLearning.Core.Nn; +using MintPlayer.AI.ReinforcementLearning.Core.Random; +using MintPlayer.AI.ReinforcementLearning.Core.Training; +using MintPlayer.AI.ReinforcementLearning.Environments.Snake; + +namespace MintPlayer.AI.ReinforcementLearning.Tests; + +public class SnakeSearchAgentTests +{ + private static DuelingQNet UntrainedNet() => + new(SnakeEnv.ObservationSize, [32, 32], SnakeEnv.ActionCount, new Xoshiro256StarStar(0)); + + // Plays a full episode driven by the search agent; returns food eaten. Every move Step accepts proves legality + // (Step throws on the reversal), so a completed episode is itself the "only legal moves" assertion. + private static int PlaySearch(SnakeEnv env, ulong seed, SnakeSearchOptions opts) + { + var agent = new SnakeSearchAgent(env, UntrainedNet(), opts); + env.Reset(seed); + while (true) + { + var step = env.Step(agent.Act()); + if (step.Done) break; + } + return env.FoodEaten; + } + + [Fact] + public void Search_BeatsReactiveGreedy_WithTheSameUntrainedNet() + { + // The net is random, so reactive greedy ≈ random play. Pure look-ahead survival (net weight 0) should eat + // far more on the same net — the whole point of search: it doesn't depend on a good value function to avoid + // walking into walls/itself. A small grid keeps the (long, healthy) search episode quick. + var net = UntrainedNet(); + var greedyEnv = new SnakeEnv(7); + var greedy = new GreedyQAgent(net, SnakeEnv.ActionCount); + var (obs, _) = greedyEnv.Reset(1); + while (true) + { + var step = greedyEnv.Step(greedy.Act(obs, greedyEnv.CurrentActionMask(), greedy: true)); + obs = step.Observation; + if (step.Done) break; + } + + int searchFood = PlaySearch(new SnakeEnv(7), 1, new SnakeSearchOptions { MaxDepth = 6, NetWeight = 0f }); + + Assert.True(searchFood > greedyEnv.FoodEaten, + $"search ate {searchFood}, reactive greedy ate {greedyEnv.FoodEaten} — search should dominate a random net"); + Assert.True(searchFood > 5, $"search only ate {searchFood} — survival look-ahead should sustain a long game"); + } + + [Fact] + public void Search_IsDeterministic_ForAGivenSeed() + { + var opts = new SnakeSearchOptions { MaxDepth = 6 }; + int a = PlaySearch(new SnakeEnv(7), 7, opts); + int b = PlaySearch(new SnakeEnv(7), 7, opts); + Assert.Equal(a, b); + } + + [Fact] + public void Search_NeverReturnsTheReversal() + { + // After moving right a few times, Left is the reversal; the planner must never pick it (Step would throw). + var env = new SnakeEnv(); + var agent = new SnakeSearchAgent(env, UntrainedNet(), new SnakeSearchOptions { MaxDepth = 6, NetWeight = 0f }); + env.Reset(3); + for (int i = 0; i < 50; i++) + { + int action = agent.Act(); + Assert.True(env.CurrentActionMask()[action], $"planner chose masked action {action}"); + if (env.Step(action).Done) break; + } + } +} diff --git a/tools/MintPlayer.AI.ReinforcementLearning.Lab/SnakeDqnCampaign.cs b/tools/MintPlayer.AI.ReinforcementLearning.Lab/SnakeDqnCampaign.cs index 1da12ac..1ce1977 100644 --- a/tools/MintPlayer.AI.ReinforcementLearning.Lab/SnakeDqnCampaign.cs +++ b/tools/MintPlayer.AI.ReinforcementLearning.Lab/SnakeDqnCampaign.cs @@ -15,14 +15,14 @@ /// net under `snake`/`dqn` (the id the web's SnakeModelService loads) plus the full resume state under /// `snake`/`dqn-state`. /// -internal sealed class SnakeDqnCampaign(ulong seed, int trainGrid, int evalGrid, int chunkSteps, long targetSteps, int evalEpisodes, float learningRate, float epsilonStart, int[] hidden, double gamma, float stepPenalty, bool safeMask) +internal sealed class SnakeDqnCampaign(ulong seed, int trainGrid, int evalGrid, int chunkSteps, long targetSteps, int evalEpisodes, float learningRate, float epsilonStart, int[] hidden, double gamma, float stepPenalty, bool safeMask, bool search = false, int searchDepth = 6, int searchBeam = 32, SnakeSearchOptions? searchOptions = null, int evalStarveCells = 2) : ITrainingCampaign { private const string NetId = "dqn"; // deployable DuelingQNet — the id the web loads private const string StateId = "dqn-state"; // full DqnTrainingState for lossless resume private readonly SnakeEnv _env = new(trainGrid, stepPenalty, safeMask); - private readonly SnakeEnv _evalEnv = new(evalGrid, stepPenalty, safeMask); + private readonly SnakeEnv _evalEnv = new(evalGrid, stepPenalty, safeMask, evalStarveCells); private readonly SeedSequence _seeds = new(seed); private DqnTrainingState? _state; @@ -152,10 +152,16 @@ public void Checkpoint(IModelStore store) public void Dispose() { } - /// Mean food + return over fixed-seed greedy episodes on the DEPLOYED grid (food is the gate metric). + /// + /// Mean food + return over fixed-seed episodes on the DEPLOYED grid (food is the gate metric). The action is + /// chosen greedily by the net, or — when search is on — by the net-guided multi-ply look-ahead + /// (): same net, but search amplifies it past the reactive plateau. + /// private (double Food, double Return) EvalNet(IValueNet net) { - var agent = new GreedyQAgent(net, SnakeEnv.ActionCount); + var greedy = new GreedyQAgent(net, SnakeEnv.ActionCount); + var opts = (searchOptions ?? new SnakeSearchOptions()) with { MaxDepth = searchDepth, BeamWidth = searchBeam }; + var planner = search ? new SnakeSearchAgent(_evalEnv, net, opts) : null; double totalFood = 0, totalReturn = 0; for (int e = 0; e < evalEpisodes; e++) { @@ -163,7 +169,9 @@ public void Dispose() { } double epReturn = 0; while (true) { - int action = agent.Act(obs, _evalEnv.CurrentActionMask(), greedy: true); + int action = planner is not null + ? planner.Act() + : greedy.Act(obs, _evalEnv.CurrentActionMask(), greedy: true); var step = _evalEnv.Step(action); epReturn += step.Reward; obs = step.Observation; diff --git a/tools/MintPlayer.AI.ReinforcementLearning.Lab/SnakeLab.cs b/tools/MintPlayer.AI.ReinforcementLearning.Lab/SnakeLab.cs index 5ba6c5b..10ec100 100644 --- a/tools/MintPlayer.AI.ReinforcementLearning.Lab/SnakeLab.cs +++ b/tools/MintPlayer.AI.ReinforcementLearning.Lab/SnakeLab.cs @@ -2,6 +2,7 @@ using Microsoft.Extensions.DependencyInjection; using MintPlayer.AI.ReinforcementLearning.Core.Checkpoints; using MintPlayer.AI.ReinforcementLearning.Core.Training; +using MintPlayer.AI.ReinforcementLearning.Environments.Snake; using MintPlayer.AI.ReinforcementLearning.Hosting; /// @@ -29,6 +30,12 @@ public static void Run(string[] args) float stepPenalty = -0.01f; // --step-penalty : per-step reward; ~0 removes the efficiency pressure that encourages safe starvation bool safeMask = false; // --safe-mask : forbid moves that flood-fill into a region too small for the body (anti-self-trap shield) bool evalOnly = false; + bool search = false; // --search : eval with net-guided multi-ply look-ahead (SnakeSearchAgent) instead of reactive greedy + int searchDepth = 6; // --depth : look-ahead plies + int searchBeam = 32; // --beam : live nodes carried per ply + int evalStarveCells = 2; // --starve : eval/inference starvation window as a multiple of the board (training stays at 2) + var sw = new SnakeSearchOptions(); // search leaf-score weights (overridable for tuning sweeps) + float wFood = sw.FoodWeight, wTrap = sw.TrapPenalty, wNet = sw.NetWeight, wSpace = sw.SpaceWeight; for (int i = 0; i < args.Length; i++) { if (args[i] == "--hours" && i + 1 < args.Length) hours = double.Parse(args[++i], CultureInfo.InvariantCulture); @@ -46,7 +53,16 @@ public static void Run(string[] args) else if (args[i] == "--step-penalty" && i + 1 < args.Length) stepPenalty = float.Parse(args[++i], CultureInfo.InvariantCulture); else if (args[i] == "--safe-mask") safeMask = true; else if (args[i] == "--eval-only") evalOnly = true; + else if (args[i] == "--search") search = true; + else if (args[i] == "--depth" && i + 1 < args.Length) searchDepth = int.Parse(args[++i]); + else if (args[i] == "--beam" && i + 1 < args.Length) searchBeam = int.Parse(args[++i]); + else if (args[i] == "--starve" && i + 1 < args.Length) evalStarveCells = int.Parse(args[++i]); + else if (args[i] == "--w-food" && i + 1 < args.Length) wFood = float.Parse(args[++i], CultureInfo.InvariantCulture); + else if (args[i] == "--w-trap" && i + 1 < args.Length) wTrap = float.Parse(args[++i], CultureInfo.InvariantCulture); + else if (args[i] == "--w-net" && i + 1 < args.Length) wNet = float.Parse(args[++i], CultureInfo.InvariantCulture); + else if (args[i] == "--w-space" && i + 1 < args.Length) wSpace = float.Parse(args[++i], CultureInfo.InvariantCulture); } + var searchOptions = new SnakeSearchOptions { FoodWeight = wFood, TrapPenalty = wTrap, NetWeight = wNet, SpaceWeight = wSpace }; // DI all the way: the model store, clock and CampaignRunner are resolved from the AIHost container. using var host = AIHost.CreateBuilder(dataDir).Build(); @@ -54,7 +70,7 @@ public static void Run(string[] args) var runner = host.Services.GetRequiredService(); string csvPath = Path.Combine(dataDir, "logs", "snake-dqn.csv"); runner.Run( - new SnakeDqnCampaign(seed, trainGrid, evalGrid, chunkSteps, targetSteps, evalEpisodes, learningRate, explore, hidden, gamma, stepPenalty, safeMask), + new SnakeDqnCampaign(seed, trainGrid, evalGrid, chunkSteps, targetSteps, evalEpisodes, learningRate, explore, hidden, gamma, stepPenalty, safeMask, search, searchDepth, searchBeam, searchOptions, evalStarveCells), store, new CampaignOptions { From 26b088b2ac8910899f833f58eabc947cf92309df Mon Sep 17 00:00:00 2001 From: PieterjanDeClippel Date: Tue, 23 Jun 2026 09:47:00 +0200 Subject: [PATCH 5/5] =?UTF-8?q?chore(Environments):=20bump=200.2.0=20?= =?UTF-8?q?=E2=86=92=200.3.0=20for=20the=20Snake=20search=20+=20lean-obs?= =?UTF-8?q?=20API?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New public surface on this branch since master: the 39-dim ray-cast observation, SnakeEnv.FreeSpaceAhead + configurable starveLimitCells, and SnakeSearchAgent / SnakeSearchOptions (net-guided multi-ply look-ahead). Co-Authored-By: Claude Opus 4.8 (1M context) --- .../MintPlayer.AI.ReinforcementLearning.Environments.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/MintPlayer.AI.ReinforcementLearning.Environments/MintPlayer.AI.ReinforcementLearning.Environments.csproj b/src/MintPlayer.AI.ReinforcementLearning.Environments/MintPlayer.AI.ReinforcementLearning.Environments.csproj index 390d40c..24505d1 100644 --- a/src/MintPlayer.AI.ReinforcementLearning.Environments/MintPlayer.AI.ReinforcementLearning.Environments.csproj +++ b/src/MintPlayer.AI.ReinforcementLearning.Environments/MintPlayer.AI.ReinforcementLearning.Environments.csproj @@ -12,7 +12,7 @@ - 0.2.0 + 0.3.0 Pieterjan De Clippel MintPlayer Ready-made environments for MintPlayer.AI.ReinforcementLearning: a bit-for-bit faithful CartPole-v1 port, GridWorld, FrozenLake, 2048 (with an n-tuple TD learner, ~84% win rate), and Rush Hour with a BFS oracle, puzzle generator, and an imitation-learning + policy-guided A* toolkit that solves official expert boards optimally.