diff --git a/src/MintPlayer.AI.ReinforcementLearning.Core/Checkpoints/README.md b/src/MintPlayer.AI.ReinforcementLearning.Core/Checkpoints/README.md new file mode 100644 index 0000000..e7aa499 --- /dev/null +++ b/src/MintPlayer.AI.ReinforcementLearning.Core/Checkpoints/README.md @@ -0,0 +1,48 @@ +# Checkpoints + +Versioned, resumable persistence for training runs and deployable models. Everything here sits +between the in-memory runtime objects (networks, optimizer, agents) and raw `Stream`s; the +`FileModelStore` then puts those streams on disk atomically. + +## When do I use which? + +One rule covers most cases: + +- **Resuming training** → save the *composite state* (`DqnTrainingState` for DQN, `TabularCheckpoint` + for tabular). It carries everything needed to continue *bitwise-identically* — nets, optimizer, + replay buffer, RNG streams. +- **Shipping a trained brain for inference** (the web app, an eval-only run) → save the *individual + net* checkpoint (`DuelingQNetCheckpoint` / `MlpCheckpoint` / `ResidualMlpCheckpoint`). Just shape + + weights; no optimizer or buffer. + +The campaigns do both: they persist the full resume state under `/-state` and, keep-best +gated, the deployable net under `/dqn` — the id the web `ModelService` loads. + +## The types + +| Checkpoint | Serializes | Standalone or embedded | Format | +|---|---|---|---| +| `MlpCheckpoint` / `ResidualMlpCheckpoint` / `DuelingQNetCheckpoint` | one network's architecture + weights | standalone deployable file **or** embedded via `QNetCheckpoint` | binary | +| `QNetCheckpoint` | any `IValueNet` (type-tagged: `Mlp` \| `DuelingQNet`) | embedded (inside a training state) | binary | +| `AdamCheckpoint` | optimizer hyperparameters + moment estimates — needs the net's parameters to reload | embedded only | binary | +| `ReplayBufferCheckpoint` | DQN replay transitions | embedded only | binary | +| `DqnTrainingState` | **the whole DQN run** — online+target nets + Adam + replay buffer + RNGs + env snapshot | top-level resume file | binary | +| `TabularCheckpoint` | a Q-table | standalone resume **and** deploy (small enough to be both) | JSON (human-inspectable, lossless) | + +`CheckpointFormat` holds the shared binary primitives (4-byte magic, kind + version header, float/int/ +bool arrays, RNG state). Every binary checkpoint above is built from it, so they share one on-disk +convention and one versioning scheme. + +## Persistence seam + +`IModelStore` / `FileModelStore` (`ModelStore.cs`) is the file layer: one *current* checkpoint per +`(environmentId, algorithmId)` pair, written to `/..ckpt` via a temp-file + rename so +a crash mid-save never corrupts the previous checkpoint. It deals only in raw `Stream`s and knows +nothing about net types — the `*Checkpoint` classes above own that knowledge. + +## Versioning + +Each checkpoint kind carries its own format version and stays backward-compatible: a newer reader +loads older files (e.g. `DqnTrainingState` v2 files load with a default noise RNG; `DuelingQNet` v1 +files load as plain non-noisy nets). Bump the version and branch on it in `Read`/`Load` when you add a +field — never reorder existing fields. diff --git a/src/RLDemo.Web/Services/CubeModelService.cs b/src/RLDemo.Web/Services/CubeModelService.cs index b34a636..94c103e 100644 --- a/src/RLDemo.Web/Services/CubeModelService.cs +++ b/src/RLDemo.Web/Services/CubeModelService.cs @@ -9,21 +9,45 @@ namespace RLDemo.Web.Services; /// /// Owns the Rubik's Cube nets: loads the shipped DQN baseline plus the imitation-policy, teacher-free DAVI value, /// and EfficientCube policy nets from the model store (all trained on a dev machine via the Lab campaigns and -/// committed to models/ via Git LFS — the web never trains, PRD §14). Holds the device-resident GPU -/// forwards for the deep solvers when a CUDA device is present. +/// committed to models/ via Git LFS — the web never trains, PRD §14). The three deep-solver nets hot-reload +/// during a training campaign via ; for the two GPU-scored nets the reload hook +/// rebuilds a device-resident forward (weights on device — the host-span path is transfer-bound and barely beats +/// CPU, so the resident forward is what makes the deep solvers fast). /// -public sealed class CubeModelService(IModelStore store, AdaptiveBackend backend, ILogger logger) : IModelStartupService, IDisposable +public sealed class CubeModelService : StartupModelService, IDisposable { public const string EnvironmentId = "cube"; public const string AlgorithmId = "dqn"; + public const string PolicyAlgorithmId = "policy"; + public const string ValueDaviAlgorithmId = "value-davi-res"; + public const string EfficientPolicyAlgorithmId = "policy-efficient"; public const int MaxMoves = 20; - private readonly object _lock = new(); + private readonly IModelStore _store; + private readonly AdaptiveBackend _backend; + private readonly ILogger _logger; + private GreedyQAgent? _agent; + private readonly RefreshingModel _policy; + private readonly RefreshingModel _value; + private readonly RefreshingModel _efficient; + private DeviceResidualMlp? _residentValue; + private DeviceMlp? _residentEfficient; - public ModelStatus Status { get; private set; } = ModelStatus.Loading; - public string? Error { get; private set; } + public CubeModelService(IModelStore store, AdaptiveBackend backend, ILogger logger) : base(logger) + { + _store = store; + _backend = backend; + _logger = logger; + _policy = new(store, EnvironmentId, PolicyAlgorithmId, CubePolicyNet.Load, logger, "cube policy net"); + _value = new(store, EnvironmentId, ValueDaviAlgorithmId, ResidualMlpCheckpoint.Load, logger, "cube DAVI value net", + onLoaded: RebuildResidentValue); + _efficient = new(store, EnvironmentId, EfficientPolicyAlgorithmId, CubePolicyNet.Load, logger, "EfficientCube policy net", + onLoaded: RebuildResidentEfficient); + } + + protected override string ModelName => "Rubik's Cube"; /// The greedy inference agent, or null while the model is not ready. Loads lazily from the store. public GreedyQAgent? Agent @@ -35,177 +59,81 @@ public GreedyQAgent? Agent } } - public const string PolicyAlgorithmId = "policy"; - private CubePolicyNet? _policyNet; - private DateTime _policyLoadedUtc = DateTime.MinValue; - private static readonly TimeSpan PolicyRefresh = TimeSpan.FromMinutes(5); - /// - /// The Kociemba-imitation policy/value net (preferred over the DQN when present, PLAN - /// M16), or null if the store has none. Re-read every few minutes so a long-running - /// Lab campaign's improving checkpoints are picked up without restarting the host. + /// The Kociemba-imitation policy/value net (preferred over the DQN when present, PLAN M16), or null if the store + /// has none. Re-read every few minutes so a long-running Lab campaign's checkpoints are picked up live. /// - public CubePolicyNet? PolicyNet - { - get - { - if (DateTime.UtcNow - _policyLoadedUtc < PolicyRefresh) return _policyNet; - lock (_lock) - { - if (DateTime.UtcNow - _policyLoadedUtc < PolicyRefresh) return _policyNet; - try - { - using var stream = store.TryOpenRead(EnvironmentId, PolicyAlgorithmId); - if (stream is not null) - { - _policyNet = CubePolicyNet.Load(stream); - logger.LogInformation("Loaded cube policy net from the store."); - } - } - catch (Exception ex) - { - // A mid-write or corrupt checkpoint must not break solving; keep the previous net. - logger.LogWarning(ex, "Failed to (re)load the cube policy net; keeping the previous one."); - } - _policyLoadedUtc = DateTime.UtcNow; - return _policyNet; - } - } - } + public CubePolicyNet? PolicyNet => _policy.Value; - public const string ValueDaviAlgorithmId = "value-davi-res"; - private ResidualMlp? _valueNet; - private DeviceResidualMlp? _residentValue; - private DateTime _valueLoadedUtc = DateTime.MinValue; + /// + /// The teacher-free DAVI value net (the "self-taught AI" shortest-move solver, PLAN M21), or null if the store + /// has none. Reading it also keeps in sync (rebuilt on the GPU on reload). + /// + public ResidualMlp? ValueNet => _value.Value; /// - /// The teacher-free DAVI value net (the "self-taught AI" shortest-move solver, PLAN M21), or null - /// if the store has none. Re-read on the same cadence as so an improving - /// Lab campaign's checkpoints are picked up without restarting the host. Accessing this also keeps - /// in sync (rebuilt on the GPU whenever the net reloads). + /// A device-resident GPU forward over the current value net, or null when no CUDA device is present (the solver + /// then scores on the CPU). Touching it applies any pending reload first. Thread-safe: the forward serializes + /// GPU access internally. /// - public ResidualMlp? ValueNet + public CubeValueSearch.BatchForward? ResidentValueForward { - get - { - if (DateTime.UtcNow - _valueLoadedUtc < PolicyRefresh) return _valueNet; - lock (_lock) - { - if (DateTime.UtcNow - _valueLoadedUtc < PolicyRefresh) return _valueNet; - try - { - using var stream = store.TryOpenRead(EnvironmentId, ValueDaviAlgorithmId); - if (stream is not null) - { - _valueNet = ResidualMlpCheckpoint.Load(stream); - // Mirror the freshly-loaded weights onto the GPU as a resident forward when a CUDA - // device is present — the host-span path is transfer-bound and barely beats CPU, so - // the resident forward (weights on device) is what makes the deep solver fast. - _residentValue?.Dispose(); - _residentValue = backend.Gpu is { } gpu ? gpu.CreateResidentForward(_valueNet) : null; - logger.LogInformation("Loaded cube DAVI value net from the store ({Mode}).", - _residentValue is not null ? "resident GPU forward" : "CPU forward"); - } - } - catch (Exception ex) - { - // A mid-write or corrupt checkpoint must not break solving; keep the previous net. - logger.LogWarning(ex, "Failed to (re)load the cube DAVI value net; keeping the previous one."); - } - _valueLoadedUtc = DateTime.UtcNow; - return _valueNet; - } - } + get { _ = ValueNet; return _residentValue is { } dev ? dev.Forward : null; } } /// - /// A device-resident GPU forward over the current value net, or null when no CUDA device is present - /// (the solver then scores on the CPU). Kept in sync by the getter — read that - /// first. Thread-safe: the underlying forward serializes GPU access internally. + /// The teacher-free EfficientCube policy net (the website's "self-taught AI", trained self-supervised on + /// scramble reversals — no Kociemba, no value-iteration bootstrap), or null if the store has none. Solved by + /// beam search; reading it keeps in sync. /// - public CubeValueSearch.BatchForward? ResidentValueForward => _residentValue is { } dev ? dev.Forward : null; - - public const string EfficientPolicyAlgorithmId = "policy-efficient"; - private CubePolicyNet? _efficientNet; - private DeviceMlp? _residentEfficient; - private DateTime _efficientLoadedUtc = DateTime.MinValue; + public CubePolicyNet? EfficientPolicyNet => _efficient.Value; /// - /// The teacher-free EfficientCube policy net (the website's "self-taught AI", trained self-supervised on - /// scramble reversals — no Kociemba, no value-iteration bootstrap), or null if the store has none. Solved - /// by beam search (). - /// Re-read on the same cadence as the other nets so an improving Lab campaign's checkpoints are picked up - /// without restarting the host; accessing this keeps in sync (the - /// policy head's weights mirrored onto the GPU whenever the net reloads). + /// A device-resident GPU forward over the current EfficientCube policy head (rows × 12 logits), or null when no + /// CUDA device is present (beam search then scores on the CPU). Touching it applies any pending reload first. /// - public CubePolicyNet? EfficientPolicyNet + public Func? ResidentEfficientForward { - get + get { _ = EfficientPolicyNet; return _residentEfficient is { } dev ? dev.Forward : null; } + } + + private void RebuildResidentValue(ResidualMlp net) + { + // Serialized against Dispose via Lock (both are the only writer/disposer of _residentValue). + lock (Lock) { - if (DateTime.UtcNow - _efficientLoadedUtc < PolicyRefresh) return _efficientNet; - lock (_lock) - { - if (DateTime.UtcNow - _efficientLoadedUtc < PolicyRefresh) return _efficientNet; - try - { - using var stream = store.TryOpenRead(EnvironmentId, EfficientPolicyAlgorithmId); - if (stream is not null) - { - _efficientNet = CubePolicyNet.Load(stream); - // Beam search runs the bulk of the forwards; mirror the policy head onto the GPU as a - // resident forward when a CUDA device is present (host-span is transfer-bound). - _residentEfficient?.Dispose(); - _residentEfficient = backend.Gpu is { } gpu ? gpu.CreateResidentForward(_efficientNet.PolicyAsMlp()) : null; - logger.LogInformation("Loaded EfficientCube policy net from the store ({Mode}).", - _residentEfficient is not null ? "resident GPU forward" : "CPU forward"); - } - } - catch (Exception ex) - { - // A mid-write or corrupt checkpoint must not break solving; keep the previous net. - logger.LogWarning(ex, "Failed to (re)load the EfficientCube policy net; keeping the previous one."); - } - _efficientLoadedUtc = DateTime.UtcNow; - return _efficientNet; - } + _residentValue?.Dispose(); + _residentValue = _backend.Gpu is { } gpu ? gpu.CreateResidentForward(net) : null; } + _logger.LogInformation("Cube DAVI value net scored on {Mode}.", _residentValue is not null ? "resident GPU forward" : "CPU"); } - /// - /// A device-resident GPU forward over the current EfficientCube policy head (rows × 12 logits), or null when - /// no CUDA device is present (beam search then scores on the CPU). Kept in sync by the - /// getter — read that first. Thread-safe: the forward serializes GPU access. - /// - public Func? ResidentEfficientForward => _residentEfficient is { } dev ? dev.Forward : null; + private void RebuildResidentEfficient(CubePolicyNet net) + { + lock (Lock) + { + _residentEfficient?.Dispose(); + // Beam search runs the bulk of the forwards; mirror the policy head onto the GPU when a device is present. + _residentEfficient = _backend.Gpu is { } gpu ? gpu.CreateResidentForward(net.PolicyAsMlp()) : null; + } + _logger.LogInformation("EfficientCube policy net scored on {Mode}.", _residentEfficient is not null ? "resident GPU forward" : "CPU"); + } - public bool TryLoadFromStore() + public override bool TryLoadFromStore() { - lock (_lock) + lock (Lock) { if (_agent is not null) return true; - using var stream = store.TryOpenRead(EnvironmentId, AlgorithmId); + using var stream = _store.TryOpenRead(EnvironmentId, AlgorithmId); if (stream is null) return false; var network = MlpCheckpoint.Load(stream); _agent = new GreedyQAgent(network, RubiksCubeEnv.ActionCount); Status = ModelStatus.Ready; - logger.LogInformation("Loaded Rubik's Cube model from the store."); + _logger.LogInformation("Loaded Rubik's Cube model from the store."); return true; } } - /// Loads the shipped DQN baseline at startup. The web does not train (PRD §14); the deep solver - /// nets (policy / DAVI value / EfficientCube) load lazily through their getters. - public void Initialize(CancellationToken cancellationToken) - { - if (TryLoadFromStore()) return; - lock (_lock) - { - Status = ModelStatus.Failed; - Error = "No trained Rubik's Cube model in the store."; - } - logger.LogWarning("No Rubik's Cube DQN model in the store — train it on a dev machine and commit the checkpoint to models/ (Git LFS)."); - } - /// /// Greedy rollout on a drawn cube: up to quarter-turns under /// the same no-undo mask the agent was trained with, reported honestly whether they @@ -232,7 +160,7 @@ public static (bool Solved, List Moves) Rollout(GreedyQAgent agent, Face public void Dispose() { - lock (_lock) + lock (Lock) { _residentValue?.Dispose(); _residentValue = null; diff --git a/src/RLDemo.Web/Services/Game2048ModelService.cs b/src/RLDemo.Web/Services/Game2048ModelService.cs index 75db53e..854c146 100644 --- a/src/RLDemo.Web/Services/Game2048ModelService.cs +++ b/src/RLDemo.Web/Services/Game2048ModelService.cs @@ -6,18 +6,16 @@ namespace RLDemo.Web.Services; /// /// Owns the 2048 n-tuple agent: loads the shipped checkpoint from the model store at startup (trained on a dev /// machine via self-play, the M5 recipe, and committed to models/ via Git LFS — the web never trains, -/// PRD §14). +/// PRD §14). Startup/status plumbing lives in . /// -public sealed class Game2048ModelService(IModelStore store, ILogger logger) : IModelStartupService +public sealed class Game2048ModelService(IModelStore store, ILogger logger) : StartupModelService(logger) { public const string EnvironmentId = "2048"; public const string AlgorithmId = "ntuple"; - private readonly object _lock = new(); private NTuple2048Agent? _agent; - public ModelStatus Status { get; private set; } = ModelStatus.Loading; - public string? Error { get; private set; } + protected override string ModelName => "2048"; /// The trained agent, or null while not ready. Loads lazily from the store. public NTuple2048Agent? Agent @@ -29,9 +27,9 @@ public NTuple2048Agent? Agent } } - public bool TryLoadFromStore() + public override bool TryLoadFromStore() { - lock (_lock) + lock (Lock) { if (_agent is not null) return true; using var stream = store.TryOpenRead(EnvironmentId, AlgorithmId); @@ -42,16 +40,4 @@ public bool TryLoadFromStore() return true; } } - - /// Loads the shipped checkpoint at startup. The web does not train (PRD §14). - public void Initialize(CancellationToken cancellationToken) - { - if (TryLoadFromStore()) return; - lock (_lock) - { - Status = ModelStatus.Failed; - Error = "No trained 2048 model in the store."; - } - logger.LogWarning("No 2048 model in the store — game unavailable. Train it on a dev machine and commit the checkpoint to models/ (Git LFS)."); - } } diff --git a/src/RLDemo.Web/Services/ModelServiceInfrastructure.cs b/src/RLDemo.Web/Services/ModelServiceInfrastructure.cs new file mode 100644 index 0000000..649abd9 --- /dev/null +++ b/src/RLDemo.Web/Services/ModelServiceInfrastructure.cs @@ -0,0 +1,110 @@ +using MintPlayer.AI.ReinforcementLearning.Core.Checkpoints; + +namespace RLDemo.Web.Services; + +public enum ModelStatus { Loading, Ready, Failed } + +/// +/// A game service with startup work (load its trained checkpoint from the store, or warm a cache). The web does +/// NOT train: models are produced on a dev machine (the Lab campaigns / Console) and committed to models/ +/// via Git LFS, so a single training path lives off the request path. A missing checkpoint = the game is +/// unavailable, not a trigger to train in-process. +/// +public interface IModelStartupService +{ + void Initialize(CancellationToken cancellationToken); +} + +/// +/// Base for a game's model service: owns the readiness snapshot (/) and the +/// standard "load the shipped checkpoint at startup, else mark the game unavailable" flow. +/// Subclasses implement — load the primary checkpoint, build the typed inference +/// object, set to Ready and return true — and expose that object through their own typed +/// getter (the type differs per game, so it stays in the subclass). Secondary nets that hot-reload during a +/// training campaign belong in a . +/// +public abstract class StartupModelService(ILogger logger) : IModelStartupService +{ + protected readonly object Lock = new(); + + public ModelStatus Status { get; protected set; } = ModelStatus.Loading; + public string? Error { get; protected set; } + + /// Display name used in the "no model" failure message/log (e.g. "Rush Hour", "2048"). + protected abstract string ModelName { get; } + + /// Load the primary checkpoint if present; build the inference object, set =Ready and return true. Safe to call any time. + public abstract bool TryLoadFromStore(); + + /// Loads the shipped checkpoint at startup. The web does not train (PRD §14). + public void Initialize(CancellationToken cancellationToken) + { + if (TryLoadFromStore()) return; + lock (Lock) + { + Status = ModelStatus.Failed; + Error = $"No trained {ModelName} model in the store."; + } + logger.LogWarning("No {ModelName} model in the store — game unavailable. Train it on a dev machine (Lab/Console) and commit the checkpoint to models/ (Git LFS).", ModelName); + } +} + +/// +/// A checkpoint that is (re)read from the model store on a fixed cadence, so a long-running training campaign's +/// improving checkpoints are picked up without restarting the host. re-reads at most once per +/// refresh interval, under a double-checked lock; a mid-write or corrupt checkpoint is swallowed and the previous +/// value kept, so serving never breaks on a bad read. The optional onLoaded hook runs right after each +/// successful (re)load — used to mirror freshly-loaded weights onto the GPU as a resident forward. Returns null +/// until the store first has the checkpoint. +/// +public sealed class RefreshingModel( + IModelStore store, + string environmentId, + string algorithmId, + Func load, + ILogger logger, + string name, + TimeSpan? refresh = null, + Action? onLoaded = null) where T : class +{ + private readonly TimeSpan _refresh = refresh ?? TimeSpan.FromMinutes(5); + private readonly object _lock = new(); + private T? _value; + private DateTime _loadedUtc = DateTime.MinValue; + + public T? Value + { + get + { + if (DateTime.UtcNow - _loadedUtc < _refresh) return _value; + lock (_lock) + { + if (DateTime.UtcNow - _loadedUtc < _refresh) return _value; + try + { + using var stream = store.TryOpenRead(environmentId, algorithmId); + if (stream is not null) + { + _value = load(stream); + onLoaded?.Invoke(_value); + logger.LogInformation("Loaded {Name} from the store.", name); + } + } + catch (Exception ex) + { + // A mid-write or corrupt checkpoint must not break serving; keep the previous value. + logger.LogWarning(ex, "Failed to (re)load {Name}; keeping the previous one.", name); + } + _loadedUtc = DateTime.UtcNow; + return _value; + } + } + } +} + +/// Runs every game's startup work (load checkpoint / warm caches) once, in parallel, off the request path. +public sealed class ModelStartupHostedService(IEnumerable models) : BackgroundService +{ + protected override Task ExecuteAsync(CancellationToken stoppingToken) + => Task.WhenAll(models.Select(m => Task.Run(() => m.Initialize(stoppingToken), stoppingToken))); +} diff --git a/src/RLDemo.Web/Services/RushHourModelService.cs b/src/RLDemo.Web/Services/RushHourModelService.cs index 699a94f..a89d544 100644 --- a/src/RLDemo.Web/Services/RushHourModelService.cs +++ b/src/RLDemo.Web/Services/RushHourModelService.cs @@ -1,39 +1,35 @@ using MintPlayer.AI.ReinforcementLearning.Core.Checkpoints; -using MintPlayer.AI.ReinforcementLearning.Core.Nn; using MintPlayer.AI.ReinforcementLearning.Core.Training; using MintPlayer.AI.ReinforcementLearning.Environments.RushHour; namespace RLDemo.Web.Services; -public enum ModelStatus { Loading, Ready, Failed } - /// -/// A game service with startup work (load its trained checkpoint from the store, or warm a cache). The web does -/// NOT train: models are produced on a dev machine (the Lab campaigns / Console) and committed to models/ -/// via Git LFS, so a single training path lives off the request path. A missing checkpoint = the game is -/// unavailable, not a trigger to train in-process. +/// Owns the Rush Hour nets: loads the DQN baseline from the model store at startup and serves the imitation +/// policy/value net (hot-reloaded during a training campaign) when present. The checkpoints are trained on a dev +/// machine and shipped in models/ via Git LFS — the web never trains (PRD §14). Startup/status plumbing +/// lives in ; the refreshing policy net in . /// -public interface IModelStartupService -{ - void Initialize(CancellationToken cancellationToken); -} - -/// -/// Owns the Rush Hour DQN: loads it from the model store at startup (the checkpoint is trained on a dev machine -/// and shipped in models/ via Git LFS — the web never trains, PRD §14). Also serves the imitation policy -/// net when present. Thread-safe readiness snapshot for /api/rushhour/status. -/// -public sealed class RushHourModelService(IModelStore store, ILogger logger) : IModelStartupService +public sealed class RushHourModelService : StartupModelService { public const string EnvironmentId = "rushhour"; public const string AlgorithmId = "dqn"; + public const string PolicyAlgorithmId = "policy"; public const int MaxMoves = 60; - private readonly object _lock = new(); + private readonly IModelStore _store; + private readonly ILogger _logger; + private readonly RefreshingModel _policy; private GreedyQAgent? _agent; - public ModelStatus Status { get; private set; } = ModelStatus.Loading; - public string? Error { get; private set; } + public RushHourModelService(IModelStore store, ILogger logger) : base(logger) + { + _store = store; + _logger = logger; + _policy = new(store, EnvironmentId, PolicyAlgorithmId, RushHourPolicyNet.Load, logger, "Rush Hour policy net"); + } + + protected override string ModelName => "Rush Hour"; /// The greedy inference agent, or null while the model is not ready. Loads lazily from the store. public GreedyQAgent? Agent @@ -45,76 +41,25 @@ public GreedyQAgent? Agent } } - public const string PolicyAlgorithmId = "policy"; - private RushHourPolicyNet? _policyNet; - private DateTime _policyLoadedUtc = DateTime.MinValue; - private static readonly TimeSpan PolicyRefresh = TimeSpan.FromMinutes(5); - /// - /// The imitation-learned policy/value net (preferred over the DQN when present), or - /// null if the store has none. Re-read every few minutes so a long-running training - /// campaign's improving checkpoints are picked up without restarting the host. + /// The imitation-learned policy/value net (preferred over the DQN when present), or null if the store has none. + /// Re-read every few minutes so a long-running training campaign's improving checkpoints are picked up without + /// restarting the host. /// - public RushHourPolicyNet? PolicyNet - { - get - { - if (DateTime.UtcNow - _policyLoadedUtc < PolicyRefresh) return _policyNet; - lock (_lock) - { - if (DateTime.UtcNow - _policyLoadedUtc < PolicyRefresh) return _policyNet; - try - { - using var stream = store.TryOpenRead(EnvironmentId, PolicyAlgorithmId); - if (stream is not null) - { - _policyNet = RushHourPolicyNet.Load(stream); - logger.LogInformation("Loaded Rush Hour policy net from the store."); - } - } - catch (Exception ex) - { - // A mid-write or corrupt checkpoint must not break solving; keep the previous net. - logger.LogWarning(ex, "Failed to (re)load the Rush Hour policy net; keeping the previous one."); - } - _policyLoadedUtc = DateTime.UtcNow; - return _policyNet; - } - } - } + public RushHourPolicyNet? PolicyNet => _policy.Value; - /// Loads a stored checkpoint if one exists; safe to call at any time. - public bool TryLoadFromStore() + public override bool TryLoadFromStore() { - lock (_lock) + lock (Lock) { if (_agent is not null) return true; - using var stream = store.TryOpenRead(EnvironmentId, AlgorithmId); + using var stream = _store.TryOpenRead(EnvironmentId, AlgorithmId); if (stream is null) return false; var network = MlpCheckpoint.Load(stream); _agent = new GreedyQAgent(network, RushHourBoard.ActionCount); Status = ModelStatus.Ready; - logger.LogInformation("Loaded Rush Hour model from the store."); + _logger.LogInformation("Loaded Rush Hour model from the store."); return true; } } - - /// Loads the shipped checkpoint at startup. The web does not train (PRD §14). - public void Initialize(CancellationToken cancellationToken) - { - if (TryLoadFromStore()) return; - lock (_lock) - { - Status = ModelStatus.Failed; - Error = "No trained Rush Hour model in the store."; - } - logger.LogWarning("No Rush Hour model in the store — game unavailable. Train it on a dev machine (Lab/Console) and commit the checkpoint to models/ (Git LFS)."); - } -} - -/// Runs every game's startup work (load checkpoint / warm caches) once, in parallel, off the request path. -public sealed class ModelStartupHostedService(IEnumerable models) : BackgroundService -{ - protected override Task ExecuteAsync(CancellationToken stoppingToken) - => Task.WhenAll(models.Select(m => Task.Run(() => m.Initialize(stoppingToken), stoppingToken))); } diff --git a/tools/MintPlayer.AI.ReinforcementLearning.Lab/DqnCampaignBase.cs b/tools/MintPlayer.AI.ReinforcementLearning.Lab/DqnCampaignBase.cs new file mode 100644 index 0000000..02eefe4 --- /dev/null +++ b/tools/MintPlayer.AI.ReinforcementLearning.Lab/DqnCampaignBase.cs @@ -0,0 +1,149 @@ +using MintPlayer.AI.ReinforcementLearning.Core.Checkpoints; +using MintPlayer.AI.ReinforcementLearning.Core.Environments; +using MintPlayer.AI.ReinforcementLearning.Core.Nn; +using MintPlayer.AI.ReinforcementLearning.Core.Random; +using MintPlayer.AI.ReinforcementLearning.Core.Training; + +/// +/// Shared spine for the score-maximizing DQN campaigns (Snake, FruitCake). Both train a Double+Dueling +/// resumably via , ship a keep-best deployable net under +/// <env>/dqn (the id the web loads) plus the full resume state under <env>/<StateId>, +/// and follow the identical resume → per-chunk-train → keep-best-checkpoint lifecycle. Subclasses supply only the +/// genuinely game-specific pieces: the environment, the DQN hyperparameters, the eval rollout + metrics, and +/// (optionally) how a loaded net is adapted before warm-starting. +/// +/// Keep-best: DQN eval is noisy, so the deployable net is overwritten only when the gate metric IMPROVES on the +/// best seen (seeded in from the starting net's score, so a noisy dip never regresses a good +/// shipped model). The resume state always tracks the latest net, so a continuation picks up where it left off. +/// +/// +internal abstract class DqnCampaignBase(ulong seed, int chunkSteps, long targetSteps) : ITrainingCampaign +{ + protected const string NetId = "dqn"; // deployable DuelingQNet — the id the web loads + + protected SeedSequence Seeds { get; } = new(seed); + protected DqnTrainingState? State { get; private set; } + private IValueNet? _warmNet; // deployable net to warm-start from when there's no full resume state + private double _bestGate = double.NegativeInfinity; + private double _lastGate = double.NegativeInfinity; + + /// The model-store environment id (e.g. "snake", "fruitcake"). + public abstract string Environment { get; } + + /// Model-store id for the full resume state; a separate line (e.g. noisy) overrides it. + protected virtual string StateId => "dqn-state"; + + /// The env the trainer learns on (may carry reward shaping and/or action masking). + protected abstract IEnvironment TrainEnv { get; } + + /// DQN hyperparameters; / are set per chunk by the base. + protected abstract DqnOptions BaseOptions { get; } + + /// Label for the step counter in metrics/logs ("steps", "drops", …). + protected abstract string StepNoun { get; } + + /// Human name of the keep-best gate metric for logs (e.g. "food@12", "mean score"). + protected abstract string GateNoun { get; } + + /// + /// Evaluate a net on the deployed setting. Returns the keep-best GATE metric, the display metrics (the base + /// prepends the step count and appends loss), and a summary fragment (the base wraps it with step count + loss). + /// + protected abstract (double Gate, IReadOnlyList Metrics, string Summary) EvaluateNet(IValueNet net); + + /// Adapt a loaded deployable net before warm-starting. Default identity; FruitCake promotes plain→noisy and grows the input. + protected virtual IValueNet AdaptWarmNet(DuelingQNet loaded) => loaded; + + public bool Resume(IModelStore store) + { + bool resumed = false; + using (var s = store.TryOpenRead(Environment, StateId)) + { + if (s is not null) + { + State = DqnTrainingState.Load(s); + Log($"resumed {Environment} DQN at {State.StepsCompleted:N0} {StepNoun} (last eval return {State.LastEval:F2})"); + resumed = true; + } + } + // No full resume state, but the deployable net may be present (e.g. the shipped checkpoint). Warm-start from + // it: a fresh optimizer/replay buffer continues the trained net rather than discarding its weights. + if (!resumed) + { + using var net = store.TryOpenRead(Environment, NetId); + if (net is not null) + { + _warmNet = AdaptWarmNet(DuelingQNetCheckpoint.Load(net)); + Log($"warm-starting from the deployable {Environment} net '{NetId}' (fresh optimizer + replay buffer)"); + resumed = true; + } + } + + // Seed keep-best from the starting net's score so training only ever ships a net that BEATS it. + var startNet = State?.Online ?? _warmNet; + if (startNet is not null) + { + _bestGate = EvaluateNet(startNet).Gate; + Log($"baseline {GateNoun}: {_bestGate:F2} (will only re-save the deployable net when an eval beats this)"); + } + else + { + Log($"starting fresh {Environment} DQN training"); + } + return resumed; + } + + public long TrainChunk() + { + int from = State?.StepsCompleted ?? 0; + int to = from + chunkSteps; + if (targetSteps > 0) to = (int)Math.Min(to, targetSteps); + // EvalEvery == the chunk size so the trainer's own eval fires at most once per chunk — the campaign's + // authoritative eval is Evaluate(). MaxSteps is ABSOLUTE: resuming raises the ceiling. + var options = BaseOptions with { MaxSteps = to, EvalEvery = Math.Max(1, chunkSteps) }; + // First chunk: resume the full state if present, else warm-start from the deployable net (if any). + var result = DqnTrainer.Train(TrainEnv, options, Seeds, resume: State, warmStart: State is null ? _warmNet : null); + State = result.State; + return State.StepsCompleted; + } + + /// Score-maximizing: no hard goal. Stops on the runner's time budget, or an optional absolute step cap. + public bool IsComplete => targetSteps > 0 && (State?.StepsCompleted ?? 0) >= targetSteps; + + public CampaignEval Evaluate() + { + int steps = State?.StepsCompleted ?? 0; + // Eval the trained net if we've trained this run, else the warm-start net (so eval-only works on the + // shipped checkpoint too); only "no model at all" yields the placeholder. + var net = State?.Online ?? _warmNet; + if (net is null) + return new CampaignEval([new(StepNoun, 0, "0")], "no model yet (train first)"); + + var (gate, metrics, summary) = EvaluateNet(net); + _lastGate = gate; // Checkpoint() ships the net only when this beats the best seen + + float loss = State?.LastLoss ?? 0f; + var full = new List { new(StepNoun, steps, "0") }; + full.AddRange(metrics); + full.Add(new("loss", loss, "F4")); + return new CampaignEval(full, $"{StepNoun} {steps:N0} | {summary} | loss {loss:F4}"); + } + + public void Checkpoint(IModelStore store) + { + if (State is null) return; + // The resume state always tracks the latest net (so a continuation picks up where it left off). + store.Save(Environment, StateId, s => State.Save(s)); + // The DEPLOYABLE net is keep-best: only overwrite it when this eval beat the best seen (DQN eval is noisy). + if (_lastGate > _bestGate) + { + _bestGate = _lastGate; + store.Save(Environment, NetId, s => DuelingQNetCheckpoint.Save((DuelingQNet)State.Online, s)); + Log($"new best {GateNoun} {_bestGate:F2} → saved deployable net '{NetId}'"); + } + } + + public virtual void Dispose() { } + + protected static void Log(string message) => Console.WriteLine($"{DateTime.UtcNow:HH:mm:ss} {message}"); +} diff --git a/tools/MintPlayer.AI.ReinforcementLearning.Lab/FruitCakeDqnCampaign.cs b/tools/MintPlayer.AI.ReinforcementLearning.Lab/FruitCakeDqnCampaign.cs index 48ca4a2..17120cf 100644 --- a/tools/MintPlayer.AI.ReinforcementLearning.Lab/FruitCakeDqnCampaign.cs +++ b/tools/MintPlayer.AI.ReinforcementLearning.Lab/FruitCakeDqnCampaign.cs @@ -1,4 +1,4 @@ -using MintPlayer.AI.ReinforcementLearning.Core.Checkpoints; +using MintPlayer.AI.ReinforcementLearning.Core.Environments; using MintPlayer.AI.ReinforcementLearning.Core.Nn; using MintPlayer.AI.ReinforcementLearning.Core.Random; using MintPlayer.AI.ReinforcementLearning.Core.Schedules; @@ -6,39 +6,31 @@ using MintPlayer.AI.ReinforcementLearning.Environments.FruitCake; /// -/// FruitCake DQN campaign (`--game fruitcake`, PRD FRUITCAKE_AI §4.4/§7-A2) as an -/// on — the score-maximizing paradigm (an open-ended game whose eval is the mean -/// episodic score, not a solve rate). Trains a Double+Dueling on the headless -/// (one step = one drop, simulated to rest in pure compute; rotation off). Resumable -/// bitwise-identically via ; each chunk raises the absolute -/// and continues. Persists the deployable net under `fruitcake`/`dqn` (the id -/// the web's FruitCakeModelService will load, A3) plus the full resume state under `fruitcake`/`dqn-state`. +/// FruitCake DQN campaign (`--game fruitcake`, PRD FRUITCAKE_AI §4.4/§7-A2) — the score-maximizing paradigm (an +/// open-ended game whose eval is the mean episodic score, not a solve rate). Trains a Double+Dueling +/// on the headless (one step = one drop, simulated to rest in +/// pure compute; rotation off). All the resume/warm-start/keep-best/checkpoint plumbing lives in +/// ; this type supplies the envs, the DQN config (incl. the noisy/n-step lines), the +/// score-based eval, and the plain→noisy / grow-input warm-net adaptation. /// internal sealed class FruitCakeDqnCampaign(ulong seed, int chunkSteps, long targetSteps, int evalEpisodes, float learningRate, float epsilonStart, int[] hidden, double gamma, bool noisy = false, int nStep = 1, bool shapeRewards = false) - : ITrainingCampaign + : DqnCampaignBase(seed, chunkSteps, targetSteps) { - private const string NetId = "dqn"; // deployable DuelingQNet — the id the web loads (shared by both lines) - // Noisy training is a SEPARATE line with its own resume state: it must never resume a PLAIN state as a plain - // net (which would then train with ε=0 and NO exploration). The deployable NetId is shared and keep-best gated. - private string StateId => noisy ? "dqn-noisy-state" : "dqn-state"; - // Training env carries the reward shaping (ShapingGamma matches the learner's γ for policy-invariance); the // eval env stays a plain game so keep-best/A/B always judge real merge points, never the shaped signal. private readonly FruitCakeEnv _env = new() { ShapeRewards = shapeRewards, ShapingGamma = gamma }; private readonly FruitCakeEnv _evalEnv = new(); - private readonly SeedSequence _seeds = new(seed); - - private DqnTrainingState? _state; - private IValueNet? _warmNet; // deployable net to warm-start from when there's no full resume state - // Save-best: DQN eval is noisy, so the deployable net is only overwritten when the eval mean-score IMPROVES on - // the best seen (seeded from the starting net, so a noisy dip never regresses a good shipped model). - private double _bestScore = double.NegativeInfinity; - private double _lastEvalScore = double.NegativeInfinity; - public string Environment => "fruitcake"; + public override string Environment => "fruitcake"; + // Noisy training is a SEPARATE line with its own resume state: it must never resume a PLAIN state as a plain net + // (which would then train with ε=0 and NO exploration). The deployable NetId is shared and keep-best gated. + protected override string StateId => noisy ? "dqn-noisy-state" : "dqn-state"; + protected override IEnvironment TrainEnv => _env; + protected override string StepNoun => "drops"; + protected override string GateNoun => "mean score"; /// Double+Dueling DQN; MaxSteps is managed per chunk by the runner (drops, not physics sub-steps). - private DqnOptions BaseOptions => new() + protected override DqnOptions BaseOptions => new() { Dueling = true, DoubleDqn = true, @@ -58,115 +50,38 @@ internal sealed class FruitCakeDqnCampaign(ulong seed, int chunkSteps, long targ EvalEpisodes = 10, }; - public bool Resume(IModelStore store) + protected override IValueNet AdaptWarmNet(DuelingQNet loaded) { - bool resumed = false; - using (var s = store.TryOpenRead(Environment, StateId)) - { - if (s is not null) - { - _state = DqnTrainingState.Load(s); - Log($"resumed FruitCake DQN at {_state.StepsCompleted:N0} drops (last eval return {_state.LastEval:F2})"); - resumed = true; - } - } - // No full resume state, but the deployable net may exist (e.g. the shipped checkpoint). Warm-start from it: - // a fresh optimizer/replay buffer continues the trained net rather than discarding its weights. - if (!resumed) - { - using var net = store.TryOpenRead(Environment, NetId); - if (net is not null) - { - var loaded = DuelingQNetCheckpoint.Load(net); - // Promote-plain→noisy: the shipped net is plain, but a noisy run needs a noisy net. Copy its - // trained weights into the means + add fresh σ — behaviorally identical with noise off, so the - // run continues the ~current policy and merely adds learnable exploration. - IValueNet warm = noisy && !loaded.Noisy ? loaded.ToNoisy(_seeds.CreateRng(RngStreams.Init)) : loaded; - // If the observation has gained features since this net was trained (e.g. the big-fruit-position - // inputs), grow its input to fit — function-preserving (new inputs zero-init), so the baseline eval - // and warm-start continue the trained net rather than retraining from scratch. - if (warm.InputSize != FruitCakeEnv.ObservationSize) - { - Log($"growing the loaded net's input {warm.InputSize} → {FruitCakeEnv.ObservationSize} (observation gained features; weights preserved)"); - warm = warm.GrowInput(FruitCakeEnv.ObservationSize); - } - _warmNet = warm; - Log($"warm-starting from the deployable FruitCake net '{NetId}'{(noisy ? " (promoted to noisy)" : "")} (fresh optimizer + replay buffer)"); - resumed = true; - } - } - - // Seed save-best from the starting net's score so training only ever ships a net that BEATS it. - var startNet = _state?.Online ?? _warmNet; - if (startNet is not null) - { - _bestScore = EvalNet(startNet).Score; - Log($"baseline mean score: {_bestScore:F2} (will only re-save the deployable net when an eval beats this)"); - } - else + // Promote-plain→noisy: the shipped net is plain, but a noisy run needs a noisy net. Copy its trained + // weights into the means + add fresh σ — behaviorally identical with noise off, so the run continues the + // ~current policy and merely adds learnable exploration. + IValueNet warm = noisy && !loaded.Noisy ? loaded.ToNoisy(Seeds.CreateRng(RngStreams.Init)) : loaded; + // If the observation has gained features since this net was trained (e.g. the big-fruit-position inputs), + // grow its input to fit — function-preserving (new inputs zero-init), so the baseline eval and warm-start + // continue the trained net rather than retraining from scratch. + if (warm.InputSize != FruitCakeEnv.ObservationSize) { - Log("starting fresh FruitCake DQN training"); + Log($"growing the loaded net's input {warm.InputSize} → {FruitCakeEnv.ObservationSize} (observation gained features; weights preserved)"); + warm = warm.GrowInput(FruitCakeEnv.ObservationSize); } - return resumed; + if (noisy) Log("warm-start promoted to noisy"); + return warm; } - public long TrainChunk() + protected override (double Gate, IReadOnlyList Metrics, string Summary) EvaluateNet(IValueNet net) { - int from = _state?.StepsCompleted ?? 0; - int to = from + chunkSteps; - if (targetSteps > 0) to = (int)Math.Min(to, targetSteps); - // EvalEvery == the chunk size so the trainer's own eval fires at most once per chunk — the campaign's - // authoritative eval is the mean score in Evaluate(). MaxSteps is ABSOLUTE: resuming raises the ceiling. - var options = BaseOptions with { MaxSteps = to, EvalEvery = Math.Max(1, chunkSteps) }; - var result = DqnTrainer.Train(_env, options, _seeds, resume: _state, warmStart: _state is null ? _warmNet : null); - _state = result.State; - return _state.StepsCompleted; - } - - /// Score-maximizing: no hard goal. Stops on the runner's time budget, or an optional absolute drop cap. - public bool IsComplete => targetSteps > 0 && (_state?.StepsCompleted ?? 0) >= targetSteps; - - public CampaignEval Evaluate() - { - int steps = _state?.StepsCompleted ?? 0; - var net = _state?.Online ?? _warmNet; - if (net is null) - return new CampaignEval([new("drops", 0, "0")], "no model yet (train first)"); - - var (score, maxTier, meanReturn) = EvalNet(net); - _lastEvalScore = score; // Checkpoint() ships the net only when this beats the best seen - - float loss = _state?.LastLoss ?? 0f; - var metrics = new List + var (score, maxTier, meanReturn) = RunEval(net); + var metrics = new CampaignMetric[] { - new("drops", steps, "0"), new("score", score, "F2"), new("maxTier", maxTier, "F2"), new("return", meanReturn, "F2"), - new("loss", loss, "F4"), }; - return new CampaignEval(metrics, - $"drops {steps:N0} | score {score:F2} | maxTier {maxTier:F2} | return {meanReturn:F2} | loss {loss:F4}"); + return (score, metrics, $"score {score:F2} | maxTier {maxTier:F2} | return {meanReturn:F2}"); } - public void Checkpoint(IModelStore store) - { - if (_state is null) return; - // The resume state always tracks the latest net (so a continuation picks up where it left off). - store.Save(Environment, StateId, s => _state.Save(s)); - // The DEPLOYABLE net is save-best: only overwrite it when this eval beat the best seen (DQN eval is noisy). - if (_lastEvalScore > _bestScore) - { - _bestScore = _lastEvalScore; - store.Save(Environment, NetId, s => DuelingQNetCheckpoint.Save((DuelingQNet)_state.Online, s)); - Log($"new best mean score {_bestScore:F2} → saved deployable net '{NetId}'"); - } - } - - public void Dispose() { } - /// Mean episodic score, mean max-tier reached, and mean return over fixed-seed greedy episodes. - private (double Score, double MaxTier, double Return) EvalNet(IValueNet net) + private (double Score, double MaxTier, double Return) RunEval(IValueNet net) { var agent = new GreedyQAgent(net, FruitCakeEnv.ColumnCount); double totalScore = 0, totalMaxTier = 0, totalReturn = 0; @@ -191,6 +106,4 @@ public void Dispose() { } } return (totalScore / evalEpisodes, totalMaxTier / evalEpisodes, totalReturn / evalEpisodes); } - - private static void Log(string message) => Console.WriteLine($"{DateTime.UtcNow:HH:mm:ss} {message}"); } diff --git a/tools/MintPlayer.AI.ReinforcementLearning.Lab/SnakeDqnCampaign.cs b/tools/MintPlayer.AI.ReinforcementLearning.Lab/SnakeDqnCampaign.cs index 886ec85..83fb956 100644 --- a/tools/MintPlayer.AI.ReinforcementLearning.Lab/SnakeDqnCampaign.cs +++ b/tools/MintPlayer.AI.ReinforcementLearning.Lab/SnakeDqnCampaign.cs @@ -1,42 +1,30 @@ -using MintPlayer.AI.ReinforcementLearning.Core.Checkpoints; +using MintPlayer.AI.ReinforcementLearning.Core.Environments; using MintPlayer.AI.ReinforcementLearning.Core.Nn; -using MintPlayer.AI.ReinforcementLearning.Core.Random; using MintPlayer.AI.ReinforcementLearning.Core.Schedules; using MintPlayer.AI.ReinforcementLearning.Core.Training; using MintPlayer.AI.ReinforcementLearning.Environments.Snake; /// -/// Snake DQN campaign (`--game snake`, PLAN M22) as an on -/// (PLAN M25) — the score-maximizing paradigm: an *infinite-goal* game whose eval is -/// the mean episodic score (food), not a solve rate. Trains the masked Double+Dueling on a -/// small 6×6 grid (fast, dense food; the size-invariant observation transfers to the demo grid) and evaluates mean -/// food on the deployed 12×12 grid. Resumable bitwise-identically via : each chunk -/// raises the absolute and continues from the prior state. Persists the deployable -/// net under `snake`/`dqn` (the id the web's SnakeModelService loads) plus the full resume state under -/// `snake`/`dqn-state`. +/// Snake DQN campaign (`--game snake`, PLAN M22) — the score-maximizing paradigm: an *infinite-goal* game whose +/// eval is the mean episodic score (food), not a solve rate. Trains the masked Double+Dueling +/// on a small 6×6 grid (fast, dense food; the size-invariant observation transfers to +/// the demo grid) and evaluates mean food on the deployed 12×12 grid. All the resume/warm-start/keep-best/ +/// checkpoint plumbing lives in ; this type supplies only the envs, the DQN config, +/// and the food-based eval. /// internal sealed class SnakeDqnCampaign(ulong seed, int trainGrid, int evalGrid, int chunkSteps, long targetSteps, int evalEpisodes, float learningRate, float epsilonStart, int[] hidden, double gamma, float stepPenalty, bool safeMask) - : ITrainingCampaign + : DqnCampaignBase(seed, chunkSteps, targetSteps) { - private const string NetId = "dqn"; // deployable DuelingQNet — the id the web loads - private const string StateId = "dqn-state"; // full DqnTrainingState for lossless resume - private readonly SnakeEnv _env = new(trainGrid, stepPenalty, safeMask); private readonly SnakeEnv _evalEnv = new(evalGrid, stepPenalty, safeMask); - private readonly SeedSequence _seeds = new(seed); - - private DqnTrainingState? _state; - private IValueNet? _warmNet; // the deployable net to warm-start from when there's no full resume state - // Save-best: DQN eval is noisy, so the deployable net is only overwritten when the 12×12 eval IMPROVES on the - // best seen (seeded from the starting net, so a noisy dip never regresses a good shipped model). The resume - // state always tracks the latest net. - private double _bestFood = double.NegativeInfinity; - private double _lastEvalFood = double.NegativeInfinity; - public string Environment => "snake"; + public override string Environment => "snake"; + protected override IEnvironment TrainEnv => _env; + protected override string StepNoun => "steps"; + protected override string GateNoun => $"food@{evalGrid}"; /// The proven M22 config (masked Double+Dueling DQN); MaxSteps is managed per chunk by the runner. - private DqnOptions BaseOptions => new() + protected override DqnOptions BaseOptions => new() { Dueling = true, DoubleDqn = true, @@ -53,106 +41,19 @@ internal sealed class SnakeDqnCampaign(ulong seed, int trainGrid, int evalGrid, EvalEpisodes = 20, }; - public bool Resume(IModelStore store) + protected override (double Gate, IReadOnlyList Metrics, string Summary) EvaluateNet(IValueNet net) { - bool resumed = false; - using (var s = store.TryOpenRead(Environment, StateId)) - { - if (s is not null) - { - _state = DqnTrainingState.Load(s); - Log($"resumed Snake DQN at {_state.StepsCompleted:N0} steps (last eval return {_state.LastEval:F2})"); - resumed = true; - } - } - // No full resume state, but the deployable net may be present (e.g. the shipped checkpoint, or one produced - // by the old web one-shot trainer). Warm-start from it: a fresh optimizer/replay buffer continues the - // trained net rather than throwing away its weights. Lets us further-train (and eval) the shipped model. - if (!resumed) - { - using var net = store.TryOpenRead(Environment, NetId); - if (net is not null) - { - _warmNet = DuelingQNetCheckpoint.Load(net); - Log($"warm-starting from the deployable Snake net '{NetId}' (fresh optimizer + replay buffer)"); - resumed = true; - } - } - - // Seed save-best from the starting net's score so training only ever ships a net that BEATS it. - var startNet = _state?.Online ?? _warmNet; - if (startNet is not null) - { - _bestFood = EvalNet(startNet).Food; - Log($"baseline food@{evalGrid}: {_bestFood:F2} (will only re-save the deployable net when an eval beats this)"); - } - else + var (food, meanReturn) = RunEval(net); + var metrics = new CampaignMetric[] { - Log($"starting fresh Snake DQN training (train {trainGrid}×{trainGrid}, eval {evalGrid}×{evalGrid})"); - } - return resumed; - } - - public long TrainChunk() - { - int from = _state?.StepsCompleted ?? 0; - int to = from + chunkSteps; - if (targetSteps > 0) to = (int)Math.Min(to, targetSteps); - // EvalEvery == the chunk size so the trainer's own (6×6) eval fires at most once per chunk — the campaign's - // authoritative eval is the 12×12 food in Evaluate(). MaxSteps is ABSOLUTE: resuming raises the ceiling. - var options = BaseOptions with { MaxSteps = to, EvalEvery = Math.Max(1, chunkSteps) }; - // First chunk: resume the full state if present, else warm-start from the deployable net (if any). - var result = DqnTrainer.Train(_env, options, _seeds, resume: _state, warmStart: _state is null ? _warmNet : null); - _state = result.State; - return _state.StepsCompleted; - } - - /// Score-maximizing: no hard goal. Stops on the runner's time budget, or an optional absolute step cap. - public bool IsComplete => targetSteps > 0 && (_state?.StepsCompleted ?? 0) >= targetSteps; - - public CampaignEval Evaluate() - { - int steps = _state?.StepsCompleted ?? 0; - // Eval the trained net if we've trained this run, else the warm-start net (so eval-only works on the - // shipped checkpoint too); only "no model at all" yields the placeholder. - var net = _state?.Online ?? _warmNet; - if (net is null) - return new CampaignEval([new("steps", 0, "0")], "no model yet (train first)"); - - var (food, meanReturn) = EvalNet(net); - _lastEvalFood = food; // Checkpoint() ships the net only when this beats the best seen - - float loss = _state?.LastLoss ?? 0f; - var metrics = new List - { - new("steps", steps, "0"), new($"food{evalGrid}", food, "F2"), new("return", meanReturn, "F2"), - new("loss", loss, "F4"), }; - return new CampaignEval(metrics, - $"steps {steps:N0} | food@{evalGrid} {food:F2} | return {meanReturn:F2} | loss {loss:F4}"); + return (food, metrics, $"food@{evalGrid} {food:F2} | return {meanReturn:F2}"); } - public void Checkpoint(IModelStore store) - { - if (_state is null) return; - // The resume state always tracks the latest net (so a continuation picks up where it left off). - store.Save(Environment, StateId, s => _state.Save(s)); - // The DEPLOYABLE net is save-best: only overwrite it when this eval beat the best seen — DQN eval is - // noisy and the last net is often not the best (here the 270k net scored 23.3 but 300k fell to 19.5). - if (_lastEvalFood > _bestFood) - { - _bestFood = _lastEvalFood; - store.Save(Environment, NetId, s => DuelingQNetCheckpoint.Save((DuelingQNet)_state.Online, s)); - Log($"new best food@{evalGrid} {_bestFood:F2} → saved deployable net '{NetId}'"); - } - } - - public void Dispose() { } - /// Mean food + return over fixed-seed greedy episodes on the DEPLOYED grid (food is the gate metric). - private (double Food, double Return) EvalNet(IValueNet net) + private (double Food, double Return) RunEval(IValueNet net) { var agent = new GreedyQAgent(net, SnakeEnv.ActionCount); double totalFood = 0, totalReturn = 0; @@ -173,6 +74,4 @@ public void Dispose() { } } return (totalFood / evalEpisodes, totalReturn / evalEpisodes); } - - private static void Log(string message) => Console.WriteLine($"{DateTime.UtcNow:HH:mm:ss} {message}"); }