Dev

.gitignore

@@ -11,4 +11,7 @@ outputs/
 *.egg-info
 dist/
 build/
-.cursor/
+.cursor/
+*.db
+*.log
+*.logs

configs/rewards/bot_rewards.yaml

@@ -2,6 +2,17 @@
 rewards:
   bot:
     alpha: 0.6
+    reward_aggregate: mean
+    team:
+      hunger_damage_penalty: -0.025
+      fed_no_hunger_bonus: 0.06
+      food_security_coeff: 0.004
+      food_security_threshold: 80
+      food_delta_positive_coeff: 0.015
+    terminal:
+      win: 1000.0
+      loss: -1000.0
+      draw: 0.0
     warrior:
       damage_dealt: 0.15
       kill: 8.0

configs/training/train_mappo_bots.yaml

@@ -15,6 +15,7 @@ training:
   clip_range: 0.2
   ent_coef: 0.01
   vf_coef: 0.5
+  max_grad_norm: 0.5
   critic_hidden_dim: 256
   pool_max_size: 15
   checkpoint_interval: 100000

scripts/optuna_tune_mappo.py

@@ -14,7 +14,6 @@
 
 import optuna
 from loguru import logger
-from optuna import TrialPruned
 
 from village_ai_war.config_load import load_project_config
 from village_ai_war.training.train_mappo_bots import run_mappo_bots_training
@@ -25,17 +24,46 @@ def _study_path_slug(study_name: str) -> str:
     return s or "study"
 
 
-def _valid_batch_sizes(buffer: int) -> list[int]:
-    opts = [32, 64, 128, 256, 512, 1024]
-    valid = [b for b in opts if b <= buffer and buffer % b == 0]
-    if valid:
-        return valid
-    for b in range(8, buffer + 1):
-        if buffer % b == 0:
-            return [b]
-    if buffer >= 1:
-        return [1]
-    return []
+# Optuna requires the same categorical choices for a parameter name in every trial
+# (see CategoricalDistribution does not support dynamic value space).
+_BATCH_SIZE_CHOICES = [32, 64, 128, 256, 512, 1024]
+_N_STEPS_CHOICES = [128, 256, 512]
+
+# Correlated (gamma, gae_lambda) presets — single categorical for TPE-friendly search.
+_GAMMA_GAE_PRESETS = [
+    "0.97/0.92",
+    "0.98/0.94",
+    "0.99/0.95",
+    "0.995/0.97",
+    "0.999/0.98",
+]
+
+_CHECKPOINT_INTERVAL_CHOICES = [50_000, 100_000, 200_000]
+
+
+def _rollout_labels(n_envs: int) -> list[str]:
+    """Valid (n_steps, batch_size) pairs encoded as n{n}_bs{b} for fixed Optuna categorical space."""
+    labels: list[str] = []
+    for n_steps in _N_STEPS_CHOICES:
+        buf = n_envs * n_steps
+        for bs in _BATCH_SIZE_CHOICES:
+            if bs <= buf and buf % bs == 0:
+                labels.append(f"n{n_steps}_bs{bs}")
+    return sorted(labels)
+
+
+def _parse_rollout(label: str) -> tuple[int, int]:
+    if not label.startswith("n") or "_bs" not in label:
+        raise ValueError(f"bad rollout label: {label!r}")
+    left, right = label.split("_bs", 1)
+    return int(left[1:]), int(right)
+
+
+def _parse_gamma_gae(preset: str) -> tuple[float, float]:
+    parts = preset.split("/", 1)
+    if len(parts) != 2:
+        raise ValueError(f"bad gamma/gae preset: {preset!r}")
+    return float(parts[0]), float(parts[1])
 
 
 def main() -> None:
@@ -51,9 +79,9 @@ def main() -> None:
     )
     parser.add_argument("--seed", type=int, default=0)
     parser.add_argument("--config-name", type=str, default="default")
-    parser.add_argument("--n-envs", type=int, default=1)
-    parser.add_argument("--total-timesteps", type=int, default=50_000)
-    parser.add_argument("--selfplay-iterations", type=int, default=2)
+    parser.add_argument("--n-envs", type=int, default=4)
+    parser.add_argument("--total-timesteps", type=int, default=100_000)
+    parser.add_argument("--selfplay-iterations", type=int, default=4)
     parser.add_argument(
         "--disable-tensorboard",
         action="store_true",
@@ -66,28 +94,60 @@ def main() -> None:
         metavar="KEY=VALUE",
         help="Extra Hydra-style overrides (repeatable). Applied after trial suggestions.",
     )
+    parser.add_argument(
+        "--objective",
+        type=str,
+        default="ep_rew_mean",
+        choices=("ep_rew_mean", "win_townhall_frac", "win_frac"),
+        help=(
+            "Metric to maximize. Use one objective per Optuna study (storage); "
+            "do not mix objectives when load_if_exists. "
+            "ep_rew_mean: dense signal from mean episode return in the metrics window."
+        ),
+    )
     args = parser.parse_args()
 
+    n_envs = int(args.n_envs)
+    rollout_labels = _rollout_labels(n_envs)
+    if not rollout_labels:
+        logger.error(
+            "No valid rollout (n_steps, batch_size) for n_envs={} with batch choices {}; "
+            "increase --n-envs or adjust _BATCH_SIZE_CHOICES.",
+            n_envs,
+            _BATCH_SIZE_CHOICES,
+        )
+        sys.exit(1)
+
     slug = _study_path_slug(args.study_name)
     user_overrides: list[str] = list(args.override)
 
+    logger.info(
+        "HPO objective={}; search space rollout={}, gamma_gae presets={}, "
+        "checkpoint_intervals={}. If you changed param names vs an existing DB study, "
+        "use a new --study-name or storage file.",
+        args.objective,
+        len(rollout_labels),
+        len(_GAMMA_GAE_PRESETS),
+        _CHECKPOINT_INTERVAL_CHOICES,
+    )
+
     def objective(trial: optuna.Trial) -> float:
-        n_envs = int(args.n_envs)
-        n_steps = trial.suggest_categorical("n_steps", [128, 256, 512])
-        buffer = n_envs * n_steps
-        valid_bs = _valid_batch_sizes(buffer)
-        if not valid_bs:
-            raise TrialPruned(f"no valid batch_size for buffer={buffer}")
+        rollout = trial.suggest_categorical("rollout", rollout_labels)
+        n_steps, batch_size = _parse_rollout(rollout)
 
         lr = trial.suggest_float("learning_rate", 1e-5, 1e-2, log=True)
-        batch_size = trial.suggest_categorical("batch_size", valid_bs)
         n_epochs = trial.suggest_int("n_epochs", 4, 15)
         ent_coef = trial.suggest_float("ent_coef", 1e-4, 0.1, log=True)
-        gamma = trial.suggest_float("gamma", 0.95, 0.999)
-        gae_lambda = trial.suggest_float("gae_lambda", 0.9, 0.99)
+        gamma_gae = trial.suggest_categorical("gamma_gae", _GAMMA_GAE_PRESETS)
+        gamma, gae_lambda = _parse_gamma_gae(gamma_gae)
         clip_range = trial.suggest_float("clip_range", 0.05, 0.3)
         vf_coef = trial.suggest_float("vf_coef", 0.1, 1.0)
         critic_hidden_dim = trial.suggest_categorical("critic_hidden_dim", [128, 256, 512])
+        max_grad_norm = trial.suggest_float("max_grad_norm", 0.3, 2.0)
+        pool_max_size = trial.suggest_int("pool_max_size", 8, 30)
+        checkpoint_interval = trial.suggest_categorical(
+            "checkpoint_interval", _CHECKPOINT_INTERVAL_CHOICES
+        )
 
         trial_dir = f"{slug}/trial_{trial.number}"
         overrides: list[str] = [
@@ -104,6 +164,9 @@ def objective(trial: optuna.Trial) -> float:
             f"training.clip_range={clip_range}",
             f"training.vf_coef={vf_coef}",
             f"training.critic_hidden_dim={critic_hidden_dim}",
+            f"training.max_grad_norm={max_grad_norm}",
+            f"training.pool_max_size={pool_max_size}",
+            f"training.checkpoint_interval={checkpoint_interval}",
             f"training.checkpoint_dir=checkpoints/optuna/{trial_dir}",
             f"training.pool_dir=checkpoints/optuna/{trial_dir}/pool",
             f"training.log_dir=logs/optuna/{trial_dir}",
@@ -115,14 +178,27 @@ def objective(trial: optuna.Trial) -> float:
         flat = load_project_config(_ROOT, config_name=args.config_name, overrides=overrides)
         metrics = run_mappo_bots_training(flat, return_metrics=True)
         assert metrics is not None
+        win_townhall_frac = float(metrics["win_townhall_frac"])
         win_frac = float(metrics["win_frac"])
+        mean_ep = float(metrics["mean_episode_reward"])
+        if args.objective == "ep_rew_mean":
+            value = mean_ep
+        elif args.objective == "win_townhall_frac":
+            value = win_townhall_frac
+        else:
+            value = win_frac
         logger.info(
-            "trial {} finished: win_frac={} outcomes={}",
+            "trial {} finished: objective={} value={} mean_episode_reward={} "
+            "win_townhall_frac={} win_frac={} outcomes={}",
             trial.number,
+            args.objective,
+            value,
+            mean_ep,
+            win_townhall_frac,
             win_frac,
             metrics.get("outcome_fractions"),
         )
-        return win_frac
+        return value
 
     storage = args.storage.strip() or None
     sampler = optuna.samplers.TPESampler(seed=args.seed)
@@ -136,7 +212,7 @@ def objective(trial: optuna.Trial) -> float:
     study.optimize(objective, n_trials=args.n_trials, n_jobs=args.n_jobs, show_progress_bar=True)
 
     best = study.best_trial
-    logger.info("Best trial: {} value={}", best.number, best.value)
+    logger.info("Best trial: {} {}={}", best.number, args.objective, best.value)
     logger.info("Best params: {}", best.params)
 
 

scripts/run_game.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-"""Play: human vs MAPPO (2D), or passive demo with random village manager steps."""
+"""Play: human vs MAPPO (2D), MAPPO vs same MAPPO (no human), or random village demo."""
 
 from __future__ import annotations
 
@@ -17,7 +17,10 @@
 from village_ai_war.config_load import load_project_config
 from village_ai_war.env.game_env import GameEnv
 from village_ai_war.play.human_controls import collect_blue_bot_actions_for_tick
-from village_ai_war.play.mappo_human_tick import play_mappo_human_tick
+from village_ai_war.play.mappo_human_tick import (
+    play_mappo_human_tick,
+    play_mappo_self_play_tick,
+)
 
 
 def _resolve_ckpt(path_str: str | None) -> Path | None:
@@ -61,6 +64,10 @@ def _run_random_village_demo(
         )
         env = GameEnv(flat, mode="village", team=0, render_mode=None)
 
+    # 3D render needs a real GameState; it is only set after reset().
+    rng = np.random.default_rng(seed)
+    _obs, _ = env.reset(seed=seed)
+
     if env.render_mode == "human_3d":
         try:
             env.render()
@@ -82,12 +89,10 @@ def _run_random_village_demo(
                 )
                 env.close()
                 env = GameEnv(flat, mode="village", team=0, render_mode="human")
+                _obs, _ = env.reset(seed=seed)
             else:
                 raise
 
-    rng = np.random.default_rng(seed)
-    _obs, _ = env.reset(seed=seed)
-
     if env.render_mode is not None:
         logger.info(
             "Random village demo | render_mode={} | max_steps={} | close window or Ctrl+C to stop",
@@ -114,14 +119,20 @@ def main() -> None:
         module="pygame.pkgdata",
     )
     parser = argparse.ArgumentParser(
-        description="Village AI War: human vs MAPPO, or random village-manager demo."
+        description="Village AI War: human vs MAPPO, MAPPO self-play, or random village demo."
     )
     parser.add_argument(
         "--mappo-opponent",
         default="",
         help="Path to MAPPO zip (e.g. checkpoints/bots_mappo/mappo_bot_final.zip). "
         "Human plays BLUE vs MAPPO on RED; no village AI (training-faithful micro).",
     )
+    parser.add_argument(
+        "--mappo-self-play",
+        default="",
+        help="Path to MAPPO zip: RED and BLUE both use this checkpoint (no human). "
+        "Opens 2D pygame if a display is available, otherwise runs headless.",
+    )
     parser.add_argument("--seed", type=int, default=0)
     parser.add_argument("--max-steps", type=int, default=500)
     parser.add_argument(
@@ -138,11 +149,20 @@ def main() -> None:
 
     flat = load_project_config(_ROOT)
     mappo_path = _resolve_ckpt(args.mappo_opponent or None)
+    self_play_path = _resolve_ckpt(args.mappo_self_play.strip() or None)
+
+    if str(args.mappo_opponent).strip() and str(args.mappo_self_play).strip():
+        logger.error("Use only one of --mappo-opponent or --mappo-self-play.")
+        sys.exit(1)
 
     if str(args.mappo_opponent).strip() and mappo_path is None:
         logger.error("MAPPO checkpoint not found: {}", args.mappo_opponent)
         sys.exit(1)
 
+    if str(args.mappo_self_play).strip() and self_play_path is None:
+        logger.error("MAPPO checkpoint not found: {}", args.mappo_self_play)
+        sys.exit(1)
+
     if mappo_path is not None:
         if args.human_3d:
             logger.warning("MAPPO human play uses 2D pygame only; ignoring --human-3d.")
@@ -207,6 +227,56 @@ def _render_cb(overlay_lines: tuple[str, ...] = ()) -> None:
         env.close()
         return
 
+    if self_play_path is not None:
+        if args.human_3d:
+            logger.warning(
+                "MAPPO self-play uses 2D pygame only when a display is available; "
+                "ignoring --human-3d."
+            )
+        try:
+            env = GameEnv(flat, mode="bot", team=0, render_mode="human")
+        except Exception as e:  # noqa: BLE001
+            logger.warning("Display unavailable ({}); headless MAPPO self-play", e)
+            env = GameEnv(flat, mode="bot", team=0, render_mode=None)
+
+        try:
+            mappo_model = _load_mappo_policy(self_play_path, flat)
+        except Exception as e:  # noqa: BLE001
+            logger.error("Failed to load MAPPO from {}: {}", self_play_path, e)
+            sys.exit(1)
+        logger.info("Loaded MAPPO for self-play from {}", self_play_path)
+
+        n_slots = int(flat["game"]["max_bots_for_role_change"])
+        env.reset(seed=args.seed)
+
+        if env.render_mode is not None:
+            logger.info(
+                "MAPPO vs MAPPO (same checkpoint) | max_steps={} | ESC/close to quit",
+                args.max_steps,
+            )
+        else:
+            logger.info("MAPPO self-play (headless) | max_steps={}", args.max_steps)
+
+        for t in range(args.max_steps):
+            _obs, _r, term, trunc, info = play_mappo_self_play_tick(
+                env,
+                mappo_model,
+                n_bot_slots=n_slots,
+                deterministic=args.deterministic,
+            )
+            if env.render_mode is not None:
+                env.render(
+                    overlay_lines=(
+                        f"tick={info.get('tick', '?')}  t={t}",
+                        "RED vs BLUE — same MAPPO weights",
+                    )
+                )
+            if term or trunc:
+                logger.info("Done at t={} info={}", t, info)
+                break
+        env.close()
+        return
+
     _run_random_village_demo(
         flat,
         seed=args.seed,