Add DANRA tutorial notebook with pytest-nbmake (#69)

.github/workflows/install-and-test.yml

@@ -75,9 +75,12 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-meps-reduced-example-data-v0.3.0
 
-      - name: Run tests
+      - name: Run tests (excluding notebooks)
         run: |
-          pytest -vv -s --doctest-modules
+          pytest -vv -s --doctest-modules --ignore=docs/notebooks
+
+      # Notebooks run in their own path-filtered, CPU-only workflow
+      # (.github/workflows/notebook-tests.yml).
 
       - name: Upload test figures
         uses: actions/upload-artifact@v4

.github/workflows/notebook-tests.yml

@@ -0,0 +1,42 @@
+# Runs the docs/notebooks/ tutorials end-to-end with nbmake so that API drift
+# in neural_lam breaks the notebook loudly instead of letting it rot. CPU-only.
+# Scoped to PRs that touch code the notebooks depend on (plus every push to
+# main and manual dispatch), so unrelated PRs don't pay the ~3 min cost.
+name: Notebook tests
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    paths:
+      - "neural_lam/**"
+      - "docs/notebooks/**"
+      - "tests/datastore_examples/**"
+      - "pyproject.toml"
+      - "uv.lock"
+      - ".github/workflows/notebook-tests.yml"
+  workflow_dispatch:
+
+jobs:
+  notebooks:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python 3.13
+        uses: actions/setup-python@v6
+        with:
+          python-version: 3.13
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7
+
+      - name: Install with uv
+        run: |
+          uv sync --extra cpu --group dev --locked
+          echo "$PWD/.venv/bin" >> $GITHUB_PATH
+
+      - name: Run notebook tests
+        run: |
+          pytest -vv -s --nbmake --nbmake-timeout=600 docs/notebooks/

.pre-commit-config.yaml

@@ -17,6 +17,12 @@ repos:
     hooks:
       - id: codespell
         description: Check for spelling errors
+        # codespell reads [tool.codespell] from pyproject.toml via tomllib,
+        # which is unavailable on Python <3.11 unless `tomli` is installed;
+        # on such runners the config silently fails to load. Pass --skip
+        # explicitly so it doesn't depend on that.
+        args:
+          - --skip=requirements/*,docs/notebooks/hello_world_danra.ipynb
 
   - repo: https://github.com/psf/black
     rev: 25.11.0

CHANGELOG.md

@@ -8,6 +8,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## [unreleased](https://github.com/mllam/neural-lam/compare/v0.6.0...HEAD)
 
 ### Added
+- Add `hello_world_danra.ipynb` end-to-end tutorial notebook for training on DANRA, with a dedicated path-filtered CI workflow that runs it end-to-end via `nbmake` [\#577](https://github.com/mllam/neural-lam/pull/577) @Sharkyii
+
 - Add `--num_sanity_val_steps` CLI argument to control sanity validation steps before training (#694)
 
 - Add `--train_steps_to_log` CLI option to log training loss for individual unroll steps, and deduplicate common prediction and loss computation steps across loops [\#674](https://github.com/mllam/neural-lam/issues/674) @GiGiKoneti
@@ -48,6 +50,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Fixed
 
+- `train_model.main` now re-raises on error (`@logger.catch(reraise=True)`) instead of logging and exiting 0, so training/eval failures surface to callers and CI (e.g. the notebook `nbmake` run) rather than passing silently [\#577](https://github.com/mllam/neural-lam/pull/577)
+
+- Allow `graph_lam` training and checkpoint reloads to accept the full set of
+  GNN type CLI options without passing hierarchical-only options to unsupported
+  constructors via a shared `build_predictor` helper ([#686](https://github.com/mllam/neural-lam/issues/686)).
+
 - Fix `RuntimeError` in `HiLAMParallel` forward pass on hierarchical graphs by offsetting edge indices into the global mesh node index space ([#679](https://github.com/mllam/neural-lam/issues/679))
 
 - Fix `IndexError` in HiLAM forward pass by offsetting grid nodes in `zero_index_g2m`/`zero_index_m2g` by the total mesh-node count across all levels ([#642](https://github.com/mllam/neural-lam/issues/642)) @Sir-Sloth-The-Lazy

docs/notebooks/conftest.py

@@ -0,0 +1,23 @@
+# Standard library
+import subprocess
+import sys
+from pathlib import Path
+
+# Third-party
+import pytest
+
+
+@pytest.fixture(scope="session", autouse=True)
+def setup_danra_datastore():
+    """Create the DANRA zarr datastore required by hello_world_danra.ipynb."""
+    datastore_config = Path(
+        "tests/datastore_examples/mdp/danra_100m_winds/danra.datastore.yaml"
+    )
+    zarr_output = datastore_config.parent / "danra.datastore.zarr"
+
+    # Only create if it doesn't exist
+    if not zarr_output.exists():
+        subprocess.run(
+            [sys.executable, "-m", "mllam_data_prep", str(datastore_config)],
+            check=True,
+        )

docs/notebooks/create_reduced_meps_dataset.ipynb

@@ -13,7 +13,11 @@
   {
    "cell_type": "code",
    "execution_count": 2,
-   "metadata": {},
+   "metadata": {
+    "tags": [
+     "skip-execution"
+    ]
+   },
    "outputs": [],
    "source": [
     "# Standard library\n",
@@ -36,7 +40,11 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "tags": [
+     "skip-execution"
+    ]
+   },
    "outputs": [],
    "source": [
     "# Load existing grid\n",
@@ -61,7 +69,11 @@
   {
    "cell_type": "code",
    "execution_count": 6,
-   "metadata": {},
+   "metadata": {
+    "tags": [
+     "skip-execution"
+    ]
+   },
    "outputs": [],
    "source": [
     "# Outer 10 grid points are border\n",
@@ -91,7 +103,11 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "tags": [
+     "skip-execution"
+    ]
+   },
    "outputs": [],
    "source": [
     "# Load surface_geopotential.npy, index only values from the reduced grid, and save to new file\n",
@@ -124,7 +140,11 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "tags": [
+     "skip-execution"
+    ]
+   },
    "outputs": [],
    "source": [
     "num_vars = 8\n",
@@ -162,7 +182,11 @@
   {
    "cell_type": "code",
    "execution_count": 12,
-   "metadata": {},
+   "metadata": {
+    "tags": [
+     "skip-execution"
+    ]
+   },
    "outputs": [
     {
      "name": "stdout",
@@ -188,7 +212,11 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "tags": [
+     "skip-execution"
+    ]
+   },
    "outputs": [],
    "source": [
     "for sample in ['train', 'test', 'val']:\n",