examples: drive VMEC++ from external optimizers (internal basis)

.github/actions/run-benchmarks/action.yml

@@ -53,6 +53,14 @@ runs:
         export OMP_WAIT_POLICY=passive
         pytest benchmarks/test_benchmarks.py --benchmark-json=benchmark_results.json
 
+    # github-action-benchmark's "pytest" tool always plots pytest-benchmark's
+    # `ops` field as iter/sec, but the C++ suite emits seconds.
+    # We convert both to the same format, so plotting is the same.
+    - name: Convert Python benchmark result to seconds
+      shell: bash
+      run: |
+        python3 benchmarks/to_seconds_json.py pytest benchmark_results.json benchmark_results_seconds.json
+
     - name: Store / compare Python benchmark result
       # Skip the result upload/compare for fork PRs: their GITHUB_TOKEN is
       # read-only, so comment-on-alert/auto-push hit 'Resource not accessible
@@ -61,8 +69,8 @@ runs:
       if: ${{ inputs.auto-push == 'true' || github.event.pull_request.head.repo.full_name == github.repository }}
       uses: benchmark-action/github-action-benchmark@v1.21.0
       with:
-        tool: "pytest"
-        output-file-path: benchmark_results.json
+        tool: "customSmallerIsBetter"
+        output-file-path: benchmark_results_seconds.json
         gh-pages-branch: benchmark-runs
         benchmark-data-dir-path: benchmarks
         alert-threshold: "150%"
@@ -76,7 +84,7 @@ runs:
     # Builds the four benchmark cc_binary targets, runs each with
     # --benchmark_format=json, and merges their output into a single
     # cpp_benchmark_results.json (schema: {"context": ..., "benchmarks": [...]})
-    # for the store step below. Entries with "error_occurred" (e.g. an
+    # for the conversion step below. Entries with "error_occurred" (e.g. an
     # FFT-path benchmark skipped because no FFTX codelet exists for that
     # resolution) are dropped: they carry no valid timing to track.
 
@@ -142,6 +150,14 @@ runs:
               f"({skipped} skipped/errored).")
         PY
 
+    # github-action-benchmark's "pytest" tool always plots pytest-benchmark's
+    # `ops` field as iter/sec, but the C++ suite emits seconds.
+    # We convert both to the same format, so plotting is the same.
+    - name: Convert C++ benchmark result to seconds
+      shell: bash
+      run: |
+        python3 benchmarks/to_seconds_json.py googlecpp cpp_benchmark_results.json cpp_benchmark_results_seconds.json
+
     - name: Store / compare C++ benchmark result
       # Same fork-PR skip rationale as the Python store step above.
       #
@@ -160,8 +176,8 @@ runs:
       uses: benchmark-action/github-action-benchmark@v1.21.0
       with:
         name: "C++ Microbenchmarks"
-        tool: "googlecpp"
-        output-file-path: cpp_benchmark_results.json
+        tool: "customSmallerIsBetter"
+        output-file-path: cpp_benchmark_results_seconds.json
         gh-pages-branch: benchmark-runs
         benchmark-data-dir-path: benchmarks
         skip-fetch-gh-pages: true

.github/workflows/full_validation.yaml

@@ -0,0 +1,70 @@
+name: Full V&V against reference VMEC
+
+# Runs the full Verification & Validation suite from proximafusion/vmecpp-validation
+# (all ~219 input configurations against reference Fortran VMEC2000), using the
+# vmecpp version built from this commit rather than the version pinned on PyPI/GitHub.
+# This is a lot more exhaustive (and slower) than the "short" validation that
+# vmecpp-validation itself runs on its own PRs, so it is not run on every push here.
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref || github.run_id }}
+  cancel-in-progress: ${{ github.ref_name != 'main' }}
+
+jobs:
+  full-validation:
+    name: Run full VMEC++ validation
+    runs-on: ubuntu-22.04
+    # The full parameter scan runs ~219 configurations, each computing a reference
+    # wout with Fortran VMEC2000 in Docker plus one with VMEC++: budget generously.
+    timeout-minutes: 360
+    steps:
+      - name: Check out VMEC++
+        uses: actions/checkout@v4
+        with:
+          path: vmecpp
+          lfs: true
+
+      - name: Check out vmecpp-validation
+        uses: actions/checkout@v4
+        with:
+          repository: proximafusion/vmecpp-validation
+          path: vmecpp-validation
+          lfs: true
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+
+      - name: Install required system packages for Ubuntu
+        run: |
+          sudo apt-get update && sudo apt-get install -y \
+            build-essential cmake libnetcdf-dev liblapack-dev liblapacke-dev libopenmpi-dev \
+            libomp-dev libeigen3-dev nlohmann-json3-dev libhdf5-dev
+
+      - name: Install vmecpp-validation's Python requirements
+        run: |
+          cd vmecpp-validation
+          # Install everything except vmecpp itself: we build and install vmecpp
+          # from this commit below instead of the version pinned in requirements.txt.
+          grep -v '^vmecpp@' requirements.txt > requirements.no-vmecpp.txt
+          python -m pip install -r requirements.no-vmecpp.txt
+
+      - name: Install VMEC++ from this commit
+        run: python -m pip install ./vmecpp
+
+      - name: Run full validation
+        working-directory: vmecpp-validation
+        run: python validate_vmec.py
+
+      - name: Upload V&V results
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: vnvresults
+          path: vmecpp-validation/vnvresults_*/
+          retention-days: 30

README.md

@@ -19,6 +19,7 @@
 
 [![CI](https://github.com/proximafusion/vmecpp/actions/workflows/tests.yaml/badge.svg)](https://github.com/proximafusion/vmecpp/actions/workflows/tests.yaml)
 [![C++ core tests](https://github.com/proximafusion/vmecpp/actions/workflows/test_bazel.yaml/badge.svg)](https://github.com/proximafusion/vmecpp/actions/workflows/test_bazel.yaml)
+[![Full V&V against reference VMEC](https://github.com/proximafusion/vmecpp/actions/workflows/full_validation.yaml/badge.svg)](https://github.com/proximafusion/vmecpp/actions/workflows/full_validation.yaml)
 [![Publish wheels to PyPI](https://github.com/proximafusion/vmecpp/actions/workflows/pypi_publish.yml/badge.svg)](https://github.com/proximafusion/vmecpp/actions/workflows/pypi_publish.yml)
 
 VMEC++ is a Python-friendly, from-scratch reimplementation in C++ of the Variational Moments Equilibrium Code (VMEC),
@@ -315,6 +316,10 @@ The single-thread runtimes as well as the contents of the "wout" file produced b
 The full validation test can be found at https://github.com/proximafusion/vmecpp-validation, including a set of sensible input configurations,
 parameter scan values and tolerances that make the comparison pass. See that repo for more information.
 
+This full validation (~219 input configurations) is run against every commit to `main` and can also be triggered
+on demand from the [Full V&V against reference VMEC](https://github.com/proximafusion/vmecpp/actions/workflows/full_validation.yaml) workflow
+(click "Run workflow"). It builds `vmecpp` from the corresponding commit rather than using the version pinned by `vmecpp-validation`.
+
 ## Differences with respect to PARVMEC/VMEC2000
 
 VMEC++:

benchmarks/to_seconds_json.py

@@ -0,0 +1,89 @@
+# SPDX-FileCopyrightText: 2024-present Proxima Fusion GmbH <info@proximafusion.com>
+#
+# SPDX-License-Identifier: MIT
+"""Convert pytest-benchmark or Google Benchmark JSON to seconds.
+
+github-action-benchmark's built-in "pytest" and "googlecpp" tool parsers plot whatever
+value each framework happens to report natively -- pytest-benchmark's "iter/sec" and
+Google Benchmark's raw per-iteration time in its own time_unit (commonly nanoseconds).
+Charting those side by side, or even a single suite whose time_unit isn't fixed,
+requires the reader (or the chart's JS) to know and convert between units.
+
+This script normalizes both to plain wall-clock seconds and emits the action's generic
+"customSmallerIsBetter" schema, so the chart always plots seconds and the benchmarks.js
+frontend needs no unit-detection logic.
+"""
+
+import argparse
+import json
+
+
+def convert_pytest(data):
+    results = []
+    for bench in data["benchmarks"]:
+        stats = bench["stats"]
+        results.append(
+            {
+                "name": bench["fullname"],
+                "unit": "seconds",
+                "value": stats["mean"],
+                "range": f"stddev: {stats['stddev']}",
+                "extra": f"rounds: {stats['rounds']}",
+            }
+        )
+    return results
+
+
+_GOOGLECPP_TIME_UNIT_TO_SECONDS = {
+    "s": 1.0,
+    "ms": 1e-3,
+    "us": 1e-6,
+    "ns": 1e-9,
+}
+
+
+def convert_googlecpp(data):
+    results = []
+    for bench in data["benchmarks"]:
+        if bench.get("error_occurred"):
+            continue
+        factor = _GOOGLECPP_TIME_UNIT_TO_SECONDS[bench["time_unit"]]
+        results.append(
+            {
+                "name": bench["name"],
+                "unit": "seconds",
+                "value": bench["real_time"] * factor,
+                "extra": (
+                    f"iterations: {bench['iterations']}\n"
+                    f"cpu: {bench['cpu_time'] * factor} seconds\n"
+                    f"threads: {bench.get('threads', 1)}"
+                ),
+            }
+        )
+    return results
+
+
+_CONVERTERS = {
+    "pytest": convert_pytest,
+    "googlecpp": convert_googlecpp,
+}
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("format", choices=sorted(_CONVERTERS))
+    parser.add_argument("input", help="Raw benchmark JSON produced by the tool.")
+    parser.add_argument("output", help="Where to write the customSmallerIsBetter JSON.")
+    args = parser.parse_args()
+
+    with open(args.input) as f:
+        data = json.load(f)
+
+    results = _CONVERTERS[args.format](data)
+
+    with open(args.output, "w") as f:
+        json.dump(results, f, indent=2)
+
+
+if __name__ == "__main__":
+    main()

docs/_static/benchmarks.js

@@ -37,20 +37,6 @@
     "#1f6feb",
   ];
 
-  // Parse mean duration from the extra field (e.g. "mean: 339.88 msec\nrounds: 5")
-  function parseMeanSeconds(bench) {
-    if (!bench.extra) return null;
-    var match = bench.extra.match(/mean:\s*([\d.]+)\s*([\w]+)/);
-    if (!match) return null;
-    var value = parseFloat(match[1]);
-    var unit = match[2];
-    if (unit === "msec") return value / 1000;
-    if (unit === "usec") return value / 1000000;
-    if (unit === "nsec") return value / 1000000000;
-    if (unit === "sec") return value;
-    return null;
-  }
-
   function renderCharts(data) {
     var container = document.getElementById("benchmark-charts");
     if (!container) return;
@@ -130,8 +116,7 @@
             {
               label: benchName,
               data: dataset.map(function (d) {
-                var secs = parseMeanSeconds(d.bench);
-                return secs !== null ? secs : 1.0 / d.bench.value;
+                return d.bench.value;
               }),
               borderColor: color,
               backgroundColor: color + "30",

examples/data/README.md

@@ -1,13 +1,13 @@
 # Example input and output files for VMEC++
 
-A few cases are available for testing VMCE++ and experimenting with it:
+A few cases are available for testing VMEC++ and experimenting with it:
 
 1. `cth_like_fixed_bdy.json` - Stellarator case, similar to the Compact Toroidal Hybrid ([CTH](https://www.auburn.edu/cosam/departments/physics/research/plasma_physics/compact_toroidal_hybrid/index.htm)) device
    1. `input.cth_like_fixed_bdy` - Fortran namelist input file to be used with Fortran VMEC
    1. `cth_like_fixed_bdy.json` - JSON input file for VMEC++, derived from `input.cth_like_fixed_bdy` using [`indata2json`](https://github.com/jonathanschilling/indata2json)
    1. `wout_cth_like_fixed_bdy.nc` - NetCDF output file, produced using [`PARVMEC`](https://github.com/ORNL-Fusion/PARVMEC) from `input.cth_like_fixed_bdy`, for testing the loading of a `wout` file using VMEC++'s tooling
 
-1. `input.nfp4_QH_warm_start` - quasi-helically example for use with SIMSOPT
+1. `input.nfp4_QH_warm_start` - quasi-helical example for use with SIMSOPT
 
 1. `solovev` - axisymmetric Tokamak case, similar to the Solov'ev equilibrium used in the [1983 Hirshman & Whitson article](https://doi.org/10.1063/1.864116)
     1. `input.solovev` - Fortran namelist input file for use with Fortran VMEC