Capture optimizer ESDLs as CI artifacts to update ESDL references

.github/workflows/ci.yml

@@ -19,3 +19,10 @@ jobs:
     - name: run system tests
       run: |
         ./scripts/test_system.sh
+    - name: upload optimizer output ESDLs
+      if: always()
+      uses: actions/upload-artifact@v4
+      with:
+        name: optimizer-output-esdls-py${{ matrix.python-version }}
+        path: system_tests_artifacts/
+        if-no-files-found: warn

.gitignore

@@ -219,4 +219,5 @@ fabric.properties
 
 unit_test_coverage/
 .env.test
-gurobi/
+gurobi/
+system_tests_artifacts/

system_tests/docker-compose.override.yml

@@ -3,6 +3,8 @@ services:
     build: system_tests/ # This is referenced from root of repo due to how docker-compose works.
     networks:
       - omotes
+    volumes:
+      - ./system_tests_artifacts:/app/test_esdl/artifacts
     depends_on:
       rabbitmq:
         condition: service_healthy

system_tests/src/test_workflows_steps.py

@@ -62,6 +62,18 @@
 
 ESDL_VALUES_PRECISION = 1e-6
 
+ARTIFACTS_DIR = Path(os.environ.get("OPTIMIZER_ARTIFACTS_DIR", "/app/test_esdl/artifacts"))
+
+
+def save_optimizer_output(test_name: str, output_esdl: str) -> None:
+    """Write the optimizer output ESDL to the artifacts directory for CI upload."""
+    try:
+        ARTIFACTS_DIR.mkdir(parents=True, exist_ok=True)
+        (ARTIFACTS_DIR / f"{test_name}.esdl").write_text(output_esdl, encoding="utf-8")
+    except OSError as exc:
+        print(f"[{test_name}] WARNING: could not save artifact: {exc}", flush=True)
+
+
 class OmotesJobHandler:
     progress_updates: list[JobProgressUpdate]
     status_updates: list[JobStatusUpdate]
@@ -266,6 +278,8 @@ def test__grow_optimizer_default__happy_path(self) -> None:
 
         # Assert
         self.expect_a_result(result_handler, JobResult.SUCCEEDED)
+        if result_handler.result.output_esdl:
+            save_optimizer_output("test__grow_optimizer_default__happy_path", result_handler.result.output_esdl)
         expected_esdl = retrieve_esdl_file(
             "./test_esdl/output/test__grow_optimizer_default__happy_path.esdl"
         )
@@ -288,6 +302,8 @@ def test__grow_optimizer_no_heat_losses__happy_path(self) -> None:
 
         # Assert
         self.expect_a_result(result_handler, JobResult.SUCCEEDED)
+        if result_handler.result.output_esdl:
+            save_optimizer_output("test__grow_optimizer_no_heat_losses__happy_path", result_handler.result.output_esdl)
         expected_esdl = retrieve_esdl_file(
             "./test_esdl/output/test__grow_optimizer_no_heat_losses__happy_path.esdl"
         )
@@ -390,6 +406,8 @@ def test__grow_optimizer_default__happy_path_1source(self) -> None:
 
         # Assert
         self.expect_a_result(result_handler, JobResult.SUCCEEDED)
+        if result_handler.result.output_esdl:
+            save_optimizer_output("test__grow_optimizer_default__happy_path_1source", result_handler.result.output_esdl)
         expected_esdl = retrieve_esdl_file(
             "./test_esdl/output/test__grow_optimizer_default__happy_path_1source.esdl"
         )
@@ -414,6 +432,8 @@ def test__grow_optimizer_default__happy_path_2ndsource(self) -> None:
 
         # Assert
         self.expect_a_result(result_handler, JobResult.SUCCEEDED)
+        if result_handler.result.output_esdl:
+            save_optimizer_output("test__grow_optimizer_default__happy_path_2ndsource", result_handler.result.output_esdl)
         expected_esdl = retrieve_esdl_file(
             "./test_esdl/output/test__grow_optimizer_default__happy_path_2ndsource.esdl"
         )
@@ -440,6 +460,8 @@ def test__grow_optimizer_default__happy_path_2ndsource_merit_order_swapped(
 
         # Assert
         self.expect_a_result(result_handler, JobResult.SUCCEEDED)
+        if result_handler.result.output_esdl:
+            save_optimizer_output("test__grow_optimizer_default__happy_path_2ndsource_merit_order_swapped", result_handler.result.output_esdl)
         expected_esdl = retrieve_esdl_file(
             "./test_esdl/output/test__grow_optimizer_default__happy_path_2ndsource_merit_order_swapped.esdl"
         )
@@ -683,7 +705,7 @@ def wait_for_all_jobs(timeout_seconds: float) -> None:
             def _watch_job(result_handler: OmotesJobHandler):
                 result_handler.wait_until_result(timeout_seconds)
                 with result_ids_lock:
-                    ordered_job_result_ids.append(result_handler.result.uuid)
+                    ordered_job_result_ids.append(str(result_handler.result.uuid))
                 with condition:
                     condition.notify_all()