Require per-state scenario sheet ID for YAML generation

rate_design/hp_rates/md/config/scenarios/scenarios_bge.yaml

@@ -0,0 +1,240 @@
+runs:
+  1:
+    run_name: md_bge_run1_up00_precalc__default__pc
+    state: MD
+    utility: bge
+    run_type: precalc
+    upgrade: '0'
+    path_tariff_maps_electric: tariff_maps/electric/bge_default.csv
+    path_tariff_maps_gas: tariff_maps/gas/bge_u00.csv
+    path_resstock_metadata: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/metadata/state=MD/upgrade=00/metadata-sb.parquet
+    path_resstock_loads: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/load_curve_hourly/state=MD/upgrade=00/
+    path_dist_and_sub_tx_mc: s3://data.sb/switchbox/marginal_costs/md/dist_and_sub_tx/utility=bge/year=2025/data.parquet
+    path_utility_assignment: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/metadata_utility/state=MD/utility_assignment.parquet
+    path_tariffs_gas: tariffs/gas
+    path_outputs: /data.sb/switchbox/cairo/outputs/hp_rates/md/bge/<execution_time>/md_bge_run1_up00_precalc__default__pc
+    path_supply_energy_mc: s3://data.sb/switchbox/marginal_costs/md/supply/energy/utility=bge/year=2025/zero.parquet
+    path_supply_capacity_mc: s3://data.sb/switchbox/marginal_costs/ny/supply/capacity/utility=bge/year=2025/zero.parquet
+    path_tariffs_electric:
+      all: tariffs/electric/bge_default.json
+    utility_revenue_requirement: rev_requirement/bge.yaml
+    run_includes_supply: false
+    run_includes_subclasses: false
+    residual_allocation_delivery: percustomer
+    residual_allocation_supply: passthrough
+    path_electric_utility_stats: s3://data.sb/eia/861/electric_utility_stats/year=2024/state=MD/data.parquet
+    path_bulk_tx_mc: s3://data.sb/switchbox/marginal_costs/md/bulk_tx/utility=bge/year=2025/data.parquet
+    solar_pv_compensation: net_metering
+    year_run: 2025
+    year_dollar_conversion: 2025
+    process_workers: 8
+    elasticity: 0.0
+  2:
+    run_name: md_bge_run2_up00_precalc_supply__default__pc
+    state: MD
+    utility: bge
+    run_type: precalc
+    upgrade: '0'
+    path_tariff_maps_electric: tariff_maps/electric/bge_default_supply.csv
+    path_tariff_maps_gas: tariff_maps/gas/bge_u00.csv
+    path_resstock_metadata: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/metadata/state=MD/upgrade=00/metadata-sb.parquet
+    path_resstock_loads: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/load_curve_hourly/state=MD/upgrade=00/
+    path_dist_and_sub_tx_mc: s3://data.sb/switchbox/marginal_costs/md/dist_and_sub_tx/utility=bge/year=2025/data.parquet
+    path_utility_assignment: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/metadata_utility/state=MD/utility_assignment.parquet
+    path_tariffs_gas: tariffs/gas
+    path_outputs: /data.sb/switchbox/cairo/outputs/hp_rates/md/bge/<execution_time>/md_bge_run2_up00_precalc_supply__default__pc
+    path_supply_energy_mc: s3://data.sb/switchbox/marginal_costs/md/supply/energy/utility=bge/year=2025/data.parquet
+    path_supply_capacity_mc: s3://data.sb/switchbox/marginal_costs/md/supply/capacity/utility=bge/year=2025/data.parquet
+    path_tariffs_electric:
+      all: tariffs/electric/bge_default_supply.json
+    utility_revenue_requirement: rev_requirement/bge.yaml
+    run_includes_supply: true
+    run_includes_subclasses: false
+    residual_allocation_delivery: percustomer
+    residual_allocation_supply: passthrough
+    path_electric_utility_stats: s3://data.sb/eia/861/electric_utility_stats/year=2024/state=MD/data.parquet
+    path_bulk_tx_mc: s3://data.sb/switchbox/marginal_costs/md/bulk_tx/utility=bge/year=2025/data.parquet
+    solar_pv_compensation: net_metering
+    year_run: 2025
+    year_dollar_conversion: 2025
+    process_workers: 8
+    elasticity: 0.0
+  3:
+    run_name: md_bge_run3_up02_default__default
+    state: MD
+    utility: bge
+    run_type: default
+    upgrade: '2'
+    path_tariff_maps_electric: tariff_maps/electric/bge_default_calibrated.csv
+    path_tariff_maps_gas: tariff_maps/gas/bge_u02.csv
+    path_resstock_metadata: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/metadata/state=MD/upgrade=02/metadata-sb.parquet
+    path_resstock_loads: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/load_curve_hourly/state=MD/upgrade=02/
+    path_dist_and_sub_tx_mc: s3://data.sb/switchbox/marginal_costs/md/dist_and_sub_tx/utility=bge/year=2025/data.parquet
+    path_utility_assignment: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/metadata_utility/state=MD/utility_assignment.parquet
+    path_tariffs_gas: tariffs/gas
+    path_outputs: /data.sb/switchbox/cairo/outputs/hp_rates/md/bge/<execution_time>/md_bge_run3_up02_default__default
+    path_supply_energy_mc: s3://data.sb/switchbox/marginal_costs/md/supply/energy/utility=bge/year=2025/zero.parquet
+    path_supply_capacity_mc: s3://data.sb/switchbox/marginal_costs/ny/supply/capacity/utility=bge/year=2025/zero.parquet
+    path_tariffs_electric:
+      all: tariffs/electric/bge_default_calibrated.json
+    utility_revenue_requirement: rev_requirement/bge_large_number.yaml
+    run_includes_supply: false
+    run_includes_subclasses: false
+    path_electric_utility_stats: s3://data.sb/eia/861/electric_utility_stats/year=2024/state=MD/data.parquet
+    path_bulk_tx_mc: s3://data.sb/switchbox/marginal_costs/md/bulk_tx/utility=bge/year=2025/data.parquet
+    solar_pv_compensation: net_metering
+    year_run: 2025
+    year_dollar_conversion: 2025
+    process_workers: 8
+    elasticity: 0.0
+  4:
+    run_name: md_bge_run4_up02_default_supply__default
+    state: MD
+    utility: bge
+    run_type: default
+    upgrade: '2'
+    path_tariff_maps_electric: tariff_maps/electric/bge_default_calibrated_supply.csv
+    path_tariff_maps_gas: tariff_maps/gas/bge_u02.csv
+    path_resstock_metadata: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/metadata/state=MD/upgrade=02/metadata-sb.parquet
+    path_resstock_loads: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/load_curve_hourly/state=MD/upgrade=02/
+    path_dist_and_sub_tx_mc: s3://data.sb/switchbox/marginal_costs/md/dist_and_sub_tx/utility=bge/year=2025/data.parquet
+    path_utility_assignment: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/metadata_utility/state=MD/utility_assignment.parquet
+    path_tariffs_gas: tariffs/gas
+    path_outputs: /data.sb/switchbox/cairo/outputs/hp_rates/md/bge/<execution_time>/md_bge_run4_up02_default_supply__default
+    path_supply_energy_mc: s3://data.sb/switchbox/marginal_costs/md/supply/energy/utility=bge/year=2025/data.parquet
+    path_supply_capacity_mc: s3://data.sb/switchbox/marginal_costs/md/supply/capacity/utility=bge/year=2025/data.parquet
+    path_tariffs_electric:
+      all: tariffs/electric/bge_default_supply_calibrated.json
+    utility_revenue_requirement: rev_requirement/bge_large_number.yaml
+    run_includes_supply: true
+    run_includes_subclasses: false
+    path_electric_utility_stats: s3://data.sb/eia/861/electric_utility_stats/year=2024/state=MD/data.parquet
+    path_bulk_tx_mc: s3://data.sb/switchbox/marginal_costs/md/bulk_tx/utility=bge/year=2025/data.parquet
+    solar_pv_compensation: net_metering
+    year_run: 2025
+    year_dollar_conversion: 2025
+    process_workers: 8
+    elasticity: 0.0
+  5:
+    run_name: md_bge_run5_up00_precalc__hp_seasonal_vs_nhpdefault__pc
+    state: MD
+    utility: bge
+    run_type: precalc
+    upgrade: '0'
+    path_tariff_maps_electric: tariff_maps/electric/bge_hp_seasonal_vs_nhpdefault.csv
+    path_tariff_maps_gas: tariff_maps/gas/bge_u00.csv
+    path_resstock_metadata: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/metadata/state=MD/upgrade=00/metadata-sb.parquet
+    path_resstock_loads: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/load_curve_hourly/state=MD/upgrade=00/
+    path_dist_and_sub_tx_mc: s3://data.sb/switchbox/marginal_costs/md/dist_and_sub_tx/utility=bge/year=2025/data.parquet
+    path_utility_assignment: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/metadata_utility/state=MD/utility_assignment.parquet
+    path_tariffs_gas: tariffs/gas
+    path_outputs: /data.sb/switchbox/cairo/outputs/hp_rates/md/bge/<execution_time>/md_bge_run5_up00_precalc__hp_seasonal_vs_nhpdefault__pc
+    path_supply_energy_mc: s3://data.sb/switchbox/marginal_costs/md/supply/energy/utility=bge/year=2025/zero.parquet
+    path_supply_capacity_mc: s3://data.sb/switchbox/marginal_costs/ny/supply/capacity/utility=bge/year=2025/zero.parquet
+    path_tariffs_electric:
+      hp: tariffs/electric/bge_hp_seasonal.json
+      non-hp: tariffs/electric/bge_nonhp_nhpdefault.json
+    utility_revenue_requirement: rev_requirement/bge_hp_vs_nonhp.yaml
+    run_includes_supply: false
+    run_includes_subclasses: true
+    residual_allocation_delivery: percustomer
+    residual_allocation_supply: passthrough
+    path_electric_utility_stats: s3://data.sb/eia/861/electric_utility_stats/year=2024/state=MD/data.parquet
+    path_bulk_tx_mc: s3://data.sb/switchbox/marginal_costs/md/bulk_tx/utility=bge/year=2025/data.parquet
+    solar_pv_compensation: net_metering
+    year_run: 2025
+    year_dollar_conversion: 2025
+    process_workers: 8
+    elasticity: 0.0
+  6:
+    run_name: md_bge_run6_up00_precalc_supply__hp_seasonal_vs_nhpdefault__pc
+    state: MD
+    utility: bge
+    run_type: precalc
+    upgrade: '0'
+    path_tariff_maps_electric: tariff_maps/electric/bge_hp_seasonal_vs_nhpdefault_supply.csv
+    path_tariff_maps_gas: tariff_maps/gas/bge_u00.csv
+    path_resstock_metadata: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/metadata/state=MD/upgrade=00/metadata-sb.parquet
+    path_resstock_loads: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/load_curve_hourly/state=MD/upgrade=00/
+    path_dist_and_sub_tx_mc: s3://data.sb/switchbox/marginal_costs/md/dist_and_sub_tx/utility=bge/year=2025/data.parquet
+    path_utility_assignment: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/metadata_utility/state=MD/utility_assignment.parquet
+    path_tariffs_gas: tariffs/gas
+    path_outputs: /data.sb/switchbox/cairo/outputs/hp_rates/md/bge/<execution_time>/md_bge_run6_up00_precalc_supply__hp_seasonal_vs_nhpdefault__pc
+    path_supply_energy_mc: s3://data.sb/switchbox/marginal_costs/md/supply/energy/utility=bge/year=2025/data.parquet
+    path_supply_capacity_mc: s3://data.sb/switchbox/marginal_costs/md/supply/capacity/utility=bge/year=2025/data.parquet
+    path_tariffs_electric:
+      hp: tariffs/electric/bge_hp_seasonal.json
+      non-hp: tariffs/electric/bge_nonhp_nhpdefault.json
+    utility_revenue_requirement: rev_requirement/bge_hp_vs_nonhp.yaml
+    run_includes_supply: true
+    run_includes_subclasses: true
+    residual_allocation_delivery: percustomer
+    residual_allocation_supply: passthrough
+    path_electric_utility_stats: s3://data.sb/eia/861/electric_utility_stats/year=2024/state=MD/data.parquet
+    path_bulk_tx_mc: s3://data.sb/switchbox/marginal_costs/md/bulk_tx/utility=bge/year=2025/data.parquet
+    solar_pv_compensation: net_metering
+    year_run: 2025
+    year_dollar_conversion: 2025
+    process_workers: 8
+    elasticity: 0.0
+  7:
+    run_name: md_bge_run7_up02_default__hp_seasonal
+    state: MD
+    utility: bge
+    run_type: default
+    upgrade: '2'
+    path_tariff_maps_electric: tariff_maps/electric/bge_hp_seasonal_calibrated.csv
+    path_tariff_maps_gas: tariff_maps/gas/bge_u02.csv
+    path_resstock_metadata: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/metadata/state=MD/upgrade=02/metadata-sb.parquet
+    path_resstock_loads: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/load_curve_hourly/state=MD/upgrade=02/
+    path_dist_and_sub_tx_mc: s3://data.sb/switchbox/marginal_costs/md/dist_and_sub_tx/utility=bge/year=2025/data.parquet
+    path_utility_assignment: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/metadata_utility/state=MD/utility_assignment.parquet
+    path_tariffs_gas: tariffs/gas
+    path_outputs: /data.sb/switchbox/cairo/outputs/hp_rates/md/bge/<execution_time>/md_bge_run7_up02_default__hp_seasonal
+    path_supply_energy_mc: s3://data.sb/switchbox/marginal_costs/md/supply/energy/utility=bge/year=2025/zero.parquet
+    path_supply_capacity_mc: s3://data.sb/switchbox/marginal_costs/ny/supply/capacity/utility=bge/year=2025/zero.parquet
+    path_tariffs_electric:
+      all: tariffs/electric/bge_hp_seasonal_calibrated.json
+    utility_revenue_requirement: rev_requirement/bge_large_number.yaml
+    run_includes_supply: false
+    run_includes_subclasses: false
+    path_electric_utility_stats: s3://data.sb/eia/861/electric_utility_stats/year=2024/state=MD/data.parquet
+    path_bulk_tx_mc: s3://data.sb/switchbox/marginal_costs/md/bulk_tx/utility=bge/year=2025/data.parquet
+    solar_pv_compensation: net_metering
+    year_run: 2025
+    year_dollar_conversion: 2025
+    process_workers: 8
+    elasticity: 0.0
+  8:
+    run_name: md_bge_run8_up02_default_supply__hp_seasonal
+    state: MD
+    utility: bge
+    run_type: default
+    upgrade: '2'
+    path_tariff_maps_electric: tariff_maps/electric/bge_hp_seasonal_calibrated_supply.csv
+    path_tariff_maps_gas: tariff_maps/gas/bge_u02.csv
+    path_resstock_metadata: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/metadata/state=MD/upgrade=02/metadata-sb.parquet
+    path_resstock_loads: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/load_curve_hourly/state=MD/upgrade=02/
+    path_dist_and_sub_tx_mc: s3://data.sb/switchbox/marginal_costs/md/dist_and_sub_tx/utility=bge/year=2025/data.parquet
+    path_utility_assignment: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/metadata_utility/state=MD/utility_assignment.parquet
+    path_tariffs_gas: tariffs/gas
+    path_outputs: /data.sb/switchbox/cairo/outputs/hp_rates/md/bge/<execution_time>/md_bge_run8_up02_default_supply__hp_seasonal
+    path_supply_energy_mc: s3://data.sb/switchbox/marginal_costs/md/supply/energy/utility=bge/year=2025/data.parquet
+    path_supply_capacity_mc: s3://data.sb/switchbox/marginal_costs/md/supply/capacity/utility=bge/year=2025/data.parquet
+    path_tariffs_electric:
+      all: tariffs/electric/bge_hp_seasonal_supply_calibrated.json
+    utility_revenue_requirement: rev_requirement/bge_large_number.yaml
+    run_includes_supply: true
+    run_includes_subclasses: false
+    path_electric_utility_stats: s3://data.sb/eia/861/electric_utility_stats/year=2024/state=MD/data.parquet
+    path_bulk_tx_mc: s3://data.sb/switchbox/marginal_costs/md/bulk_tx/utility=bge/year=2025/data.parquet
+    solar_pv_compensation: net_metering
+    year_run: 2025
+    year_dollar_conversion: 2025
+    process_workers: 8
+    elasticity: 0.0
+subclass_config:
+  group_col: has_hp
+  selectors:
+    hp: 'true'
+    non-hp: 'false'

rate_design/hp_rates/md/state.env

@@ -1,4 +1,6 @@
 STATE=md
+# Runs & Charts Google Sheet for MD scenario YAML generation (just create-scenario-yamls).
+SCENARIO_SHEET_ID=1OJyQvtD2Gb5GTEsSBMGk4VsgPiCZQE8wNQOfhOh_swg
 REGION=pjm
 YEAR=2025
 DIST_LOAD_YEAR=2025

rate_design/hp_rates/ny/state.env

@@ -1,4 +1,6 @@
 STATE=ny
+# Runs & Charts Google Sheet for NY scenario YAML generation (just create-scenario-yamls).
+SCENARIO_SHEET_ID=14naAchDw95hom88a9tdw4Y8DVxfqhx948x2UbBPujuI
 REGION=nyiso
 CAMBIUM_BA=p127
 YEAR=2025

rate_design/hp_rates/ri/state.env

@@ -1,4 +1,6 @@
 STATE=ri
+# Runs & Charts Google Sheet for RI scenario YAML generation (just create-scenario-yamls).
+SCENARIO_SHEET_ID=14naAchDw95hom88a9tdw4Y8DVxfqhx948x2UbBPujuI
 REGION=isone
 CAMBIUM_BA=p133
 YEAR=2025

tests/test_create_scenario_yamls.py

@@ -2,8 +2,8 @@
 
 from utils.pre.create_scenario_yamls import (
     _hoist_shared_subclass_config,
-    _normalize_deprecated_elec_heat_seasonal_paths,
     _parse_subclass_selectors,
+    _require_utility_for_data_row,
     _row_to_run,
 )
 
@@ -27,7 +27,7 @@ def _base_row() -> dict[str, str]:
         "path_outputs": "/data.sb/switchbox/cairo/outputs/hp_rates/ri/test",
         "path_tariffs_electric": "hp: tariffs/electric/rie_hp_seasonalTOU_supply.json, non-hp: tariffs/electric/rie_flat_supply.json",
         "utility_revenue_requirement": "rev_requirement/rie_hp_vs_nonhp.yaml",
-        "add_supply_revenue_requirement": "FALSE",
+        "run_includes_supply": "FALSE",
         "run_includes_subclasses": "TRUE",
         "subclass_group_col": "has_hp",
         "subclass_selectors": "hp:true,non-hp:false",
@@ -42,13 +42,21 @@ def _base_row() -> dict[str, str]:
 
 
 def test_row_to_run_uses_run_includes_supply() -> None:
-    """run_includes_supply is emitted (from either new or old column name)."""
+    """run_includes_supply is read from the sheet column."""
     row = _base_row()
     headers = list(row.keys())
     run = _row_to_run(row, headers)
-    assert "run_includes_supply" in run
     assert run["run_includes_supply"] is False
-    assert "add_supply_revenue_requirement" not in run
+
+
+def test_row_to_run_requires_run_includes_supply() -> None:
+    import pytest
+
+    row = _base_row()
+    row.pop("run_includes_supply")
+    headers = list(row.keys())
+    with pytest.raises(ValueError, match="run_includes_supply"):
+        _row_to_run(row, headers)
 
 
 def test_row_to_run_reads_run_includes_subclasses_from_sheet() -> None:
@@ -108,51 +116,37 @@ def test_row_to_run_omits_path_tou_supply_mc_when_blank() -> None:
     assert "path_tou_supply_mc" not in run
 
 
-def test_normalize_deprecated_elec_heat_seasonal_paths() -> None:
-    assert _normalize_deprecated_elec_heat_seasonal_paths("") == ""
-    assert (
-        _normalize_deprecated_elec_heat_seasonal_paths(
-            "tariff_maps/electric/u_elec_heat_seasonal_epmc_vs_non_elec_heat_supply.csv"
-        )
-        == "tariff_maps/electric/u_elec_heat_seasonal_vs_non_elec_heat_supply.csv"
+def test_require_utility_for_data_row_accepts_valid_utility() -> None:
+    row = {"state": "NY", "utility": "coned", "num": "1"}
+    _require_utility_for_data_row(
+        row, state_key="state", utility_key="utility", num_key="num"
     )
-    assert (
-        _normalize_deprecated_elec_heat_seasonal_paths(
-            "tariffs/electric/u_elec_heat_seasonal_epmc_supply_calibrated.json"
+
+
+def test_require_utility_for_data_row_raises_on_blank_utility() -> None:
+    import pytest
+
+    row = {"state": "NY", "utility": "", "num": "5"}
+    with pytest.raises(ValueError, match="blank or invalid utility"):
+        _require_utility_for_data_row(
+            row, state_key="state", utility_key="utility", num_key="num"
         )
-        == "tariffs/electric/u_elec_heat_seasonal_supply_calibrated.json"
-    )
-    # hp_seasonal_epmc paths must be unchanged
-    assert (
-        _normalize_deprecated_elec_heat_seasonal_paths(
-            "tariffs/electric/u_hp_seasonal_epmc.json"
+
+
+def test_require_utility_for_data_row_raises_on_numeric_utility() -> None:
+    import pytest
+
+    row = {"state": "RI", "utility": "123", "num": "2"}
+    with pytest.raises(ValueError, match="blank or invalid utility"):
+        _require_utility_for_data_row(
+            row, state_key="state", utility_key="utility", num_key="num"
         )
-        == "tariffs/electric/u_hp_seasonal_epmc.json"
-    )
 
 
-def test_row_to_run_normalizes_deprecated_elec_heat_seasonal_sheet_paths() -> None:
-    """Runs 33–36 sheet rows may still reference *_elec_heat_seasonal_epmc* files."""
-    row = _base_row()
-    row["path_tariff_maps_electric"] = (
-        "tariff_maps/electric/rie_elec_heat_seasonal_epmc_vs_non_elec_heat.csv"
-    )
-    row["path_tariffs_electric"] = (
-        "electric_heating: tariffs/electric/rie_elec_heat_seasonal_epmc.json, "
-        "non_electric_heating: tariffs/electric/rie_non_electric_heating.json"
-    )
-    headers = list(row.keys())
-    run = _row_to_run(row, headers)
-    assert (
-        run["path_tariff_maps_electric"]
-        == "tariff_maps/electric/rie_elec_heat_seasonal_vs_non_elec_heat.csv"
-    )
-    path_tariffs_electric = cast(dict[str, str], run["path_tariffs_electric"])
-    assert path_tariffs_electric["electric_heating"] == (
-        "tariffs/electric/rie_elec_heat_seasonal.json"
-    )
-    assert path_tariffs_electric["non_electric_heating"] == (
-        "tariffs/electric/rie_non_electric_heating.json"
+def test_require_utility_for_data_row_skips_rows_without_state() -> None:
+    row = {"state": "", "utility": "", "num": "1"}
+    _require_utility_for_data_row(
+        row, state_key="state", utility_key="utility", num_key="num"
     )