diff --git a/rate_design/hp_rates/md/config/scenarios/scenarios_bge.yaml b/rate_design/hp_rates/md/config/scenarios/scenarios_bge.yaml new file mode 100644 index 00000000..28746f20 --- /dev/null +++ b/rate_design/hp_rates/md/config/scenarios/scenarios_bge.yaml @@ -0,0 +1,240 @@ +runs: + 1: + run_name: md_bge_run1_up00_precalc__default__pc + state: MD + utility: bge + run_type: precalc + upgrade: '0' + path_tariff_maps_electric: tariff_maps/electric/bge_default.csv + path_tariff_maps_gas: tariff_maps/gas/bge_u00.csv + path_resstock_metadata: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/metadata/state=MD/upgrade=00/metadata-sb.parquet + path_resstock_loads: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/load_curve_hourly/state=MD/upgrade=00/ + path_dist_and_sub_tx_mc: s3://data.sb/switchbox/marginal_costs/md/dist_and_sub_tx/utility=bge/year=2025/data.parquet + path_utility_assignment: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/metadata_utility/state=MD/utility_assignment.parquet + path_tariffs_gas: tariffs/gas + path_outputs: /data.sb/switchbox/cairo/outputs/hp_rates/md/bge//md_bge_run1_up00_precalc__default__pc + path_supply_energy_mc: s3://data.sb/switchbox/marginal_costs/md/supply/energy/utility=bge/year=2025/zero.parquet + path_supply_capacity_mc: s3://data.sb/switchbox/marginal_costs/ny/supply/capacity/utility=bge/year=2025/zero.parquet + path_tariffs_electric: + all: tariffs/electric/bge_default.json + utility_revenue_requirement: rev_requirement/bge.yaml + run_includes_supply: false + run_includes_subclasses: false + residual_allocation_delivery: percustomer + residual_allocation_supply: passthrough + path_electric_utility_stats: s3://data.sb/eia/861/electric_utility_stats/year=2024/state=MD/data.parquet + path_bulk_tx_mc: s3://data.sb/switchbox/marginal_costs/md/bulk_tx/utility=bge/year=2025/data.parquet + solar_pv_compensation: net_metering + year_run: 2025 + year_dollar_conversion: 2025 + process_workers: 8 + elasticity: 0.0 + 2: + run_name: md_bge_run2_up00_precalc_supply__default__pc + state: MD + utility: bge + run_type: precalc + upgrade: '0' + path_tariff_maps_electric: tariff_maps/electric/bge_default_supply.csv + path_tariff_maps_gas: tariff_maps/gas/bge_u00.csv + path_resstock_metadata: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/metadata/state=MD/upgrade=00/metadata-sb.parquet + path_resstock_loads: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/load_curve_hourly/state=MD/upgrade=00/ + path_dist_and_sub_tx_mc: s3://data.sb/switchbox/marginal_costs/md/dist_and_sub_tx/utility=bge/year=2025/data.parquet + path_utility_assignment: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/metadata_utility/state=MD/utility_assignment.parquet + path_tariffs_gas: tariffs/gas + path_outputs: /data.sb/switchbox/cairo/outputs/hp_rates/md/bge//md_bge_run2_up00_precalc_supply__default__pc + path_supply_energy_mc: s3://data.sb/switchbox/marginal_costs/md/supply/energy/utility=bge/year=2025/data.parquet + path_supply_capacity_mc: s3://data.sb/switchbox/marginal_costs/md/supply/capacity/utility=bge/year=2025/data.parquet + path_tariffs_electric: + all: tariffs/electric/bge_default_supply.json + utility_revenue_requirement: rev_requirement/bge.yaml + run_includes_supply: true + run_includes_subclasses: false + residual_allocation_delivery: percustomer + residual_allocation_supply: passthrough + path_electric_utility_stats: s3://data.sb/eia/861/electric_utility_stats/year=2024/state=MD/data.parquet + path_bulk_tx_mc: s3://data.sb/switchbox/marginal_costs/md/bulk_tx/utility=bge/year=2025/data.parquet + solar_pv_compensation: net_metering + year_run: 2025 + year_dollar_conversion: 2025 + process_workers: 8 + elasticity: 0.0 + 3: + run_name: md_bge_run3_up02_default__default + state: MD + utility: bge + run_type: default + upgrade: '2' + path_tariff_maps_electric: tariff_maps/electric/bge_default_calibrated.csv + path_tariff_maps_gas: tariff_maps/gas/bge_u02.csv + path_resstock_metadata: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/metadata/state=MD/upgrade=02/metadata-sb.parquet + path_resstock_loads: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/load_curve_hourly/state=MD/upgrade=02/ + path_dist_and_sub_tx_mc: s3://data.sb/switchbox/marginal_costs/md/dist_and_sub_tx/utility=bge/year=2025/data.parquet + path_utility_assignment: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/metadata_utility/state=MD/utility_assignment.parquet + path_tariffs_gas: tariffs/gas + path_outputs: /data.sb/switchbox/cairo/outputs/hp_rates/md/bge//md_bge_run3_up02_default__default + path_supply_energy_mc: s3://data.sb/switchbox/marginal_costs/md/supply/energy/utility=bge/year=2025/zero.parquet + path_supply_capacity_mc: s3://data.sb/switchbox/marginal_costs/ny/supply/capacity/utility=bge/year=2025/zero.parquet + path_tariffs_electric: + all: tariffs/electric/bge_default_calibrated.json + utility_revenue_requirement: rev_requirement/bge_large_number.yaml + run_includes_supply: false + run_includes_subclasses: false + path_electric_utility_stats: s3://data.sb/eia/861/electric_utility_stats/year=2024/state=MD/data.parquet + path_bulk_tx_mc: s3://data.sb/switchbox/marginal_costs/md/bulk_tx/utility=bge/year=2025/data.parquet + solar_pv_compensation: net_metering + year_run: 2025 + year_dollar_conversion: 2025 + process_workers: 8 + elasticity: 0.0 + 4: + run_name: md_bge_run4_up02_default_supply__default + state: MD + utility: bge + run_type: default + upgrade: '2' + path_tariff_maps_electric: tariff_maps/electric/bge_default_calibrated_supply.csv + path_tariff_maps_gas: tariff_maps/gas/bge_u02.csv + path_resstock_metadata: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/metadata/state=MD/upgrade=02/metadata-sb.parquet + path_resstock_loads: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/load_curve_hourly/state=MD/upgrade=02/ + path_dist_and_sub_tx_mc: s3://data.sb/switchbox/marginal_costs/md/dist_and_sub_tx/utility=bge/year=2025/data.parquet + path_utility_assignment: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/metadata_utility/state=MD/utility_assignment.parquet + path_tariffs_gas: tariffs/gas + path_outputs: /data.sb/switchbox/cairo/outputs/hp_rates/md/bge//md_bge_run4_up02_default_supply__default + path_supply_energy_mc: s3://data.sb/switchbox/marginal_costs/md/supply/energy/utility=bge/year=2025/data.parquet + path_supply_capacity_mc: s3://data.sb/switchbox/marginal_costs/md/supply/capacity/utility=bge/year=2025/data.parquet + path_tariffs_electric: + all: tariffs/electric/bge_default_supply_calibrated.json + utility_revenue_requirement: rev_requirement/bge_large_number.yaml + run_includes_supply: true + run_includes_subclasses: false + path_electric_utility_stats: s3://data.sb/eia/861/electric_utility_stats/year=2024/state=MD/data.parquet + path_bulk_tx_mc: s3://data.sb/switchbox/marginal_costs/md/bulk_tx/utility=bge/year=2025/data.parquet + solar_pv_compensation: net_metering + year_run: 2025 + year_dollar_conversion: 2025 + process_workers: 8 + elasticity: 0.0 + 5: + run_name: md_bge_run5_up00_precalc__hp_seasonal_vs_nhpdefault__pc + state: MD + utility: bge + run_type: precalc + upgrade: '0' + path_tariff_maps_electric: tariff_maps/electric/bge_hp_seasonal_vs_nhpdefault.csv + path_tariff_maps_gas: tariff_maps/gas/bge_u00.csv + path_resstock_metadata: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/metadata/state=MD/upgrade=00/metadata-sb.parquet + path_resstock_loads: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/load_curve_hourly/state=MD/upgrade=00/ + path_dist_and_sub_tx_mc: s3://data.sb/switchbox/marginal_costs/md/dist_and_sub_tx/utility=bge/year=2025/data.parquet + path_utility_assignment: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/metadata_utility/state=MD/utility_assignment.parquet + path_tariffs_gas: tariffs/gas + path_outputs: /data.sb/switchbox/cairo/outputs/hp_rates/md/bge//md_bge_run5_up00_precalc__hp_seasonal_vs_nhpdefault__pc + path_supply_energy_mc: s3://data.sb/switchbox/marginal_costs/md/supply/energy/utility=bge/year=2025/zero.parquet + path_supply_capacity_mc: s3://data.sb/switchbox/marginal_costs/ny/supply/capacity/utility=bge/year=2025/zero.parquet + path_tariffs_electric: + hp: tariffs/electric/bge_hp_seasonal.json + non-hp: tariffs/electric/bge_nonhp_nhpdefault.json + utility_revenue_requirement: rev_requirement/bge_hp_vs_nonhp.yaml + run_includes_supply: false + run_includes_subclasses: true + residual_allocation_delivery: percustomer + residual_allocation_supply: passthrough + path_electric_utility_stats: s3://data.sb/eia/861/electric_utility_stats/year=2024/state=MD/data.parquet + path_bulk_tx_mc: s3://data.sb/switchbox/marginal_costs/md/bulk_tx/utility=bge/year=2025/data.parquet + solar_pv_compensation: net_metering + year_run: 2025 + year_dollar_conversion: 2025 + process_workers: 8 + elasticity: 0.0 + 6: + run_name: md_bge_run6_up00_precalc_supply__hp_seasonal_vs_nhpdefault__pc + state: MD + utility: bge + run_type: precalc + upgrade: '0' + path_tariff_maps_electric: tariff_maps/electric/bge_hp_seasonal_vs_nhpdefault_supply.csv + path_tariff_maps_gas: tariff_maps/gas/bge_u00.csv + path_resstock_metadata: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/metadata/state=MD/upgrade=00/metadata-sb.parquet + path_resstock_loads: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/load_curve_hourly/state=MD/upgrade=00/ + path_dist_and_sub_tx_mc: s3://data.sb/switchbox/marginal_costs/md/dist_and_sub_tx/utility=bge/year=2025/data.parquet + path_utility_assignment: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/metadata_utility/state=MD/utility_assignment.parquet + path_tariffs_gas: tariffs/gas + path_outputs: /data.sb/switchbox/cairo/outputs/hp_rates/md/bge//md_bge_run6_up00_precalc_supply__hp_seasonal_vs_nhpdefault__pc + path_supply_energy_mc: s3://data.sb/switchbox/marginal_costs/md/supply/energy/utility=bge/year=2025/data.parquet + path_supply_capacity_mc: s3://data.sb/switchbox/marginal_costs/md/supply/capacity/utility=bge/year=2025/data.parquet + path_tariffs_electric: + hp: tariffs/electric/bge_hp_seasonal.json + non-hp: tariffs/electric/bge_nonhp_nhpdefault.json + utility_revenue_requirement: rev_requirement/bge_hp_vs_nonhp.yaml + run_includes_supply: true + run_includes_subclasses: true + residual_allocation_delivery: percustomer + residual_allocation_supply: passthrough + path_electric_utility_stats: s3://data.sb/eia/861/electric_utility_stats/year=2024/state=MD/data.parquet + path_bulk_tx_mc: s3://data.sb/switchbox/marginal_costs/md/bulk_tx/utility=bge/year=2025/data.parquet + solar_pv_compensation: net_metering + year_run: 2025 + year_dollar_conversion: 2025 + process_workers: 8 + elasticity: 0.0 + 7: + run_name: md_bge_run7_up02_default__hp_seasonal + state: MD + utility: bge + run_type: default + upgrade: '2' + path_tariff_maps_electric: tariff_maps/electric/bge_hp_seasonal_calibrated.csv + path_tariff_maps_gas: tariff_maps/gas/bge_u02.csv + path_resstock_metadata: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/metadata/state=MD/upgrade=02/metadata-sb.parquet + path_resstock_loads: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/load_curve_hourly/state=MD/upgrade=02/ + path_dist_and_sub_tx_mc: s3://data.sb/switchbox/marginal_costs/md/dist_and_sub_tx/utility=bge/year=2025/data.parquet + path_utility_assignment: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/metadata_utility/state=MD/utility_assignment.parquet + path_tariffs_gas: tariffs/gas + path_outputs: /data.sb/switchbox/cairo/outputs/hp_rates/md/bge//md_bge_run7_up02_default__hp_seasonal + path_supply_energy_mc: s3://data.sb/switchbox/marginal_costs/md/supply/energy/utility=bge/year=2025/zero.parquet + path_supply_capacity_mc: s3://data.sb/switchbox/marginal_costs/ny/supply/capacity/utility=bge/year=2025/zero.parquet + path_tariffs_electric: + all: tariffs/electric/bge_hp_seasonal_calibrated.json + utility_revenue_requirement: rev_requirement/bge_large_number.yaml + run_includes_supply: false + run_includes_subclasses: false + path_electric_utility_stats: s3://data.sb/eia/861/electric_utility_stats/year=2024/state=MD/data.parquet + path_bulk_tx_mc: s3://data.sb/switchbox/marginal_costs/md/bulk_tx/utility=bge/year=2025/data.parquet + solar_pv_compensation: net_metering + year_run: 2025 + year_dollar_conversion: 2025 + process_workers: 8 + elasticity: 0.0 + 8: + run_name: md_bge_run8_up02_default_supply__hp_seasonal + state: MD + utility: bge + run_type: default + upgrade: '2' + path_tariff_maps_electric: tariff_maps/electric/bge_hp_seasonal_calibrated_supply.csv + path_tariff_maps_gas: tariff_maps/gas/bge_u02.csv + path_resstock_metadata: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/metadata/state=MD/upgrade=02/metadata-sb.parquet + path_resstock_loads: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/load_curve_hourly/state=MD/upgrade=02/ + path_dist_and_sub_tx_mc: s3://data.sb/switchbox/marginal_costs/md/dist_and_sub_tx/utility=bge/year=2025/data.parquet + path_utility_assignment: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/metadata_utility/state=MD/utility_assignment.parquet + path_tariffs_gas: tariffs/gas + path_outputs: /data.sb/switchbox/cairo/outputs/hp_rates/md/bge//md_bge_run8_up02_default_supply__hp_seasonal + path_supply_energy_mc: s3://data.sb/switchbox/marginal_costs/md/supply/energy/utility=bge/year=2025/data.parquet + path_supply_capacity_mc: s3://data.sb/switchbox/marginal_costs/md/supply/capacity/utility=bge/year=2025/data.parquet + path_tariffs_electric: + all: tariffs/electric/bge_hp_seasonal_supply_calibrated.json + utility_revenue_requirement: rev_requirement/bge_large_number.yaml + run_includes_supply: true + run_includes_subclasses: false + path_electric_utility_stats: s3://data.sb/eia/861/electric_utility_stats/year=2024/state=MD/data.parquet + path_bulk_tx_mc: s3://data.sb/switchbox/marginal_costs/md/bulk_tx/utility=bge/year=2025/data.parquet + solar_pv_compensation: net_metering + year_run: 2025 + year_dollar_conversion: 2025 + process_workers: 8 + elasticity: 0.0 +subclass_config: + group_col: has_hp + selectors: + hp: 'true' + non-hp: 'false' diff --git a/rate_design/hp_rates/md/state.env b/rate_design/hp_rates/md/state.env index e78229ef..07421743 100644 --- a/rate_design/hp_rates/md/state.env +++ b/rate_design/hp_rates/md/state.env @@ -1,4 +1,6 @@ STATE=md +# Runs & Charts Google Sheet for MD scenario YAML generation (just create-scenario-yamls). +SCENARIO_SHEET_ID=1OJyQvtD2Gb5GTEsSBMGk4VsgPiCZQE8wNQOfhOh_swg REGION=pjm YEAR=2025 DIST_LOAD_YEAR=2025 diff --git a/rate_design/hp_rates/ny/state.env b/rate_design/hp_rates/ny/state.env index cf8d34db..6c813671 100644 --- a/rate_design/hp_rates/ny/state.env +++ b/rate_design/hp_rates/ny/state.env @@ -1,4 +1,6 @@ STATE=ny +# Runs & Charts Google Sheet for NY scenario YAML generation (just create-scenario-yamls). +SCENARIO_SHEET_ID=14naAchDw95hom88a9tdw4Y8DVxfqhx948x2UbBPujuI REGION=nyiso CAMBIUM_BA=p127 YEAR=2025 diff --git a/rate_design/hp_rates/ri/state.env b/rate_design/hp_rates/ri/state.env index 34a05286..806294c9 100644 --- a/rate_design/hp_rates/ri/state.env +++ b/rate_design/hp_rates/ri/state.env @@ -1,4 +1,6 @@ STATE=ri +# Runs & Charts Google Sheet for RI scenario YAML generation (just create-scenario-yamls). +SCENARIO_SHEET_ID=14naAchDw95hom88a9tdw4Y8DVxfqhx948x2UbBPujuI REGION=isone CAMBIUM_BA=p133 YEAR=2025 diff --git a/tests/test_create_scenario_yamls.py b/tests/test_create_scenario_yamls.py index 08d9de72..68a5df4b 100644 --- a/tests/test_create_scenario_yamls.py +++ b/tests/test_create_scenario_yamls.py @@ -2,8 +2,8 @@ from utils.pre.create_scenario_yamls import ( _hoist_shared_subclass_config, - _normalize_deprecated_elec_heat_seasonal_paths, _parse_subclass_selectors, + _require_utility_for_data_row, _row_to_run, ) @@ -27,7 +27,7 @@ def _base_row() -> dict[str, str]: "path_outputs": "/data.sb/switchbox/cairo/outputs/hp_rates/ri/test", "path_tariffs_electric": "hp: tariffs/electric/rie_hp_seasonalTOU_supply.json, non-hp: tariffs/electric/rie_flat_supply.json", "utility_revenue_requirement": "rev_requirement/rie_hp_vs_nonhp.yaml", - "add_supply_revenue_requirement": "FALSE", + "run_includes_supply": "FALSE", "run_includes_subclasses": "TRUE", "subclass_group_col": "has_hp", "subclass_selectors": "hp:true,non-hp:false", @@ -42,13 +42,21 @@ def _base_row() -> dict[str, str]: def test_row_to_run_uses_run_includes_supply() -> None: - """run_includes_supply is emitted (from either new or old column name).""" + """run_includes_supply is read from the sheet column.""" row = _base_row() headers = list(row.keys()) run = _row_to_run(row, headers) - assert "run_includes_supply" in run assert run["run_includes_supply"] is False - assert "add_supply_revenue_requirement" not in run + + +def test_row_to_run_requires_run_includes_supply() -> None: + import pytest + + row = _base_row() + row.pop("run_includes_supply") + headers = list(row.keys()) + with pytest.raises(ValueError, match="run_includes_supply"): + _row_to_run(row, headers) def test_row_to_run_reads_run_includes_subclasses_from_sheet() -> None: @@ -108,51 +116,37 @@ def test_row_to_run_omits_path_tou_supply_mc_when_blank() -> None: assert "path_tou_supply_mc" not in run -def test_normalize_deprecated_elec_heat_seasonal_paths() -> None: - assert _normalize_deprecated_elec_heat_seasonal_paths("") == "" - assert ( - _normalize_deprecated_elec_heat_seasonal_paths( - "tariff_maps/electric/u_elec_heat_seasonal_epmc_vs_non_elec_heat_supply.csv" - ) - == "tariff_maps/electric/u_elec_heat_seasonal_vs_non_elec_heat_supply.csv" +def test_require_utility_for_data_row_accepts_valid_utility() -> None: + row = {"state": "NY", "utility": "coned", "num": "1"} + _require_utility_for_data_row( + row, state_key="state", utility_key="utility", num_key="num" ) - assert ( - _normalize_deprecated_elec_heat_seasonal_paths( - "tariffs/electric/u_elec_heat_seasonal_epmc_supply_calibrated.json" + + +def test_require_utility_for_data_row_raises_on_blank_utility() -> None: + import pytest + + row = {"state": "NY", "utility": "", "num": "5"} + with pytest.raises(ValueError, match="blank or invalid utility"): + _require_utility_for_data_row( + row, state_key="state", utility_key="utility", num_key="num" ) - == "tariffs/electric/u_elec_heat_seasonal_supply_calibrated.json" - ) - # hp_seasonal_epmc paths must be unchanged - assert ( - _normalize_deprecated_elec_heat_seasonal_paths( - "tariffs/electric/u_hp_seasonal_epmc.json" + + +def test_require_utility_for_data_row_raises_on_numeric_utility() -> None: + import pytest + + row = {"state": "RI", "utility": "123", "num": "2"} + with pytest.raises(ValueError, match="blank or invalid utility"): + _require_utility_for_data_row( + row, state_key="state", utility_key="utility", num_key="num" ) - == "tariffs/electric/u_hp_seasonal_epmc.json" - ) -def test_row_to_run_normalizes_deprecated_elec_heat_seasonal_sheet_paths() -> None: - """Runs 33–36 sheet rows may still reference *_elec_heat_seasonal_epmc* files.""" - row = _base_row() - row["path_tariff_maps_electric"] = ( - "tariff_maps/electric/rie_elec_heat_seasonal_epmc_vs_non_elec_heat.csv" - ) - row["path_tariffs_electric"] = ( - "electric_heating: tariffs/electric/rie_elec_heat_seasonal_epmc.json, " - "non_electric_heating: tariffs/electric/rie_non_electric_heating.json" - ) - headers = list(row.keys()) - run = _row_to_run(row, headers) - assert ( - run["path_tariff_maps_electric"] - == "tariff_maps/electric/rie_elec_heat_seasonal_vs_non_elec_heat.csv" - ) - path_tariffs_electric = cast(dict[str, str], run["path_tariffs_electric"]) - assert path_tariffs_electric["electric_heating"] == ( - "tariffs/electric/rie_elec_heat_seasonal.json" - ) - assert path_tariffs_electric["non_electric_heating"] == ( - "tariffs/electric/rie_non_electric_heating.json" +def test_require_utility_for_data_row_skips_rows_without_state() -> None: + row = {"state": "", "utility": "", "num": "1"} + _require_utility_for_data_row( + row, state_key="state", utility_key="utility", num_key="num" ) diff --git a/utils/Justfile b/utils/Justfile index 3dc265df..a1e37ec0 100644 --- a/utils/Justfile +++ b/utils/Justfile @@ -8,9 +8,11 @@ project_root := `git rev-parse --show-toplevel` # Create scenario YAMLs from the Runs & Charts Google Sheet (requires G_* env and browser auth). # Writes rate_design/hp_rates//config/scenarios/scenarios_.yaml. -# Pass state="" to generate for all states. -create-scenario-yamls state="": +# Pass state="" to generate for all states in the sheet (requires SCENARIO_SHEET_ID). +# Prefer: just s create-scenario-yamls (sources state.env per state). +create-scenario-yamls state="" sheet_id=env_var_or_default('SCENARIO_SHEET_ID', ''): uv run python {{ project_root }}/utils/pre/create_scenario_yamls.py \ + {{ if sheet_id != "" { "--sheet-id " + sheet_id } else { "" } }} \ {{ if state != "" { "--state " + state } else { "" } }} # Update Google Sheet with MC path formulas for all states (NY + RI). diff --git a/utils/pre/create_scenario_yamls.py b/utils/pre/create_scenario_yamls.py index 55dcbf35..7bb82be3 100644 --- a/utils/pre/create_scenario_yamls.py +++ b/utils/pre/create_scenario_yamls.py @@ -7,16 +7,16 @@ -------------------------------------- path_supply_energy_mc (required): - NY supply runs (add_supply_revenue_requirement=TRUE): + NY supply runs (run_includes_supply=TRUE): "s3://data.sb/switchbox/marginal_costs/ny/supply/energy/utility=" & LOWER($B18) & "/year=2025/data.parquet" NY delivery-only runs: "s3://data.sb/switchbox/marginal_costs/ny/supply/energy/utility=" & LOWER($B18) & "/year=2025/zero.parquet" - RI supply runs (add_supply_revenue_requirement=TRUE): + RI supply runs (run_includes_supply=TRUE): "s3://data.sb/switchbox/marginal_costs/ri/supply/energy/utility=" & LOWER($B18) & "/year=2025/data.parquet" RI delivery-only runs: "s3://data.sb/switchbox/marginal_costs/ri/supply/energy/utility=" & LOWER($B18) & "/year=2025/zero.parquet" - Full formula (where E18 = add_supply_revenue_requirement column, X = TRUE): + Full formula (where E18 = run_includes_supply column, X = TRUE): =IF(AND($A18="NY", E18="X"), "s3://data.sb/switchbox/marginal_costs/ny/supply/energy/utility=" & LOWER($B18) & "/year=2025/data.parquet", IF(AND($A18="NY", E18<>"X"), @@ -29,16 +29,16 @@ Zero parquets are placeholders for delivery-only runs where supply MCs are not needed. path_supply_capacity_mc (required): - NY supply runs (add_supply_revenue_requirement=TRUE): + NY supply runs (run_includes_supply=TRUE): "s3://data.sb/switchbox/marginal_costs/ny/supply/capacity/utility=" & LOWER($B18) & "/year=2025/data.parquet" NY delivery-only runs: "s3://data.sb/switchbox/marginal_costs/ny/supply/capacity/utility=" & LOWER($B18) & "/year=2025/zero.parquet" - RI supply runs (add_supply_revenue_requirement=TRUE): + RI supply runs (run_includes_supply=TRUE): "s3://data.sb/switchbox/marginal_costs/ri/supply/capacity/utility=" & LOWER($B18) & "/year=2025/data.parquet" RI delivery-only runs: "s3://data.sb/switchbox/marginal_costs/ri/supply/capacity/utility=" & LOWER($B18) & "/year=2025/zero.parquet" - Full formula (where E18 = add_supply_revenue_requirement column, X = TRUE): + Full formula (where E18 = run_includes_supply column, X = TRUE): =IF(AND($A18="NY", E18="X"), "s3://data.sb/switchbox/marginal_costs/ny/supply/capacity/utility=" & LOWER($B18) & "/year=2025/data.parquet", IF(AND($A18="NY", E18<>"X"), @@ -53,14 +53,14 @@ NY uses separate NYISO LBMP + ICAP parquets for supply runs, and zero-filled parquets for delivery-only runs. path_supply_ancillary_mc (optional): - RI supply runs (add_supply_revenue_requirement=TRUE): + RI supply runs (run_includes_supply=TRUE): "s3://data.sb/switchbox/marginal_costs/ri/supply/ancillary/utility=" & LOWER($B18) & "/year=2025/data.parquet" RI delivery-only runs: "" (empty, not used) NY runs: "" (empty, not used) - Full formula (where E18 = add_supply_revenue_requirement column, X = TRUE): + Full formula (where E18 = run_includes_supply column, X = TRUE): =IF(AND($A18="RI", E18="X"), "s3://data.sb/switchbox/marginal_costs/ri/supply/ancillary/utility=" & LOWER($B18) & "/year=2025/data.parquet", "") @@ -110,14 +110,8 @@ Where: - $C18 is the utility column - After updating the Google Sheet, run: just create-scenario-yamls - - Electric heat seasonal tariff filenames (NY runs 33–36): - Repo tariffs/maps were renamed from ``*_elec_heat_seasonal_epmc*`` to - ``*_elec_heat_seasonal*``. The sheet may still contain the old token; - ``create_scenario_yamls`` normalizes those path strings when writing YAML - so CAIRO resolves existing files. Prefer updating sheet formulas to the - new names when convenient. + After updating the Google Sheet, run: just s create-scenario-yamls + (reads SCENARIO_SHEET_ID from rate_design/hp_rates//state.env). """ from __future__ import annotations @@ -146,7 +140,18 @@ def _gspread_token_path() -> Path: return base / "gspread" / "authorized_user_rate_design.json" -DEFAULT_SHEET_ID = "14naAchDw95hom88a9tdw4Y8DVxfqhx948x2UbBPujuI" +SCENARIO_SHEET_ID_ENV = "SCENARIO_SHEET_ID" + + +def _resolve_sheet_id(sheet_id: str | None) -> str: + """Return sheet ID from CLI arg or SCENARIO_SHEET_ID env var.""" + resolved = (sheet_id or os.environ.get(SCENARIO_SHEET_ID_ENV) or "").strip() + if not resolved: + raise SystemExit( + "Google Spreadsheet ID is required. Pass --sheet-id or set " + f"{SCENARIO_SHEET_ID_ENV} (e.g. in rate_design/hp_rates//state.env)." + ) + return resolved def _normalize_header(name: str) -> str: @@ -157,16 +162,24 @@ def _normalize_header(name: str) -> str: return s -def _normalize_deprecated_elec_heat_seasonal_paths(value: str) -> str: - """Rewrite deprecated ``_elec_heat_seasonal_epmc`` path segments to ``_elec_heat_seasonal``. - - Tariff JSON and tariff-map CSVs were renamed on disk; the Runs & Charts sheet - may still emit the old filenames. This does not alter ``hp_seasonal_epmc`` or - other keys that do not contain ``_elec_heat_seasonal_epmc``. - """ - if not value: - return value - return value.replace("_elec_heat_seasonal_epmc", "_elec_heat_seasonal") +def _require_utility_for_data_row( + row: dict[str, str], + *, + state_key: str, + utility_key: str, + num_key: str, +) -> None: + """Raise if a sheet row has state set but utility is missing or invalid.""" + state_val = (row.get(state_key) or "").strip() + if not state_val: + return + utility_val = (row.get(utility_key) or "").strip() + num_val = (row.get(num_key) or "").strip() + if not utility_val or utility_val.isdigit(): + raise ValueError( + f"Row with state={state_val!r} num={num_val!r} has blank or invalid " + f"utility {utility_val!r}. Fill the utility column for every data row." + ) def _path_tariffs_to_dict(comma_separated: str) -> dict[str, str]: @@ -395,25 +408,17 @@ def parse_required_float(key: str) -> float: "path_tariffs_gas", "path_outputs", ): - val = get(key) - if key == "path_tariff_maps_electric": - val = _normalize_deprecated_elec_heat_seasonal_paths(val) - run[key] = val + run[key] = get(key) run["path_supply_energy_mc"] = require_non_empty("path_supply_energy_mc") run["path_supply_capacity_mc"] = require_non_empty("path_supply_capacity_mc") path_tariffs_raw = require_non_empty("path_tariffs_electric") - path_tariffs_raw = _normalize_deprecated_elec_heat_seasonal_paths(path_tariffs_raw) run["path_tariffs_electric"] = _path_tariffs_to_dict(path_tariffs_raw) run["utility_revenue_requirement"] = get("utility_revenue_requirement") - # Accept either the new column name or the old one for backward compatibility - supply_raw = get_optional("run_includes_supply") - if not supply_raw: - supply_raw = require_non_empty("add_supply_revenue_requirement") - run["run_includes_supply"] = _parse_bool(supply_raw) + run["run_includes_supply"] = _parse_bool(require_non_empty("run_includes_supply")) run["run_includes_subclasses"] = _parse_bool( require_non_empty("run_includes_subclasses") @@ -516,7 +521,7 @@ def _get_worksheet( def run( - sheet_id: str = DEFAULT_SHEET_ID, + sheet_id: str, worksheet_name: str | None = None, worksheet_index: int | None = None, output_dir: Path | None = None, @@ -604,18 +609,18 @@ def run( f"Normalized headers: {list(norm_to_header)}" ) - # Filter to data rows with state set; forward-fill utility + # Filter to data rows with state set; require utility on every row data_rows: list[dict[str, str]] = [] - prev_utility = "" for row in rows_as_dicts: state_val = (row.get(state_key) or "").strip() if not state_val: continue - utility_val = (row.get(utility_key) or "").strip() - if utility_val and not utility_val.isdigit(): - prev_utility = utility_val - elif prev_utility: - row[utility_key] = prev_utility + _require_utility_for_data_row( + row, + state_key=state_key, + utility_key=utility_key, + num_key=num_key, + ) data_rows.append(row) # Group by (state, utility) @@ -687,8 +692,11 @@ def _parse_args() -> argparse.Namespace: ) parser.add_argument( "--sheet-id", - default=DEFAULT_SHEET_ID, - help="Google Spreadsheet ID (default: Runs & Charts sheet).", + default=None, + help=( + "Google Spreadsheet ID. Defaults to SCENARIO_SHEET_ID env var " + "(set in rate_design/hp_rates//state.env)." + ), ) parser.add_argument( "--sheet-name", @@ -715,9 +723,10 @@ def _parse_args() -> argparse.Namespace: def main() -> None: + load_dotenv() args = _parse_args() run( - sheet_id=args.sheet_id, + sheet_id=_resolve_sheet_id(args.sheet_id), worksheet_name=args.sheet_name, worksheet_index=args.worksheet_index, output_dir=args.output_dir,