From b7b9671ea5e39c7bca30a45536265943d4845666 Mon Sep 17 00:00:00 2001 From: Nick Harder Date: Mon, 5 May 2025 14:02:06 +0200 Subject: [PATCH 01/11] Initial commit changing the structure of the database for better performance - use partitions - use schema during initialization - fix conversions of tensors --- assume/common/outputs.py | 134 +++++++++------ assume/common/units_operator.py | 21 ++- .../algorithms/matd3.py | 8 +- .../reinforcement_learning/learning_role.py | 2 +- .../reinforcement_learning/learning_utils.py | 22 +-- assume/strategies/naive_strategies.py | 29 ++-- assume/units/demand.py | 14 +- compose.yml | 2 + docker_configs/db-init/assume_schema.sql | 153 ++++++++++++++++++ docs/source/learning_algorithm.rst | 4 +- docs/source/units.rst | 2 +- examples/examples.py | 2 +- .../inputs/example_02a/powerplant_units.csv | 4 +- tests/test_demand.py | 5 +- 14 files changed, 308 insertions(+), 94 deletions(-) create mode 100644 docker_configs/db-init/assume_schema.sql diff --git a/assume/common/outputs.py b/assume/common/outputs.py index cccdc8cfe..58dea63cd 100644 --- a/assume/common/outputs.py +++ b/assume/common/outputs.py @@ -6,6 +6,7 @@ import shutil from collections import defaultdict from datetime import datetime +from io import StringIO from multiprocessing import Lock from pathlib import Path from typing import TypedDict @@ -137,47 +138,6 @@ def __init__( } ] - def delete_db_scenario(self, simulation_id: str): - """ - Deletes all data from the database for the given simulation id. - - Args: - simulation_id (str): The ID of the simulation as a unique classifier. - """ - - # Loop throuph all database tables - # Get list of table names in database - table_names = inspect(self.db).get_table_names() - # Iterate through each table - for table_name in table_names: - # ignore spatial_ref_sys table - if table_name == "spatial_ref_sys": - continue - # only delete rl_params and rl_meta during the first episode of learning - if table_name in ["rl_params", "rl_meta"] and not ( - self.learning_mode and self.episode == 1 - ): - continue - try: - with self.db.begin() as db: - # create index on table - query = text( - f'create index if not exists "{table_name}_scenario" on "{table_name}" (simulation)' - ) - db.execute(query) - - query = text( - f"delete from \"{table_name}\" where simulation = '{simulation_id}'" - ) - rowcount = db.execute(query).rowcount - # has to be done manually with raw queries - db.commit() - logger.debug("deleted %s rows from %s", rowcount, table_name) - except Exception as e: - logger.error( - f"could not clear old scenarios from table {table_name} - {e}" - ) - def setup(self): """ Sets up the WriteOutput instance by subscribing to messages and scheduling recurrent tasks of storing the data. @@ -193,9 +153,14 @@ def setup(self): def on_ready(self): if self.db_uri: self.db = create_engine(self.db_uri) + if self.db is not None: + # 1) delete any old partitions for this simulation first self.delete_db_scenario(self.simulation_id) + # 2) then create fresh partitions for this run + self._create_partitions(self.simulation_id) + if self.save_frequency_hours is not None: recurrency_task = rr.rrule( freq=rr.HOURLY, @@ -208,9 +173,26 @@ def on_ready(self): self.store_dfs, recurrency_task, src="no_wait", - # this should not wait for the task to finish to block the simulation ) + def delete_db_scenario(self, simulation_id: str): + if not self.db: + return + + tables = [ + "market_meta", + "market_dispatch", + "unit_dispatch", + "rl_params", + "grid_flows", + "kpis", + ] + with self.db.begin() as conn: + for tbl in tables: + part = f"{tbl}_{simulation_id}" + conn.execute(text(f"DROP TABLE IF EXISTS {part};")) + logger.debug("dropped partition %s", part) + def handle_output_message(self, content: dict, meta: MetaDict): """ Handles the incoming messages and performs corresponding actions. @@ -501,16 +483,15 @@ async def store_dfs(self): dfs.append(df) df = pd.concat(dfs, axis=0, join="outer") data_list.clear() - # concat all dataframes - # use join='outer' to keep all columns and fill missing values with NaN + if df is None or df.empty: continue - # check for tensors and convert them to floats - df = df.apply(convert_tensors) - - # check for any float64 columns and convert them to floats - df = df.map(lambda x: float(x) if isinstance(x, np.float64) else x) + if table == "rl_params": + df = df.apply(convert_tensors) + float_cols = df.select_dtypes(include=["float64"]).columns + if len(float_cols): + df[float_cols] = df[float_cols].astype("float32") if self.export_csv_path: data_path = self.export_csv_path / f"{table}.csv" @@ -523,13 +504,11 @@ async def store_dfs(self): if self.db is not None: try: - with self.db.begin() as db: - df.to_sql(table, db, if_exists="append") + self._copy_df_to_db(table, df) except (ProgrammingError, OperationalError, DataError): self.check_columns(table, df) # now try again - with self.db.begin() as db: - df.to_sql(table, db, if_exists="append") + self._copy_df_to_db(table, df) self.current_dfs_size_bytes = 0 @@ -728,6 +707,55 @@ def get_sum_reward(self, episode: int, evaluation_mode=True): return rewards_by_unit + def _copy_df_to_db(self, table: str, df: pd.DataFrame): + # 1) If there’s a named index (e.g. time/datetime), turn it into a real column + if df.index.name: + df = df.reset_index() + + # 2) Introspect the exact column order from Postgres + inspector = inspect(self.db) + cols = [col["name"] for col in inspector.get_columns(table)] + + # 3) Reindex the DataFrame to those columns (missing → NaN) + df = df.reindex(columns=cols) + + # 4) Serialize CSV with header + buf = StringIO() + df.to_csv(buf, index=False, header=True) + buf.seek(0) + + # 5) Bulk‐load via COPY, matching on header + col_list = ",".join(cols) + sql = f"COPY {table} ({col_list}) FROM STDIN WITH (FORMAT CSV, HEADER TRUE)" + raw = self.db.raw_connection() + try: + cur = raw.cursor() + cur.copy_expert(sql, buf) + raw.commit() + finally: + cur.close() + + def _create_partitions(self, simulation_id: str): + """Create one child‐partition per parent table for this simulation_id.""" + tables = [ + "market_meta", + "market_dispatch", + "unit_dispatch", + "rl_params", + "grid_flows", + "kpis", + ] + with self.db.begin() as conn: + for tbl in tables: + part = f"{tbl}_{simulation_id}" + sql = f""" + CREATE TABLE IF NOT EXISTS {part} + PARTITION OF {tbl} + FOR VALUES IN ('{simulation_id}'); + """ + conn.execute(text(sql)) + logger.debug("created partition %s", part) + class DatabaseMaintenance: """ diff --git a/assume/common/units_operator.py b/assume/common/units_operator.py index 600fc55aa..7a8b8f6a5 100644 --- a/assume/common/units_operator.py +++ b/assume/common/units_operator.py @@ -23,9 +23,10 @@ ) from assume.common.utils import ( aggregate_step_amount, + convert_tensors, timestamp2datetime, ) -from assume.strategies import BaseStrategy +from assume.strategies import BaseStrategy, LearningStrategy from assume.units import BaseUnit logger = logging.getLogger(__name__) @@ -74,6 +75,8 @@ def __init__( self.valid_orders = defaultdict(list) self.units: dict[str, BaseUnit] = {} + self.rl_operator = False + def setup(self): super().setup() self.context.subscribe_message( @@ -144,6 +147,14 @@ def add_unit( """ self.units[unit.id] = unit + # only do the market‐loop if we haven't already flipped rl_operator + if not self.rl_operator: + for market in self.available_markets: + strategy = unit.bidding_strategies.get(market.market_id) + if isinstance(strategy, LearningStrategy): + self.rl_operator = True + break + def participate(self, market: MarketConfig) -> bool: """ Method which decides if we want to participate on a given Market. @@ -545,4 +556,10 @@ async def formulate_bids( order["unit_id"] = unit_id orderbook.append(order) - return orderbook + if not self.rl_operator: + # if we are not a learning agent, we can just return the orderbook + return orderbook + # if we are a learning agent, we need to convert the orderbook to tensors + else: + # convert all CUDA tensors to CPU in one pass + orderbook = convert_tensors(orderbook) diff --git a/assume/reinforcement_learning/algorithms/matd3.py b/assume/reinforcement_learning/algorithms/matd3.py index bb1d873a2..1ad3c76e7 100644 --- a/assume/reinforcement_learning/algorithms/matd3.py +++ b/assume/reinforcement_learning/algorithms/matd3.py @@ -164,7 +164,7 @@ def load_critic_params(self, directory: str) -> None: logger.info("Agents order unchanged. Loading critic weights directly.") else: logger.info( - f"Agents length and/or order mismatch: n_old={len(loaded_id_order)}, n_new={len(new_id_order)}. Transfering weights for critics and target critics." + f"Agents length and/or order mismatch: n_old={len(loaded_id_order)}, n_new={len(new_id_order)}. Transferring weights for critics and target critics." ) for u_id, strategy in self.learning_role.rl_strats.items(): @@ -305,13 +305,15 @@ def check_strategy_dimensions(self) -> None: if len(set(obs_dim_list)) > 1: raise ValueError( - f"All observation dimensions must be the same for all RL agents. The dfined learning strategies have the following observation dimensions: {obs_dim_list}" + f"All observation dimensions must be the same for all RL agents. The defined learning strategies have the following observation dimensions: {obs_dim_list}" ) else: self.obs_dim = obs_dim_list[0] if len(set(act_dim_list)) > 1: - raise ValueError(f"All action dimensions must be the same for all RL agents. The defined learning strategies have the following action dimensions: {act_dim_list}") + raise ValueError( + f"All action dimensions must be the same for all RL agents. The defined learning strategies have the following action dimensions: {act_dim_list}" + ) else: self.act_dim = act_dim_list[0] diff --git a/assume/reinforcement_learning/learning_role.py b/assume/reinforcement_learning/learning_role.py index b8696e50c..2fb7432bf 100644 --- a/assume/reinforcement_learning/learning_role.py +++ b/assume/reinforcement_learning/learning_role.py @@ -228,7 +228,7 @@ def turn_off_initial_exploration(self, loaded_only=False) -> None: If `loaded_only=True`, only turn off exploration for strategies that were loaded (used in continue_learning mode). If `loaded_only=False`, turn it off for all strategies. - + Args: loaded_only (bool): Whether to disable exploration only for loaded strategies. """ diff --git a/assume/reinforcement_learning/learning_utils.py b/assume/reinforcement_learning/learning_utils.py index 8956df22d..4d938e5c2 100644 --- a/assume/reinforcement_learning/learning_utils.py +++ b/assume/reinforcement_learning/learning_utils.py @@ -200,20 +200,20 @@ def transfer_weights( """ Transfer weights from loaded model to new model. Copy only those obs_ and action-slices for matching IDs. New IDs keep their original (random) weights. Function only works if the neural network architeczture remained stable besides the input layer, namely with the same hidden layers. - + Args: model (th.nn.Module): The model to transfer weights to. loaded_state (dict): The state dictionary of the loaded model. loaded_id_order (list[str]): The list of unit IDs from the loaded model that shows us the order of units. - new_id_order (list[str]): The list of IDs from the new model, inlcudes potentially different agents in comparison to the loaded model. + new_id_order (list[str]): The list of IDs from the new model, includes potentially different agents in comparison to the loaded model. obs_base (int): The base observation size. act_dim (int): The action dimension size. unique_obs (int): The unique observation size per agent, smaller than obs_base as these include also shared observation values. - + returns: dict | None: The updated state dictionary with transferred weights, or None if architecture mismatch. """ - + # 1) Architecture check new_state = model.state_dict() loaded_hidden = get_hidden_sizes(loaded_state, prefix="q1_layers") @@ -245,7 +245,7 @@ def transfer_weights( # a) shared obs_base w_new[:, :obs_base] = w_loaded[:, :obs_base] - # b) matched agents’ ID + # b) matched agents’ ID # copy weights from loaded to new model for new_idx, u in enumerate(new_id_order): if u not in loaded_id_order: @@ -261,10 +261,12 @@ def transfer_weights( # action blocks for every agent new_act = new_obs_tot + act_dim * new_idx loaded_act = loaded_obs_tot + act_dim * loaded_idx - w_new[:, new_act : new_act + act_dim] = w_loaded[:, loaded_act : loaded_act + act_dim] + w_new[:, new_act : new_act + act_dim] = w_loaded[ + :, loaded_act : loaded_act + act_dim + ] - # c) unmatched agents’ ID - # use randomly initilized weights for unmatched agents + # c) unmatched agents’ ID + # use randomly initialized weights for unmatched agents for new_idx, u in enumerate(new_id_order): if new_idx == 0 or u in loaded_id_order: continue @@ -279,6 +281,8 @@ def transfer_weights( new_state_copy[f"{prefix}.{i}.weight"].copy_( loaded_state[f"{prefix}.{i}.weight"] ) - new_state_copy[f"{prefix}.{i}.bias"].copy_(loaded_state[f"{prefix}.{i}.bias"]) + new_state_copy[f"{prefix}.{i}.bias"].copy_( + loaded_state[f"{prefix}.{i}.bias"] + ) return new_state_copy diff --git a/assume/strategies/naive_strategies.py b/assume/strategies/naive_strategies.py index c32bd0512..682975f4a 100644 --- a/assume/strategies/naive_strategies.py +++ b/assume/strategies/naive_strategies.py @@ -363,7 +363,7 @@ def calculate_bids( class ElasticDemandStrategy(BaseStrategy): """ A bidding strategy for a demand unit that submits multiple bids to approximate - a marginal utility curve, based on linear or isoelastic demand theory. + a marginal utility curve, based on linear or isoelastic demand theory. P = Price, Q = Quantity, E = Elasticity. - Linear model: P = P_max + slope * Q (slope is only defined by P_max and Q_max, negative value) @@ -384,9 +384,8 @@ def calculate_bids( product_tuples: list[Product], **kwargs, ) -> Orderbook: - bids = [] - + for product in product_tuples: start, end, only_hours = product max_abs_power = max(abs(unit.min_power), abs(unit.max_power)) @@ -402,14 +401,14 @@ def calculate_bids( # integrate: # \int 1/Q dQ = E * \int 1/P dP # ln(Q) = E * ln(P) + C - # exp(ln(Q)) = exp(E * ln(p) + C) + # exp(ln(Q)) = exp(E * ln(p) + C) # Q(p) = (exp(ln(p))^E) * exp(C) # Q(p) = P^E * exp(C) # possibly C = 0, C = 1 or C >= 1. We assume C >= 1. # C shifts the demand curve (demand vs. price) up / down and # can be interpreted as the demand that will always be served # we set C = ln(Q_max) and derive the demand curve from there: - # P = Q^(1/E) * exp(-C/E) and C = ln(Q_max) + # P = Q^(1/E) * exp(-C/E) and C = ln(Q_max) # => P = Q^(1/E) * exp(-ln(Q_max)/E) and because exp(-ln(Q_max)/E) = Q_max^(-1/E) # => P = Q^(1/E) * Q_max^(-1/E) # finally @@ -430,7 +429,7 @@ def calculate_bids( "node": unit.node, } ) - + remaining_volume = max_abs_power - first_bid_volume if remaining_volume < 0: raise ValueError( @@ -468,12 +467,12 @@ def calculate_bids( bids.append( { - "start_time": start, - "end_time": end, - "only_hours": only_hours, - "price": bid_price, - "volume": -bid_volume, - "node": unit.node, + "start_time": start, + "end_time": end, + "only_hours": only_hours, + "price": bid_price, + "volume": -bid_volume, + "node": unit.node, } ) @@ -486,7 +485,7 @@ def find_first_block_bid( Calculate the first block bid volume at max_price. P = Price, Q = Quantity, E = Elasticity. Assumes isoelastic demand: Q = Q_max * P^E - + The first block bid is the volume that is always bid at maximum price, because the willingness to pay for it is higher than the markets maximal price. The first block bid volume is calculated by finding the intersection of the isoelastic demand @@ -495,14 +494,14 @@ def find_first_block_bid( Q_first = Q Q_first = Q_max * P^E - + Therefore: Q_first = max_power * (max_price ** E) Returns: float: Volume > 0, demand that is always bought at max willingness to pay """ - volume = max_power * max_price**elasticity + volume = max_power * max_price**elasticity if abs(volume) > abs(max_power): raise ValueError( diff --git a/assume/units/demand.py b/assume/units/demand.py index 4fe045aa4..6fe57cb9e 100644 --- a/assume/units/demand.py +++ b/assume/units/demand.py @@ -80,16 +80,22 @@ def __init__( f"Invalid elasticity_model '{self.elasticity_model}' at unit {self.id}. Choose 'linear' or 'isoelastic'." ) if self.num_bids <= 1: - raise ValueError(f"'num_bids' parameter must be >= 1 for elastic demand at unit {self.id}") + raise ValueError( + f"'num_bids' parameter must be >= 1 for elastic demand at unit {self.id}" + ) if self.elasticity_model == "isoelastic": if self.elasticity >= 0.0: - raise ValueError(f"'elasticity' parameter must be given and negative for isoelastic demand at unit {self.id}.") + raise ValueError( + f"'elasticity' parameter must be given and negative for isoelastic demand at unit {self.id}." + ) if self.elasticity_model == "linear": - if -(self.max_price / max(abs(self.min_power), abs(self.max_power))) >= 0.0: + if ( + -(self.max_price / max(abs(self.min_power), abs(self.max_power))) + >= 0.0 + ): raise ValueError( f"Invalid slope of demand curve at unit {self.id}. Slope must be negative for linear demand. Set 'max_price' positive." ) - def execute_current_dispatch( self, diff --git a/compose.yml b/compose.yml index 0a4945de5..d8c881ecd 100644 --- a/compose.yml +++ b/compose.yml @@ -22,6 +22,8 @@ services: volumes: # needed for normal image - ./assume-db:/var/lib/postgresql/data + # new init-script folder (runs only on first container init) + - ./docker_configs/db-init:/docker-entrypoint-initdb.d:ro ports: - 5432:5432 deploy: diff --git a/docker_configs/db-init/assume_schema.sql b/docker_configs/db-init/assume_schema.sql new file mode 100644 index 000000000..e1d855f7d --- /dev/null +++ b/docker_configs/db-init/assume_schema.sql @@ -0,0 +1,153 @@ +-- SPDX-FileCopyrightText: ASSUME Developers +-- +-- SPDX-License-Identifier: AGPL-3.0-or-later + +-- 0) Enable extensions +CREATE EXTENSION IF NOT EXISTS timescaledb CASCADE; +CREATE EXTENSION IF NOT EXISTS postgis; + +-- 1) market_meta (partitioned by simulation) +CREATE TABLE IF NOT EXISTS market_meta ( + simulation TEXT NOT NULL, + market_id TEXT, + "index" INTEGER, + time TIMESTAMP NOT NULL, + node TEXT, + product_start TIMESTAMP NOT NULL, + product_end TIMESTAMP NOT NULL, + only_hours TEXT, + price REAL, + max_price REAL, + min_price REAL, + supply_volume REAL, + supply_volume_energy REAL, + demand_volume REAL, + demand_volume_energy REAL +) +PARTITION BY LIST (simulation); + +-- 2) market_dispatch (partitioned by simulation) +CREATE TABLE IF NOT EXISTS market_dispatch ( + simulation TEXT NOT NULL, + "index" INTEGER, + market_id TEXT, + datetime TIMESTAMP NOT NULL, + unit_id TEXT, + power REAL +) +PARTITION BY LIST (simulation); + +-- 3) market_orders (static) +CREATE TABLE IF NOT EXISTS market_orders ( + simulation TEXT NOT NULL, + market_id TEXT, + start_time TIMESTAMP NOT NULL, + volume REAL, + accepted_volume REAL, + price REAL, + unit_id TEXT, + bid_type TEXT, + node TEXT, + evaluation_frequency TEXT, + eligible_lambda TEXT +); + +-- 4) unit_dispatch (partitioned by simulation) +CREATE TABLE IF NOT EXISTS unit_dispatch ( + simulation TEXT NOT NULL, + time TIMESTAMP NOT NULL, + "index" INTEGER, + unit TEXT, + power REAL, + heat REAL, + soc REAL, + energy_generation_costs REAL, + energy_cashflow REAL, + total_costs REAL +) +PARTITION BY LIST (simulation); + +-- 5) power_plant_meta (static) +CREATE TABLE IF NOT EXISTS power_plant_meta ( + simulation TEXT NOT NULL, + "index" TEXT, + technology TEXT, + unit_operator TEXT, + node TEXT, + max_power REAL, + min_power REAL, + emission_factor REAL, + efficiency REAL +); + +-- 6) storage_meta (static) +CREATE TABLE IF NOT EXISTS storage_meta ( + simulation TEXT NOT NULL, + "index" TEXT, + unit_type TEXT, + max_soc REAL, + min_soc REAL, + max_power_charge REAL, + max_power_discharge REAL, + min_power_charge REAL, + min_power_discharge REAL, + efficiency_charge REAL, + efficiency_discharge REAL +); + +-- 7) demand_meta (static; add additional fields as needed) +CREATE TABLE IF NOT EXISTS demand_meta ( + simulation TEXT NOT NULL, + "index" TEXT +); + +-- 8) rl_params (partitioned by simulation) +CREATE TABLE IF NOT EXISTS rl_params ( + simulation TEXT NOT NULL, + "index" INTEGER, + unit TEXT, + datetime TIMESTAMP NOT NULL, + evaluation_mode BOOLEAN, + episode INTEGER, + profit REAL, + reward REAL, + regret REAL, + actions TEXT, + exploration_noise REAL, + critic_loss REAL, + total_grad_norm REAL, + max_grad_norm REAL, + learning_rate REAL +) +PARTITION BY LIST (simulation); + +-- 9) rl_meta (static) +CREATE TABLE IF NOT EXISTS rl_meta ( + simulation TEXT NOT NULL, + "index" INTEGER, + episode INTEGER, + eval_episode INTEGER, + learning_mode BOOLEAN, + evaluation_mode BOOLEAN +); + +-- 10) grid_flows (partitioned by simulation) +CREATE TABLE IF NOT EXISTS grid_flows ( + "index" INTEGER, + datetime TIMESTAMP NOT NULL, + line TEXT, + flow REAL, + simulation TEXT +) +PARTITION BY LIST (simulation); + +-- 11) kpis (partitioned by simulation) +CREATE TABLE IF NOT EXISTS kpis ( + "index" INTEGER, + variable TEXT, + ident TEXT, + value REAL, + simulation TEXT, + time TIMESTAMP DEFAULT now() +) +PARTITION BY LIST (simulation); diff --git a/docs/source/learning_algorithm.rst b/docs/source/learning_algorithm.rst index 5a2dfce4f..48910e067 100644 --- a/docs/source/learning_algorithm.rst +++ b/docs/source/learning_algorithm.rst @@ -53,7 +53,7 @@ The following table shows the options that can be adjusted and gives a short exp early_stopping_threshold The value by which the average reward needs to improve to avoid early stopping. ======================================== ========================================================================================================== -How to use continue learning +How to use continue learning ---------------------------- The continue learning function allows you to load pre-trained strategies (actor and critic networks) and continue the learning process with these networks. @@ -64,7 +64,7 @@ The learning process will then start from these pre-trained networks instead of In other words, the input layer of the critics will vary depending on the number of agents. To enable the use of continue learning between simulations with varying agent sizes, a mapping is implemented that ensures the loaded critics are adapted to match the new number of agents. -This process will fail, when the number of hidden layers differes between the loaded critic and the new critic. In this case, you will need to retrain the networks from scratch. Further, different chosen neural network arhcitectures for the critic (or actor) between the loaded and new networks will also lead to a failure of the continue learning process. +This process will fail, when the number of hidden layers differs between the loaded critic and the new critic. In this case, you will need to retrain the networks from scratch. Further, different chosen neural network arhcitectures for the critic (or actor) between the loaded and new networks will also lead to a failure of the continue learning process. The Algorithms diff --git a/docs/source/units.rst b/docs/source/units.rst index 28f13e8b4..2af818340 100644 --- a/docs/source/units.rst +++ b/docs/source/units.rst @@ -14,7 +14,7 @@ The primary unit types in this context include: 2. **Storage Units**: These units, like batteries or pumped hydro storage, can store electricity when supply exceeds demand and release it when needed, adding flexibility to the grid. -3. **Demand Units**: These represent consumers of electricity, such as households, industries, or commercial buildings, whose electricity consumption is typically fixed and not easily adjustable based on real-time grid conditions. Demand units will therefore be modelled with inelastic demand most often. However, representation of elastic bidding is possible with this unit type. +3. **Demand Units**: These represent consumers of electricity, such as households, industries, or commercial buildings, whose electricity consumption is typically fixed and not easily adjustable based on real-time grid conditions. Demand units will therefore be modelled with inelastic demand most often. However, representation of elastic bidding is possible with this unit type. Each unit type has specific characteristics that affect how the power system operates, and understanding these is key to modelling and optimizing grid performance. diff --git a/examples/examples.py b/examples/examples.py index 50d65d69d..8c8feafa1 100644 --- a/examples/examples.py +++ b/examples/examples.py @@ -113,7 +113,7 @@ # select to store the simulation results in a local database or in timescale # when using timescale, you need to have docker installed and can access the grafana dashboard - data_format = "local_db" # "local_db" or "timescale" + data_format = "timescale" # "local_db" or "timescale" # select the example to run from the available examples above example = "small_with_vre_and_storage" diff --git a/examples/inputs/example_02a/powerplant_units.csv b/examples/inputs/example_02a/powerplant_units.csv index 1bb383838..106fd11d5 100644 --- a/examples/inputs/example_02a/powerplant_units.csv +++ b/examples/inputs/example_02a/powerplant_units.csv @@ -4,5 +4,5 @@ pp_2,nuclear,naive_eom,uranium,0,1000,1,0.36,10,Operator 1 pp_3,lignite,naive_eom,lignite,0.406,1000,1,0.41,2.0,Operator 1 pp_4,hard coal,naive_eom,hard coal,0.335,1000,1,0.48,1.0,Operator 1 pp_5,hard coal,naive_eom,hard coal,0.335,1000,1,0.48,1.0,Operator 1 -pp_6,combined cycle gas turbine,pp_learning,natural gas,0.201,2500,1,0.60,4.0,Operator-RL -pp_7,combined cycle gas turbine,naive_eom,natural gas,0.201,5000,1,0.60,34.0,Operator 1 +pp_6,combined cycle gas turbine,pp_learning,natural gas,0.201,2500,1,0.60,4.0,Operator 2 +pp_7,combined cycle gas turbine,naive_eom,natural gas,0.201,5000,1,0.60,34.0,Operator 3 diff --git a/tests/test_demand.py b/tests/test_demand.py index dc65de0b4..d79b1ad13 100644 --- a/tests/test_demand.py +++ b/tests/test_demand.py @@ -194,7 +194,10 @@ def test_elastic_demand_config_and_errors(): assert "price" in bid and "volume" in bid and bid["price"] > 0 # Invalid: elasticity is positive (isoelastic model) - with pytest.raises(ValueError, match="'elasticity' parameter must be given and negative for isoelastic demand"): + with pytest.raises( + ValueError, + match="'elasticity' parameter must be given and negative for isoelastic demand", + ): Demand( id="bad_elasticity", unit_operator="UO1", From 827cc1d4d42f2e5597cffde5543b2a4f235b5fe1 Mon Sep 17 00:00:00 2001 From: Nick Harder Date: Mon, 5 May 2025 16:19:26 +0200 Subject: [PATCH 02/11] - improve schema - improve flow and add checks for tables --- assume/common/outputs.py | 124 +++++++++++++++-------- docker_configs/db-init/assume_schema.sql | 28 +++-- 2 files changed, 101 insertions(+), 51 deletions(-) diff --git a/assume/common/outputs.py b/assume/common/outputs.py index 58dea63cd..43f844ca5 100644 --- a/assume/common/outputs.py +++ b/assume/common/outputs.py @@ -18,7 +18,12 @@ from pandas.api.types import is_bool_dtype, is_numeric_dtype from psycopg2.errors import UndefinedColumn from sqlalchemy import create_engine, inspect, text -from sqlalchemy.exc import DataError, OperationalError, ProgrammingError +from sqlalchemy.exc import ( + DataError, + NoSuchTableError, + OperationalError, + ProgrammingError, +) from assume.common.market_objects import MetaDict from assume.common.utils import ( @@ -104,6 +109,8 @@ def __init__( self.write_buffers: dict = defaultdict(list) self.locks = defaultdict(lambda: Lock()) + self._column_cache: dict[str, list[str]] = {} + self.kpi_defs: dict[str, OutputDef] = { "avg_price": { "value": "avg(price)", @@ -487,12 +494,15 @@ async def store_dfs(self): if df is None or df.empty: continue + # check and correct dtypes if table == "rl_params": df = df.apply(convert_tensors) + float_cols = df.select_dtypes(include=["float64"]).columns if len(float_cols): df[float_cols] = df[float_cols].astype("float32") + # export to csv if path is set if self.export_csv_path: data_path = self.export_csv_path / f"{table}.csv" df.to_csv( @@ -502,11 +512,17 @@ async def store_dfs(self): float_format="%.5g", ) + # store to db if db is set if self.db is not None: try: self._copy_df_to_db(table, df) + except NoSuchTableError: + # if the table does not exist, create it + self._ensure_table(table, df) + # now try again + self._copy_df_to_db(table, df) except (ProgrammingError, OperationalError, DataError): - self.check_columns(table, df) + self._check_columns(table, df) # now try again self._copy_df_to_db(table, df) @@ -573,49 +589,11 @@ def create_line(row): df.to_sql(geo_table, db, if_exists="append") except (ProgrammingError, OperationalError, DataError, UndefinedColumn): # if a column is missing, check and try again - self.check_columns(geo_table, df) + self._check_columns(geo_table, df) # now try again with self.db.begin() as db: df.to_sql(geo_table, db, if_exists="append") - def check_columns(self, table: str, df: pd.DataFrame, index: bool = True): - """ - Checks and adds columns to the database table if necessary. - - Args: - table (str): The name of the database table. - df (pandas.DataFrame): The DataFrame to be checked. - """ - with self.db.begin() as db: - # Read table into Pandas DataFrame - query = f"select * from {table} where 1=0" - db_columns = pd.read_sql(query, db).columns - - for column in df.columns: - if column.lower() not in db_columns: - try: - # TODO this only works for float and text - if is_bool_dtype(df[column]): - column_type = "boolean" - elif is_numeric_dtype(df[column]): - column_type = "float" - else: - column_type = "text" - query = f"ALTER TABLE {table} ADD COLUMN {column} {column_type}" - with self.db.begin() as db: - db.execute(text(query)) - except Exception: - logger.exception("Error converting column") - - if index and df.index.name: - df.index.name = df.index.name.lower() - if df.index.name in db_columns: - return - column_type = "float" if is_numeric_dtype(df.index) else "text" - query = f"ALTER TABLE {table} ADD COLUMN {df.index.name} {column_type}" - with self.db.begin() as db: - db.execute(text(query)) - async def on_stop(self): """ This function makes it possible to calculate Key Performance Indicators. @@ -707,14 +685,21 @@ def get_sum_reward(self, episode: int, evaluation_mode=True): return rewards_by_unit + def _get_columns(self, table: str) -> list[str]: + if table not in self._column_cache: + inspector = inspect(self.db) + self._column_cache[table] = [ + c["name"] for c in inspector.get_columns(table) + ] + return self._column_cache[table] + def _copy_df_to_db(self, table: str, df: pd.DataFrame): # 1) If there’s a named index (e.g. time/datetime), turn it into a real column if df.index.name: df = df.reset_index() # 2) Introspect the exact column order from Postgres - inspector = inspect(self.db) - cols = [col["name"] for col in inspector.get_columns(table)] + cols = self._get_columns(table) # 3) Reindex the DataFrame to those columns (missing → NaN) df = df.reindex(columns=cols) @@ -756,6 +741,59 @@ def _create_partitions(self, simulation_id: str): conn.execute(text(sql)) logger.debug("created partition %s", part) + def _ensure_table(self, table: str, df: pd.DataFrame): + """ + If `table` doesn’t exist in the DB yet, create it using df.head(0).to_sql(), + so that future copies will succeed. + """ + if not inspect(self.db).has_table(table): + # Use zero‐row to create the right columns & types + df.head(0).to_sql( + name=table, + con=self.db, + if_exists="append", # create if missing, then do nothing + index=bool(df.index.name), + method=None, + ) + + def _check_columns(self, table: str, df: pd.DataFrame, index: bool = True): + """ + Checks and adds columns to the database table if necessary. + + Args: + table (str): The name of the database table. + df (pandas.DataFrame): The DataFrame to be checked. + """ + with self.db.begin() as db: + # Read table into Pandas DataFrame + query = f"select * from {table} where 1=0" + db_columns = pd.read_sql(query, db).columns + + for column in df.columns: + if column.lower() not in db_columns: + try: + # TODO this only works for float and text + if is_bool_dtype(df[column]): + column_type = "boolean" + elif is_numeric_dtype(df[column]): + column_type = "float" + else: + column_type = "text" + query = f"ALTER TABLE {table} ADD COLUMN {column} {column_type}" + with self.db.begin() as db: + db.execute(text(query)) + except Exception: + logger.exception("Error converting column") + + if index and df.index.name: + df.index.name = df.index.name.lower() + if df.index.name in db_columns: + return + column_type = "float" if is_numeric_dtype(df.index) else "text" + query = f"ALTER TABLE {table} ADD COLUMN {df.index.name} {column_type}" + with self.db.begin() as db: + db.execute(text(query)) + class DatabaseMaintenance: """ diff --git a/docker_configs/db-init/assume_schema.sql b/docker_configs/db-init/assume_schema.sql index e1d855f7d..d80fa2f05 100644 --- a/docker_configs/db-init/assume_schema.sql +++ b/docker_configs/db-init/assume_schema.sql @@ -67,7 +67,7 @@ CREATE TABLE IF NOT EXISTS unit_dispatch ( ) PARTITION BY LIST (simulation); --- 5) power_plant_meta (static) +-- 5.1) power_plant_meta (static) CREATE TABLE IF NOT EXISTS power_plant_meta ( simulation TEXT NOT NULL, "index" TEXT, @@ -80,7 +80,7 @@ CREATE TABLE IF NOT EXISTS power_plant_meta ( efficiency REAL ); --- 6) storage_meta (static) +-- 5.2) storage_meta (static) CREATE TABLE IF NOT EXISTS storage_meta ( simulation TEXT NOT NULL, "index" TEXT, @@ -95,13 +95,25 @@ CREATE TABLE IF NOT EXISTS storage_meta ( efficiency_discharge REAL ); --- 7) demand_meta (static; add additional fields as needed) +-- 5.3) demand_meta (static; add additional fields as needed) CREATE TABLE IF NOT EXISTS demand_meta ( simulation TEXT NOT NULL, - "index" TEXT + "index" TEXT, + unit_type TEXT, + max_power REAL, + min_power REAL, ); --- 8) rl_params (partitioned by simulation) +-- 5.4) exchange_meta (static; add additional fields as needed) +CREATE TABLE IF NOT EXISTS exchange_meta ( + simulation TEXT NOT NULL, + "index" TEXT, + unit_type TEXT, + price_import REAL, + price_export REAL, +); + +-- 6) rl_params (partitioned by simulation) CREATE TABLE IF NOT EXISTS rl_params ( simulation TEXT NOT NULL, "index" INTEGER, @@ -121,7 +133,7 @@ CREATE TABLE IF NOT EXISTS rl_params ( ) PARTITION BY LIST (simulation); --- 9) rl_meta (static) +-- 7) rl_meta (static) CREATE TABLE IF NOT EXISTS rl_meta ( simulation TEXT NOT NULL, "index" INTEGER, @@ -131,7 +143,7 @@ CREATE TABLE IF NOT EXISTS rl_meta ( evaluation_mode BOOLEAN ); --- 10) grid_flows (partitioned by simulation) +-- 8) grid_flows (partitioned by simulation) CREATE TABLE IF NOT EXISTS grid_flows ( "index" INTEGER, datetime TIMESTAMP NOT NULL, @@ -141,7 +153,7 @@ CREATE TABLE IF NOT EXISTS grid_flows ( ) PARTITION BY LIST (simulation); --- 11) kpis (partitioned by simulation) +-- 9) kpis (partitioned by simulation) CREATE TABLE IF NOT EXISTS kpis ( "index" INTEGER, variable TEXT, From 66c08052304bebfaef6aa51792caa470d2563dca Mon Sep 17 00:00:00 2001 From: Nick Harder Date: Mon, 5 May 2025 16:26:47 +0200 Subject: [PATCH 03/11] - update database maintenance class --- assume/common/outputs.py | 208 ++++++++++++++++++--------------------- 1 file changed, 98 insertions(+), 110 deletions(-) diff --git a/assume/common/outputs.py b/assume/common/outputs.py index 43f844ca5..02d583628 100644 --- a/assume/common/outputs.py +++ b/assume/common/outputs.py @@ -797,128 +797,116 @@ def _check_columns(self, table: str, df: pd.DataFrame, index: bool = True): class DatabaseMaintenance: """ - A utility class for managing simulation data stored in a database. + Utility class for managing simulation data in a partitioned TimescaleDB setup. - This class creates a database engine from a provided URI and offers methods to: - 1. Retrieve a list of unique simulation IDs across all tables. - 2. Delete specific simulations from every table. - 3. Delete all simulations, or all except those specified, across every table. + Supports: + - Listing all simulation IDs. + - Dropping specific simulation partitions. + - Dropping all simulations (or all except exclusions). - It assumes that each table (except for system tables like "spatial_ref_sys") contains a column - named 'simulation' that uniquely identifies the simulation. - - Args: - db_uri (str): The URI of the database engine used to create a SQLAlchemy engine. + Tables partitioned by simulation: market_meta, market_dispatch, unit_dispatch, + rl_params, grid_flows, kpis. Other tables are dropped via DELETE if needed. """ - def __init__(self, db_uri: str): - """ - Initializes the DatabaseMaintenance instance by creating a database engine. - - Args: - db_uri (str): The URI of the database engine. - """ - self.db_uri = db_uri - self.db = create_engine(self.db_uri) - - def get_unique_simulation_ids(self) -> list[str]: - """ - Retrieves a list of unique simulation IDs found in all tables. - - This method inspects all tables in the database (skipping system tables such as "spatial_ref_sys") - and returns the distinct simulation IDs found in the 'simulation' column. + PARTITIONED_TABLES = [ + "market_meta", + "market_dispatch", + "unit_dispatch", + "rl_params", + "grid_flows", + "kpis", + ] - Returns: - list[str]: A list of unique simulation IDs. - """ - unique_ids = set() - inspector = inspect(self.db) - table_names = inspector.get_table_names() - for table in table_names: - if table == "spatial_ref_sys": + def __init__(self, db_uri: str): + self.db = create_engine(db_uri) + self._inspector = inspect(self.db) + self._table_cache = None + + def _get_tables(self) -> list[str]: + # List only parent tables (exclude partition children) + if self._table_cache is None: + all_tables = self._inspector.get_table_names() + # exclude system table + tables = [t for t in all_tables if t != "spatial_ref_sys"] + # only include parent tables, not partitions of form '_' + parent_tables = [] + for t in tables: + if any(t.startswith(f"{p}_") for p in self.PARTITIONED_TABLES): + continue + parent_tables.append(t) + self._table_cache = parent_tables + return self._table_cache + + def get_simulation_ids(self) -> list[str]: + ids = set() + for table in self._get_tables(): + # skip tables without simulation column + cols = [c["name"] for c in self._inspector.get_columns(table)] + if "simulation" not in cols: continue - try: - query = text(f'SELECT DISTINCT simulation FROM "{table}"') - with self.db.begin() as conn: - result = conn.execute(query) - for row in result: - if row[0]: - unique_ids.add(row[0]) - except Exception as e: - logger.error( - "Error retrieving simulation ids from table %s: %s", table, e - ) - return list(unique_ids) + query = text(f"SELECT DISTINCT simulation FROM {table}") + with self.db.begin() as conn: + for (sim,) in conn.execute(query): + if sim: + ids.add(sim) + return sorted(ids) def delete_simulations(self, simulation_ids: list[str]) -> None: - """ - Deletes specific simulation records from all tables. - - This method deletes rows from every table where the 'simulation' column matches any of the - provided simulation IDs. An index is created on the simulation column to optimize the deletion, - if one does not already exist. - - Args: - simulation_ids (list[str]): A list of simulation IDs to delete. - """ if not simulation_ids: - logger.info("No simulation IDs provided for deletion.") + logger.info("No simulations specified to delete.") return - - inspector = inspect(self.db) - table_names = inspector.get_table_names() - for table in table_names: - if table == "spatial_ref_sys": - continue - try: - with self.db.begin() as conn: - conn.execute( - text( - f'CREATE INDEX IF NOT EXISTS "{table}_simulation_idx" ON "{table}" (simulation)' - ) - ) - # Safe parameterized query - delete_query = text( - f'DELETE FROM "{table}" WHERE simulation = ANY(:simulations)' + with self.db.begin() as conn: + # Drop partitions for partitioned tables + for table in self.PARTITIONED_TABLES: + for sim in simulation_ids: + part = f"{table}_{sim}" + conn.execute(text(f"DROP TABLE IF EXISTS {part};")) + logger.debug("Dropped partition %s", part) + # For non-partitioned tables, do DELETE + non_part = set(self._get_tables()) - set(self.PARTITIONED_TABLES) + for table in non_part: + # ensure simulation index + conn.execute( + text( + f"CREATE INDEX IF NOT EXISTS {table}_simulation_idx" + f" ON {table}(simulation)" ) - result = conn.execute(delete_query, {"simulations": simulation_ids}) - logger.debug("Deleted %s rows from %s", result.rowcount, table) - except Exception as e: - logger.error( - "Could not delete simulation(s) from table %s: %s", table, e ) + del_q = text(f"DELETE FROM {table} WHERE simulation = ANY(:s)") + res = conn.execute(del_q, {"s": simulation_ids}) + logger.debug("Deleted %s rows from %s", res.rowcount, table) def delete_all_simulations(self, exclude: list[str] = None) -> None: - """ - Deletes all simulation records from every table, with an option to exclude specific simulations. - - If an exclusion list is provided, simulations with those IDs will not be deleted. Otherwise, - all simulation records are removed from all tables (excluding system tables). - - Args: - exclude (list[str], optional): A list of simulation IDs that should NOT be deleted. - If None, all simulation records are deleted. - """ - inspector = inspect(self.db) - table_names = inspector.get_table_names() - for table in table_names: - if table == "spatial_ref_sys": - continue - try: - with self.db.begin() as conn: - conn.execute( - text( - f'CREATE INDEX IF NOT EXISTS "{table}_simulation_idx" ON "{table}" (simulation)' - ) + with self.db.begin() as conn: + # Partitioned: drop partitions + for table in self.PARTITIONED_TABLES: + # list existing partitions + parts = [ + p + for p in self._inspector.get_table_names() + if p.startswith(f"{table}_") + ] + to_drop = [] + if exclude: + keep = {f"{table}_{s}" for s in exclude} + to_drop = [p for p in parts if p not in keep] + else: + to_drop = parts + for part in to_drop: + conn.execute(text(f"DROP TABLE IF EXISTS {part};")) + logger.debug("Dropped partition %s", part) + # Non-partitioned: delete or exclude + non_part = set(self._get_tables()) - set(self.PARTITIONED_TABLES) + for table in non_part: + conn.execute( + text( + f"CREATE INDEX IF NOT EXISTS {table}_simulation_idx" + f" ON {table}(simulation)" ) - if exclude: - exclude_str = ", ".join([f"'{sim}'" for sim in exclude]) - delete_query = text( - f'DELETE FROM "{table}" WHERE simulation NOT IN ({exclude_str})' - ) - else: - delete_query = text(f'DELETE FROM "{table}"') - result = conn.execute(delete_query) - logger.debug("Deleted %s rows from %s", result.rowcount, table) - except Exception as e: - logger.error("Could not delete simulations from table %s: %s", table, e) + ) + if exclude: + q = text(f"DELETE FROM {table} WHERE simulation NOT IN :ex") + res = conn.execute(q, {"ex": exclude}) + else: + res = conn.execute(text(f"DELETE FROM {table}")) + logger.debug("Deleted %s rows from %s", res.rowcount, table) From 4fe996668b8e52b3e2bb6a1a688e468e01c55dc7 Mon Sep 17 00:00:00 2001 From: Nick Harder Date: Mon, 5 May 2025 16:41:55 +0200 Subject: [PATCH 04/11] - update schema --- docker_configs/db-init/assume_schema.sql | 26 +++++++++++++----------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/docker_configs/db-init/assume_schema.sql b/docker_configs/db-init/assume_schema.sql index d80fa2f05..09aee04b1 100644 --- a/docker_configs/db-init/assume_schema.sql +++ b/docker_configs/db-init/assume_schema.sql @@ -71,9 +71,8 @@ PARTITION BY LIST (simulation); CREATE TABLE IF NOT EXISTS power_plant_meta ( simulation TEXT NOT NULL, "index" TEXT, - technology TEXT, + unit_type TEXT, unit_operator TEXT, - node TEXT, max_power REAL, min_power REAL, emission_factor REAL, @@ -85,6 +84,7 @@ CREATE TABLE IF NOT EXISTS storage_meta ( simulation TEXT NOT NULL, "index" TEXT, unit_type TEXT, + unit_operator TEXT, max_soc REAL, min_soc REAL, max_power_charge REAL, @@ -97,20 +97,22 @@ CREATE TABLE IF NOT EXISTS storage_meta ( -- 5.3) demand_meta (static; add additional fields as needed) CREATE TABLE IF NOT EXISTS demand_meta ( - simulation TEXT NOT NULL, - "index" TEXT, - unit_type TEXT, - max_power REAL, - min_power REAL, + simulation TEXT NOT NULL, + "index" TEXT, + unit_type TEXT, + unit_operator TEXT, + max_power REAL, + min_power REAL, ); -- 5.4) exchange_meta (static; add additional fields as needed) CREATE TABLE IF NOT EXISTS exchange_meta ( - simulation TEXT NOT NULL, - "index" TEXT, - unit_type TEXT, - price_import REAL, - price_export REAL, + simulation TEXT NOT NULL, + "index" TEXT, + unit_type TEXT, + unit_operator TEXT, + price_import REAL, + price_export REAL, ); -- 6) rl_params (partitioned by simulation) From 9ac18a06ce1e6e3730beb1af559144f3b702fecb Mon Sep 17 00:00:00 2001 From: Nick Harder Date: Mon, 5 May 2025 16:59:16 +0200 Subject: [PATCH 05/11] - fix example.py --- examples/examples.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/examples.py b/examples/examples.py index 8c8feafa1..50d65d69d 100644 --- a/examples/examples.py +++ b/examples/examples.py @@ -113,7 +113,7 @@ # select to store the simulation results in a local database or in timescale # when using timescale, you need to have docker installed and can access the grafana dashboard - data_format = "timescale" # "local_db" or "timescale" + data_format = "local_db" # "local_db" or "timescale" # select the example to run from the available examples above example = "small_with_vre_and_storage" From fb1224cf1080d237b7c67b35c2a6dbc0fe75d4f2 Mon Sep 17 00:00:00 2001 From: Nick Harder Date: Mon, 5 May 2025 17:11:29 +0200 Subject: [PATCH 06/11] - add fallbacks for localdb using SQLite --- assume/common/outputs.py | 49 +++++++++++++++++++++++++++++++++++----- 1 file changed, 43 insertions(+), 6 deletions(-) diff --git a/assume/common/outputs.py b/assume/common/outputs.py index 02d583628..7f0890e94 100644 --- a/assume/common/outputs.py +++ b/assume/common/outputs.py @@ -158,14 +158,39 @@ def setup(self): ) def on_ready(self): + """ + Called once when the agent/container starts. + Sets up the database connection (per‐simulation SQLite or Postgres), + tears down any old data, creates partitions if needed, and schedules + the periodic store_dfs task. + """ + # 1) Initialize the engine if self.db_uri: - self.db = create_engine(self.db_uri) - - if self.db is not None: - # 1) delete any old partitions for this simulation first + # Always create a temporary engine to inspect the dialect + engine = create_engine(self.db_uri) + + if engine.dialect.name == "sqlite": + # We’re in “local_db” mode: derive a per‐simulation filename + db_path = engine.url.database # e.g. "./examples/local_db/assume_db.db" + base = Path(db_path) + sim_db = base.with_name( + f"{base.stem}_{self.simulation_id}{base.suffix}" + ) + # Remove any old file so we start fresh + if sim_db.exists(): + sim_db.unlink() + logger.debug("Removed existing local DB %s", sim_db) + # Point SQLAlchemy at the new per‐simulation file + engine = create_engine(f"sqlite:///{sim_db}") + + # For any other dialect (postgresql, etc.) just use the original engine + self.db = engine + + # 2) If using Postgres/Timescale, drop & recreate partitions + if self.db is not None and self.db.dialect.name == "postgresql": + # drop old partitions (instant metadata operation) self.delete_db_scenario(self.simulation_id) - - # 2) then create fresh partitions for this run + # create fresh partitions for this run self._create_partitions(self.simulation_id) if self.save_frequency_hours is not None: @@ -694,6 +719,18 @@ def _get_columns(self, table: str) -> list[str]: return self._column_cache[table] def _copy_df_to_db(self, table: str, df: pd.DataFrame): + # ---- SQLite fallback ---- + if self.db.dialect.name == "sqlite": + # simple append via pandas + df.to_sql( + name=table, + con=self.db, + if_exists="append", + index=False, + method="multi", # batches INSERTs + ) + return + # 1) If there’s a named index (e.g. time/datetime), turn it into a real column if df.index.name: df = df.reset_index() From 3b5f3f45098e2ed418b8f2282bc81061a2c88042 Mon Sep 17 00:00:00 2001 From: Nick Harder Date: Tue, 6 May 2025 10:24:21 +0200 Subject: [PATCH 07/11] - improve and small fixes --- assume/common/outputs.py | 214 ++++++++++++----------- docker_configs/db-init/assume_schema.sql | 20 +-- 2 files changed, 123 insertions(+), 111 deletions(-) diff --git a/assume/common/outputs.py b/assume/common/outputs.py index 7f0890e94..19731e28d 100644 --- a/assume/common/outputs.py +++ b/assume/common/outputs.py @@ -34,6 +34,26 @@ logger = logging.getLogger(__name__) +# time-series (partitioned) tables +PARTITIONED_TABLES = [ + "market_meta", + "market_dispatch", + "market_orders", + "unit_dispatch", + "rl_params", + "grid_flows", + "kpis", +] + +# all tables that carry a `simulation` column, whether static or hypertable +ALL_SIM_TABLES = PARTITIONED_TABLES + [ + "power_plant_meta", + "storage_meta", + "demand_meta", + "exchange_meta", + "rl_meta", +] + class OutputDef(TypedDict): name: str @@ -164,34 +184,23 @@ def on_ready(self): tears down any old data, creates partitions if needed, and schedules the periodic store_dfs task. """ - # 1) Initialize the engine + # 1) Initialize the engine from URI and an inspector if self.db_uri: - # Always create a temporary engine to inspect the dialect - engine = create_engine(self.db_uri) - - if engine.dialect.name == "sqlite": - # We’re in “local_db” mode: derive a per‐simulation filename - db_path = engine.url.database # e.g. "./examples/local_db/assume_db.db" - base = Path(db_path) - sim_db = base.with_name( - f"{base.stem}_{self.simulation_id}{base.suffix}" - ) - # Remove any old file so we start fresh - if sim_db.exists(): - sim_db.unlink() - logger.debug("Removed existing local DB %s", sim_db) - # Point SQLAlchemy at the new per‐simulation file - engine = create_engine(f"sqlite:///{sim_db}") - - # For any other dialect (postgresql, etc.) just use the original engine - self.db = engine - - # 2) If using Postgres/Timescale, drop & recreate partitions - if self.db is not None and self.db.dialect.name == "postgresql": - # drop old partitions (instant metadata operation) - self.delete_db_scenario(self.simulation_id) - # create fresh partitions for this run - self._create_partitions(self.simulation_id) + self.db = create_engine(self.db_uri) + self._inspector = inspect(self.db) + + # 2) Clear out previous data + if self.db is not None: + dialect = self.db.dialect.name + + if dialect == "sqlite": + # SQLite fallback: DELETE all rows for this simulation + self._purge_simulation_data_sqlite(self.simulation_id) + + elif dialect == "postgresql": + # Postgres/Timescale: drop & recreate partitions + self._purge_simulation_data_postgresql(self.simulation_id) + self._create_partitions(self.simulation_id) if self.save_frequency_hours is not None: recurrency_task = rr.rrule( @@ -207,24 +216,6 @@ def on_ready(self): src="no_wait", ) - def delete_db_scenario(self, simulation_id: str): - if not self.db: - return - - tables = [ - "market_meta", - "market_dispatch", - "unit_dispatch", - "rl_params", - "grid_flows", - "kpis", - ] - with self.db.begin() as conn: - for tbl in tables: - part = f"{tbl}_{simulation_id}" - conn.execute(text(f"DROP TABLE IF EXISTS {part};")) - logger.debug("dropped partition %s", part) - def handle_output_message(self, content: dict, meta: MetaDict): """ Handles the incoming messages and performs corresponding actions. @@ -358,12 +349,12 @@ def convert_units_definition(self, unit_info: dict): Args: unit_info (dict): The unit information. """ - del unit_info["unit_type"] - unit_info["simulation"] = self.simulation_id - u_info = {unit_info["id"]: unit_info} - del unit_info["id"] + info = dict(unit_info) + info.pop("unit_type", None) + row_id = info.pop("id", None) + info["simulation"] = self.simulation_id - return pd.DataFrame(u_info).T + return pd.DataFrame([info], index=[row_id]) def convert_market_dispatch(self, market_dispatch: list[dict]): """ @@ -550,6 +541,9 @@ async def store_dfs(self): self._check_columns(table, df) # now try again self._copy_df_to_db(table, df) + except Exception as e: + logger.error("could not write to db: %s", e) + continue self.current_dfs_size_bytes = 0 @@ -712,9 +706,8 @@ def get_sum_reward(self, episode: int, evaluation_mode=True): def _get_columns(self, table: str) -> list[str]: if table not in self._column_cache: - inspector = inspect(self.db) self._column_cache[table] = [ - c["name"] for c in inspector.get_columns(table) + c["name"] for c in self._inspector.get_columns(table) ] return self._column_cache[table] @@ -726,7 +719,8 @@ def _copy_df_to_db(self, table: str, df: pd.DataFrame): name=table, con=self.db, if_exists="append", - index=False, + index=df.index.name is not None, + index_label=df.index.name, method="multi", # batches INSERTs ) return @@ -742,48 +736,40 @@ def _copy_df_to_db(self, table: str, df: pd.DataFrame): df = df.reindex(columns=cols) # 4) Serialize CSV with header - buf = StringIO() - df.to_csv(buf, index=False, header=True) - buf.seek(0) - - # 5) Bulk‐load via COPY, matching on header - col_list = ",".join(cols) - sql = f"COPY {table} ({col_list}) FROM STDIN WITH (FORMAT CSV, HEADER TRUE)" - raw = self.db.raw_connection() - try: - cur = raw.cursor() - cur.copy_expert(sql, buf) - raw.commit() - finally: - cur.close() + with StringIO() as buffer: + df.to_csv(buffer, index=False, header=True) + buffer.seek(0) + + # 5) Bulk‐load via COPY, matching on header + col_list = ",".join(cols) + sql = f"COPY {table} ({col_list}) FROM STDIN WITH (FORMAT CSV, HEADER TRUE)" + raw_connection = self.db.raw_connection() + try: + cursor = raw_connection.cursor() + cursor.copy_expert(sql, buffer) + raw_connection.commit() + finally: + cursor.close() def _create_partitions(self, simulation_id: str): """Create one child‐partition per parent table for this simulation_id.""" - tables = [ - "market_meta", - "market_dispatch", - "unit_dispatch", - "rl_params", - "grid_flows", - "kpis", - ] with self.db.begin() as conn: - for tbl in tables: - part = f"{tbl}_{simulation_id}" + for table in PARTITIONED_TABLES: + partition = f"{table}_{simulation_id}" sql = f""" - CREATE TABLE IF NOT EXISTS {part} - PARTITION OF {tbl} + CREATE TABLE IF NOT EXISTS {partition} + PARTITION OF {table} FOR VALUES IN ('{simulation_id}'); """ conn.execute(text(sql)) - logger.debug("created partition %s", part) + logger.debug("created partition %s", partition) def _ensure_table(self, table: str, df: pd.DataFrame): """ If `table` doesn’t exist in the DB yet, create it using df.head(0).to_sql(), so that future copies will succeed. """ - if not inspect(self.db).has_table(table): + if not self._inspector.has_table(table): # Use zero‐row to create the right columns & types df.head(0).to_sql( name=table, @@ -831,6 +817,41 @@ def _check_columns(self, table: str, df: pd.DataFrame, index: bool = True): with self.db.begin() as db: db.execute(text(query)) + def _purge_simulation_data_sqlite(self, simulation_id: str): + """ + SQLite‐only fallback for clearing out old simulation data. + Deletes rows from every table where simulation = simulation_id, + but skips any tables that don’t yet exist. + """ + with self.db.begin() as conn: + for table in ALL_SIM_TABLES: + if not self._inspector.has_table(table): + logger.debug("Skipping purge: table %s does not exist", table) + continue + + sql = text(f'DELETE FROM "{table}" WHERE simulation = :sim') + res = conn.execute(sql, {"sim": simulation_id}) + logger.debug("Deleted %s rows from %s", res.rowcount, table) + + def _purge_simulation_data_postgresql(self, simulation_id: str): + """ + TimescaleDB/Postgres cleanup: + 1) Drop any simulation-specific partitions. + 2) Delete all rows in the static tables for that simulation. + """ + with self.db.begin() as conn: + # 1) drop per-simulation partitions + for tbl in PARTITIONED_TABLES: + conn.execute(text(f"DROP TABLE IF EXISTS {tbl}_{simulation_id};")) + + # 2) delete from the static tables + static_tables = [t for t in ALL_SIM_TABLES if t not in PARTITIONED_TABLES] + for tbl in static_tables: + conn.execute( + text(f"DELETE FROM {tbl} WHERE simulation = :sim;"), + {"sim": simulation_id}, + ) + class DatabaseMaintenance: """ @@ -845,15 +866,6 @@ class DatabaseMaintenance: rl_params, grid_flows, kpis. Other tables are dropped via DELETE if needed. """ - PARTITIONED_TABLES = [ - "market_meta", - "market_dispatch", - "unit_dispatch", - "rl_params", - "grid_flows", - "kpis", - ] - def __init__(self, db_uri: str): self.db = create_engine(db_uri) self._inspector = inspect(self.db) @@ -868,7 +880,7 @@ def _get_tables(self) -> list[str]: # only include parent tables, not partitions of form '_' parent_tables = [] for t in tables: - if any(t.startswith(f"{p}_") for p in self.PARTITIONED_TABLES): + if any(t.startswith(f"{p}_") for p in PARTITIONED_TABLES): continue parent_tables.append(t) self._table_cache = parent_tables @@ -894,13 +906,13 @@ def delete_simulations(self, simulation_ids: list[str]) -> None: return with self.db.begin() as conn: # Drop partitions for partitioned tables - for table in self.PARTITIONED_TABLES: + for table in PARTITIONED_TABLES: for sim in simulation_ids: - part = f"{table}_{sim}" - conn.execute(text(f"DROP TABLE IF EXISTS {part};")) - logger.debug("Dropped partition %s", part) + partition = f"{table}_{sim}" + conn.execute(text(f"DROP TABLE IF EXISTS {partition};")) + logger.debug("Dropped partition %s", partition) # For non-partitioned tables, do DELETE - non_part = set(self._get_tables()) - set(self.PARTITIONED_TABLES) + non_part = set(self._get_tables()) - set(PARTITIONED_TABLES) for table in non_part: # ensure simulation index conn.execute( @@ -916,7 +928,7 @@ def delete_simulations(self, simulation_ids: list[str]) -> None: def delete_all_simulations(self, exclude: list[str] = None) -> None: with self.db.begin() as conn: # Partitioned: drop partitions - for table in self.PARTITIONED_TABLES: + for table in PARTITIONED_TABLES: # list existing partitions parts = [ p @@ -929,11 +941,11 @@ def delete_all_simulations(self, exclude: list[str] = None) -> None: to_drop = [p for p in parts if p not in keep] else: to_drop = parts - for part in to_drop: - conn.execute(text(f"DROP TABLE IF EXISTS {part};")) - logger.debug("Dropped partition %s", part) + for partition in to_drop: + conn.execute(text(f"DROP TABLE IF EXISTS {partition};")) + logger.debug("Dropped partition %s", partition) # Non-partitioned: delete or exclude - non_part = set(self._get_tables()) - set(self.PARTITIONED_TABLES) + non_part = set(self._get_tables()) - set(PARTITIONED_TABLES) for table in non_part: conn.execute( text( diff --git a/docker_configs/db-init/assume_schema.sql b/docker_configs/db-init/assume_schema.sql index 09aee04b1..63ccea4e4 100644 --- a/docker_configs/db-init/assume_schema.sql +++ b/docker_configs/db-init/assume_schema.sql @@ -1,5 +1,4 @@ -- SPDX-FileCopyrightText: ASSUME Developers --- -- SPDX-License-Identifier: AGPL-3.0-or-later -- 0) Enable extensions @@ -37,7 +36,7 @@ CREATE TABLE IF NOT EXISTS market_dispatch ( ) PARTITION BY LIST (simulation); --- 3) market_orders (static) +-- 3) market_orders (partitioned by simulation) CREATE TABLE IF NOT EXISTS market_orders ( simulation TEXT NOT NULL, market_id TEXT, @@ -50,7 +49,8 @@ CREATE TABLE IF NOT EXISTS market_orders ( node TEXT, evaluation_frequency TEXT, eligible_lambda TEXT -); +) +PARTITION BY LIST (simulation); -- 4) unit_dispatch (partitioned by simulation) CREATE TABLE IF NOT EXISTS unit_dispatch ( @@ -95,24 +95,24 @@ CREATE TABLE IF NOT EXISTS storage_meta ( efficiency_discharge REAL ); --- 5.3) demand_meta (static; add additional fields as needed) +-- 5.3) demand_meta (static) CREATE TABLE IF NOT EXISTS demand_meta ( simulation TEXT NOT NULL, "index" TEXT, unit_type TEXT, unit_operator TEXT, max_power REAL, - min_power REAL, + min_power REAL ); --- 5.4) exchange_meta (static; add additional fields as needed) +-- 5.4) exchange_meta (static) CREATE TABLE IF NOT EXISTS exchange_meta ( simulation TEXT NOT NULL, "index" TEXT, unit_type TEXT, unit_operator TEXT, price_import REAL, - price_export REAL, + price_export REAL ); -- 6) rl_params (partitioned by simulation) @@ -147,21 +147,21 @@ CREATE TABLE IF NOT EXISTS rl_meta ( -- 8) grid_flows (partitioned by simulation) CREATE TABLE IF NOT EXISTS grid_flows ( + simulation TEXT NOT NULL, "index" INTEGER, datetime TIMESTAMP NOT NULL, line TEXT, - flow REAL, - simulation TEXT + flow REAL ) PARTITION BY LIST (simulation); -- 9) kpis (partitioned by simulation) CREATE TABLE IF NOT EXISTS kpis ( + simulation TEXT NOT NULL, "index" INTEGER, variable TEXT, ident TEXT, value REAL, - simulation TEXT, time TIMESTAMP DEFAULT now() ) PARTITION BY LIST (simulation); From e03e9696a379281725530a8effa793f2e850d703 Mon Sep 17 00:00:00 2001 From: Nick Harder Date: Tue, 6 May 2025 11:50:05 +0200 Subject: [PATCH 08/11] - update schema and introduce unique keys to ensure consistency - fix table locking --- assume/common/outputs.py | 64 +++++++----- docker_configs/db-init/assume_schema.sql | 121 ++++++++++++++++------- 2 files changed, 125 insertions(+), 60 deletions(-) diff --git a/assume/common/outputs.py b/assume/common/outputs.py index 19731e28d..505bc6e9d 100644 --- a/assume/common/outputs.py +++ b/assume/common/outputs.py @@ -349,12 +349,21 @@ def convert_units_definition(self, unit_info: dict): Args: unit_info (dict): The unit information. """ + # 1) Copy so we don't mutate the original info = dict(unit_info) + + # 2) Remove anything you don't want info.pop("unit_type", None) - row_id = info.pop("id", None) + + # 3) Pull out the id and stash it back in as its own column + unit_id = info.pop("id", None) + info["unit_id"] = unit_id + + # 4) Always include the simulation info["simulation"] = self.simulation_id - return pd.DataFrame([info], index=[row_id]) + # 5) Return a single-row DataFrame; pandas will treat unit_id & simulation as columns + return pd.DataFrame([info]) def convert_market_dispatch(self, market_dispatch: list[dict]): """ @@ -715,14 +724,8 @@ def _copy_df_to_db(self, table: str, df: pd.DataFrame): # ---- SQLite fallback ---- if self.db.dialect.name == "sqlite": # simple append via pandas - df.to_sql( - name=table, - con=self.db, - if_exists="append", - index=df.index.name is not None, - index_label=df.index.name, - method="multi", # batches INSERTs - ) + with self.db.begin() as db: + df.to_sql(table, db, if_exists="append") return # 1) If there’s a named index (e.g. time/datetime), turn it into a real column @@ -823,12 +826,19 @@ def _purge_simulation_data_sqlite(self, simulation_id: str): Deletes rows from every table where simulation = simulation_id, but skips any tables that don’t yet exist. """ - with self.db.begin() as conn: - for table in ALL_SIM_TABLES: - if not self._inspector.has_table(table): - logger.debug("Skipping purge: table %s does not exist", table) - continue + for table in ALL_SIM_TABLES: + # do not delete rl table if in learning mode + if not self._inspector.has_table(table): + logger.debug("Skipping purge: table %s does not exist", table) + continue + # only delete rl_params and rl_meta during the first episode of learning + if table in ["rl_params", "rl_meta"] and not ( + self.learning_mode and self.episode == 1 + ): + continue + + with self.db.begin() as conn: sql = text(f'DELETE FROM "{table}" WHERE simulation = :sim') res = conn.execute(sql, {"sim": simulation_id}) logger.debug("Deleted %s rows from %s", res.rowcount, table) @@ -839,16 +849,24 @@ def _purge_simulation_data_postgresql(self, simulation_id: str): 1) Drop any simulation-specific partitions. 2) Delete all rows in the static tables for that simulation. """ - with self.db.begin() as conn: - # 1) drop per-simulation partitions - for tbl in PARTITIONED_TABLES: - conn.execute(text(f"DROP TABLE IF EXISTS {tbl}_{simulation_id};")) + # 1) drop per-simulation partitions + for table in PARTITIONED_TABLES: + # only delete rl_params during the first episode of learning + if table == "rl_params" and not (self.learning_mode and self.episode == 1): + continue - # 2) delete from the static tables - static_tables = [t for t in ALL_SIM_TABLES if t not in PARTITIONED_TABLES] - for tbl in static_tables: + with self.db.begin() as conn: + conn.execute(text(f"DROP TABLE IF EXISTS {table}_{simulation_id};")) + + # 2) delete from the static tables + static_tables = [t for t in ALL_SIM_TABLES if t not in PARTITIONED_TABLES] + for table in static_tables: + if table == "rl_meta" and not (self.learning_mode and self.episode == 1): + continue + + with self.db.begin() as conn: conn.execute( - text(f"DELETE FROM {tbl} WHERE simulation = :sim;"), + text(f"DELETE FROM {table} WHERE simulation = :sim;"), {"sim": simulation_id}, ) diff --git a/docker_configs/db-init/assume_schema.sql b/docker_configs/db-init/assume_schema.sql index 63ccea4e4..a09f1a471 100644 --- a/docker_configs/db-init/assume_schema.sql +++ b/docker_configs/db-init/assume_schema.sql @@ -5,85 +5,108 @@ CREATE EXTENSION IF NOT EXISTS timescaledb CASCADE; CREATE EXTENSION IF NOT EXISTS postgis; +---------------------------- -- 1) market_meta (partitioned by simulation) +---------------------------- CREATE TABLE IF NOT EXISTS market_meta ( simulation TEXT NOT NULL, market_id TEXT, - "index" INTEGER, time TIMESTAMP NOT NULL, - node TEXT, product_start TIMESTAMP NOT NULL, product_end TIMESTAMP NOT NULL, - only_hours TEXT, + only_hours TIMESTAMP, + supply_volume REAL, + demand_volume REAL, + supply_volume_energy REAL, + demand_volume_energy REAL, price REAL, max_price REAL, min_price REAL, - supply_volume REAL, - supply_volume_energy REAL, - demand_volume REAL, - demand_volume_energy REAL + node TEXT, + PRIMARY KEY (simulation, market_id, time) ) PARTITION BY LIST (simulation); +CREATE INDEX ON market_meta (simulation, time); + +---------------------------- -- 2) market_dispatch (partitioned by simulation) +---------------------------- CREATE TABLE IF NOT EXISTS market_dispatch ( simulation TEXT NOT NULL, - "index" INTEGER, market_id TEXT, datetime TIMESTAMP NOT NULL, unit_id TEXT, - power REAL + power REAL, + PRIMARY KEY (simulation, market_id, datetime, unit_id) ) PARTITION BY LIST (simulation); +CREATE INDEX ON market_dispatch (simulation, datetime); + +---------------------------- -- 3) market_orders (partitioned by simulation) +---------------------------- CREATE TABLE IF NOT EXISTS market_orders ( simulation TEXT NOT NULL, market_id TEXT, start_time TIMESTAMP NOT NULL, - volume REAL, - accepted_volume REAL, + end_time TIMESTAMP NOT NULL, price REAL, - unit_id TEXT, + volume REAL, bid_type TEXT, node TEXT, - evaluation_frequency TEXT, - eligible_lambda TEXT + bid_id TEXT, + unit_id TEXT, + accepted_price REAL, + accepted_volume REAL, + PRIMARY KEY (simulation, market_id, start_time, bid_id) ) PARTITION BY LIST (simulation); +CREATE INDEX ON market_orders (simulation, start_time); + +---------------------------- -- 4) unit_dispatch (partitioned by simulation) +---------------------------- CREATE TABLE IF NOT EXISTS unit_dispatch ( simulation TEXT NOT NULL, time TIMESTAMP NOT NULL, - "index" INTEGER, unit TEXT, power REAL, heat REAL, soc REAL, energy_generation_costs REAL, energy_cashflow REAL, - total_costs REAL + total_costs REAL, + PRIMARY KEY (simulation, unit, time) ) PARTITION BY LIST (simulation); +CREATE INDEX ON unit_dispatch (simulation, time); + +---------------------------- -- 5.1) power_plant_meta (static) +---------------------------- CREATE TABLE IF NOT EXISTS power_plant_meta ( simulation TEXT NOT NULL, - "index" TEXT, - unit_type TEXT, + unit_id TEXT, unit_operator TEXT, max_power REAL, min_power REAL, emission_factor REAL, - efficiency REAL + efficiency REAL, + technology TEXT, + node TEXT, + PRIMARY KEY (simulation, unit_id) ); +---------------------------- -- 5.2) storage_meta (static) +---------------------------- CREATE TABLE IF NOT EXISTS storage_meta ( simulation TEXT NOT NULL, - "index" TEXT, - unit_type TEXT, + unit_id TEXT, unit_operator TEXT, max_soc REAL, min_soc REAL, @@ -92,33 +115,46 @@ CREATE TABLE IF NOT EXISTS storage_meta ( min_power_charge REAL, min_power_discharge REAL, efficiency_charge REAL, - efficiency_discharge REAL + efficiency_discharge REAL, + technology TEXT, + node TEXT, + PRIMARY KEY (simulation, unit_id) ); +---------------------------- -- 5.3) demand_meta (static) +---------------------------- CREATE TABLE IF NOT EXISTS demand_meta ( simulation TEXT NOT NULL, - "index" TEXT, + unit_id TEXT, unit_type TEXT, unit_operator TEXT, max_power REAL, - min_power REAL + min_power REAL, + technology TEXT, + node TEXT, + PRIMARY KEY (simulation, unit_id) ); +---------------------------- -- 5.4) exchange_meta (static) +---------------------------- CREATE TABLE IF NOT EXISTS exchange_meta ( simulation TEXT NOT NULL, - "index" TEXT, - unit_type TEXT, + unit_id TEXT, unit_operator TEXT, price_import REAL, - price_export REAL + price_export REAL, + technology TEXT, + node TEXT, + PRIMARY KEY (simulation, unit_id) ); +---------------------------- -- 6) rl_params (partitioned by simulation) +---------------------------- CREATE TABLE IF NOT EXISTS rl_params ( simulation TEXT NOT NULL, - "index" INTEGER, unit TEXT, datetime TIMESTAMP NOT NULL, evaluation_mode BOOLEAN, @@ -126,42 +162,53 @@ CREATE TABLE IF NOT EXISTS rl_params ( profit REAL, reward REAL, regret REAL, - actions TEXT, - exploration_noise REAL, critic_loss REAL, total_grad_norm REAL, max_grad_norm REAL, - learning_rate REAL + learning_rate REAL, + PRIMARY KEY (simulation, episode, evaluation_mode, datetime) ) PARTITION BY LIST (simulation); +CREATE INDEX ON rl_params (simulation, datetime); + +---------------------------- -- 7) rl_meta (static) +---------------------------- CREATE TABLE IF NOT EXISTS rl_meta ( simulation TEXT NOT NULL, - "index" INTEGER, episode INTEGER, eval_episode INTEGER, learning_mode BOOLEAN, - evaluation_mode BOOLEAN + evaluation_mode BOOLEAN, + PRIMARY KEY (simulation, episode, evaluation_mode) ); +---------------------------- -- 8) grid_flows (partitioned by simulation) +---------------------------- CREATE TABLE IF NOT EXISTS grid_flows ( simulation TEXT NOT NULL, - "index" INTEGER, datetime TIMESTAMP NOT NULL, line TEXT, - flow REAL + flow REAL, + PRIMARY KEY (simulation, datetime) ) PARTITION BY LIST (simulation); +CREATE INDEX ON grid_flows (simulation, datetime); + +---------------------------- -- 9) kpis (partitioned by simulation) +---------------------------- CREATE TABLE IF NOT EXISTS kpis ( simulation TEXT NOT NULL, - "index" INTEGER, variable TEXT, ident TEXT, value REAL, - time TIMESTAMP DEFAULT now() + time TIMESTAMP DEFAULT now(), + PRIMARY KEY (simulation, variable, ident) ) PARTITION BY LIST (simulation); + +CREATE INDEX ON kpis (simulation, time); From 189699e929af0b5ad218ef89ad2ed8f10f1c8fc8 Mon Sep 17 00:00:00 2001 From: Nick Harder Date: Tue, 6 May 2025 14:17:46 +0200 Subject: [PATCH 09/11] - update dashboards - update schema - introduce order for unit_dispatch --- assume/common/outputs.py | 228 +- .../ASSUME Comparison.json | 18 +- .../dashboard-definitions/ASSUME-Support.json | 5280 ----------------- .../ASSUME-Support.json.license | 3 - .../dashboard-definitions/ASSUME.json | 84 +- docker_configs/db-init/assume_schema.sql | 22 +- 6 files changed, 234 insertions(+), 5401 deletions(-) delete mode 100644 docker_configs/dashboard-definitions/ASSUME-Support.json delete mode 100644 docker_configs/dashboard-definitions/ASSUME-Support.json.license diff --git a/assume/common/outputs.py b/assume/common/outputs.py index 505bc6e9d..f70a6c1da 100644 --- a/assume/common/outputs.py +++ b/assume/common/outputs.py @@ -17,7 +17,9 @@ from mango import Role from pandas.api.types import is_bool_dtype, is_numeric_dtype from psycopg2.errors import UndefinedColumn -from sqlalchemy import create_engine, inspect, text +from sqlalchemy import MetaData, Table, create_engine, inspect, text +from sqlalchemy.dialects.postgresql import insert as pg_insert +from sqlalchemy.dialects.sqlite import insert as sqlite_insert from sqlalchemy.exc import ( DataError, NoSuchTableError, @@ -295,6 +297,11 @@ def convert_market_results(self, market_results: list[dict]): df = pd.DataFrame(market_results) df["simulation"] = self.simulation_id + + # in df["node"] replace null with "node0" for consistency in the database + if "node" in df.columns: + df["node"] = df["node"].fillna("node0") + return df def convert_market_orders(self, market_orders: any, market_id: str): @@ -541,15 +548,6 @@ async def store_dfs(self): if self.db is not None: try: self._copy_df_to_db(table, df) - except NoSuchTableError: - # if the table does not exist, create it - self._ensure_table(table, df) - # now try again - self._copy_df_to_db(table, df) - except (ProgrammingError, OperationalError, DataError): - self._check_columns(table, df) - # now try again - self._copy_df_to_db(table, df) except Exception as e: logger.error("could not write to db: %s", e) continue @@ -721,38 +719,141 @@ def _get_columns(self, table: str) -> list[str]: return self._column_cache[table] def _copy_df_to_db(self, table: str, df: pd.DataFrame): - # ---- SQLite fallback ---- - if self.db.dialect.name == "sqlite": - # simple append via pandas - with self.db.begin() as db: - df.to_sql(table, db, if_exists="append") - return - - # 1) If there’s a named index (e.g. time/datetime), turn it into a real column + """ + Main entry: normalize index, then dispatch to the correct backend + and the correct strategy (upsert only for unit_dispatch). + """ + # 1) If there’s a named index, turn it into a real column if df.index.name: df = df.reset_index() - # 2) Introspect the exact column order from Postgres - cols = self._get_columns(table) + # 2) Dispatch based on dialect and table + if self.db.dialect.name == "sqlite": + if table == "unit_dispatch": + return self._copy_df_to_db_sqlite_upsert(table, df) + else: + return self._copy_df_to_db_sqlite_append(table, df) + else: + if table == "unit_dispatch": + return self._copy_df_to_db_postgres_upsert(table, df) + else: + return self._copy_df_to_db_postgres_copy(table, df) + + def _copy_df_to_db_sqlite_append(self, table: str, df: pd.DataFrame): + """ + SQLite append‐only: pandas.to_sql, with retry‐on‐missing‐schema. + """ + try: + with self.db.begin() as conn: + df.to_sql( + name=table, + con=conn, + if_exists="append", + index=df.index.name is not None, + index_label=df.index.name, + ) + except NoSuchTableError: + self._ensure_table(table, df) + return self._copy_df_to_db_sqlite_append(table, df) + except (ProgrammingError, OperationalError, DataError, UndefinedColumn): + self._check_columns(table, df) + return self._copy_df_to_db_sqlite_append(table, df) + except Exception as e: + logger.error("SQLite append failed for %s: %s", table, e) + + def _copy_df_to_db_sqlite_upsert(self, table: str, df: pd.DataFrame): + """ + SQLite UPSERT for unit_dispatch: INSERT...ON CONFLICT DO UPDATE. + """ + # 1) Ensure table exists + if not self._inspector.has_table(table): + self._ensure_table(table, df) + + try: + meta = MetaData() + tbl = Table(table, meta, autoload_with=self.db) + records = df.to_dict(orient="records") + stmt = sqlite_insert(tbl).values(records) + + pk_cols = [c.name for c in tbl.primary_key.columns] + update_cols = [c for c in df.columns if c not in pk_cols] + + stmt = stmt.on_conflict_do_update( + index_elements=pk_cols, + set_={col: getattr(stmt.excluded, col) for col in update_cols}, + ) + with self.db.begin() as conn: + conn.execute(stmt) - # 3) Reindex the DataFrame to those columns (missing → NaN) - df = df.reindex(columns=cols) + except (ProgrammingError, OperationalError, DataError, UndefinedColumn): + self._check_columns(table, df) + return self._copy_df_to_db_sqlite_upsert(table, df) + except Exception as e: + logger.error("SQLite upsert failed for %s: %s", table, e) - # 4) Serialize CSV with header - with StringIO() as buffer: - df.to_csv(buffer, index=False, header=True) - buffer.seek(0) + def _copy_df_to_db_postgres_copy(self, table: str, df: pd.DataFrame): + """ + Postgres COPY for all tables except unit_dispatch. + """ + try: + buf = StringIO() + df.to_csv(buf, index=False, header=True) + buf.seek(0) + + cols = df.columns.tolist() + col_list = ",".join(f'"{c}"' for c in cols) + sql = ( + f'COPY "{table}" ({col_list}) FROM STDIN WITH (FORMAT CSV, HEADER TRUE)' + ) - # 5) Bulk‐load via COPY, matching on header - col_list = ",".join(cols) - sql = f"COPY {table} ({col_list}) FROM STDIN WITH (FORMAT CSV, HEADER TRUE)" - raw_connection = self.db.raw_connection() + raw = self.db.raw_connection() + cur = raw.cursor() + cur.copy_expert(sql, buf) + raw.commit() + + except NoSuchTableError: + self._ensure_table(table, df) + return self._copy_df_to_db_postgres_copy(table, df) + except (ProgrammingError, OperationalError, DataError, UndefinedColumn): + self._check_columns(table, df) + return self._copy_df_to_db_postgres_copy(table, df) + except Exception as e: + logger.error("Postgres COPY failed for %s: %s", table, e) + finally: try: - cursor = raw_connection.cursor() - cursor.copy_expert(sql, buffer) - raw_connection.commit() - finally: - cursor.close() + cur.close() + except Exception: + pass + + def _copy_df_to_db_postgres_upsert(self, table: str, df: pd.DataFrame): + """ + Postgres UPSERT for unit_dispatch: INSERT...ON CONFLICT DO UPDATE. + """ + # 1) Ensure table exists + if not self._inspector.has_table(table): + self._ensure_table(table, df) + + try: + meta = MetaData() + tbl = Table(table, meta, autoload_with=self.db) + records = df.to_dict(orient="records") + + stmt = pg_insert(tbl).values(records) + pk_cols = [c.name for c in tbl.primary_key.columns] + update_cols = [c for c in df.columns if c not in pk_cols] + + stmt = stmt.on_conflict_do_update( + index_elements=pk_cols, + set_={col: getattr(stmt.excluded, col) for col in update_cols}, + ) + with self.db.begin() as conn: + conn.execute(stmt) + + except (ProgrammingError, OperationalError, DataError, UndefinedColumn): + self._check_columns(table, df) + return self._copy_df_to_db_postgres_upsert(table, df) + except Exception as e: + logger.error("Postgres upsert failed for %s: %s", table, e) def _create_partitions(self, simulation_id: str): """Create one child‐partition per parent table for this simulation_id.""" @@ -789,36 +890,53 @@ def _check_columns(self, table: str, df: pd.DataFrame, index: bool = True): Args: table (str): The name of the database table. df (pandas.DataFrame): The DataFrame to be checked. + index (bool): Whether to also ensure the index name exists as a column. """ - with self.db.begin() as db: - # Read table into Pandas DataFrame - query = f"select * from {table} where 1=0" - db_columns = pd.read_sql(query, db).columns + # 1) Fetch current columns from the database + with self.db.begin() as conn: + query = f"SELECT * FROM {table} WHERE 1=0" + db_columns = pd.read_sql(query, conn).columns + + # Normalize to lowercase for robust comparison + db_cols_lower = [c.lower() for c in db_columns] + # 2) Add any missing DataFrame columns for column in df.columns: - if column.lower() not in db_columns: + if column.lower() not in db_cols_lower: try: - # TODO this only works for float and text if is_bool_dtype(df[column]): - column_type = "boolean" + column_type = "BOOLEAN" elif is_numeric_dtype(df[column]): - column_type = "float" + column_type = "DOUBLE PRECISION" else: - column_type = "text" - query = f"ALTER TABLE {table} ADD COLUMN {column} {column_type}" - with self.db.begin() as db: - db.execute(text(query)) + column_type = "TEXT" + alter = f'ALTER TABLE {table} ADD COLUMN "{column}" {column_type}' + with self.db.begin() as conn: + conn.execute(text(alter)) + logger.debug("Added column %s to table %s", column, table) + # update our lowercase cache list immediately + db_cols_lower.append(column.lower()) except Exception: - logger.exception("Error converting column") + logger.exception("Error adding column %s to %s", column, table) + # 3) Optionally add the index name as a column if index and df.index.name: - df.index.name = df.index.name.lower() - if df.index.name in db_columns: - return - column_type = "float" if is_numeric_dtype(df.index) else "text" - query = f"ALTER TABLE {table} ADD COLUMN {df.index.name} {column_type}" - with self.db.begin() as db: - db.execute(text(query)) + idx = df.index.name.lower() + if idx not in db_cols_lower: + try: + column_type = ( + "DOUBLE PRECISION" if is_numeric_dtype(df.index) else "TEXT" + ) + alter = f'ALTER TABLE {table} ADD COLUMN "{idx}" {column_type}' + with self.db.begin() as conn: + conn.execute(text(alter)) + logger.info("Added index-column %s to table %s", idx, table) + db_cols_lower.append(idx) + except Exception: + logger.exception("Error adding index column %s to %s", idx, table) + + # 4) Invalidate the cached column list so _get_columns() will re-fetch next time + self._column_cache.pop(table, None) def _purge_simulation_data_sqlite(self, simulation_id: str): """ diff --git a/docker_configs/dashboard-definitions/ASSUME Comparison.json b/docker_configs/dashboard-definitions/ASSUME Comparison.json index a3b843f68..9f238e2a6 100644 --- a/docker_configs/dashboard-definitions/ASSUME Comparison.json +++ b/docker_configs/dashboard-definitions/ASSUME Comparison.json @@ -1463,7 +1463,7 @@ "group": [], "metricColumn": "none", "rawQuery": true, - "rawSql": "SELECT * FROM power_plant_meta\nWHERE index in ($Gen_Units) and simulation = '$simulation'\n", + "rawSql": "SELECT * FROM power_plant_meta\nWHERE unit_id in ($Gen_Units) and simulation = '$simulation'\n", "refId": "A", "select": [ [ @@ -1949,7 +1949,7 @@ "group": [], "metricColumn": "none", "rawQuery": true, - "rawSql": "SELECT * FROM demand_meta\nWHERE index in ($Demand_Units) and simulation = '$simulation'\n", + "rawSql": "SELECT * FROM demand_meta\nWHERE unit_id in ($Demand_Units) and simulation = '$simulation'\n", "refId": "A", "select": [ [ @@ -2212,7 +2212,7 @@ "group": [], "metricColumn": "none", "rawQuery": true, - "rawSql": "SELECT * FROM storage_meta\nWHERE index in ($Storage_Units) and simulation = '$simulation'\n", + "rawSql": "SELECT * FROM storage_meta\nWHERE unit_id in ($Storage_Units) and simulation = '$simulation'\n", "refId": "A", "select": [ [ @@ -2324,13 +2324,13 @@ "type": "postgres", "uid": "P7B13B9DF907EC40C" }, - "definition": "SELECT index\nFROM power_plant_meta\nwhere simulation = '$simulation';", + "definition": "SELECT unit_id\nFROM power_plant_meta\nwhere simulation = '$simulation';", "description": "Can choose which units we want to display ", "includeAll": false, "multi": true, "name": "Gen_Units", "options": [], - "query": "SELECT index\nFROM power_plant_meta\nwhere simulation = '$simulation';", + "query": "SELECT unit_id\nFROM power_plant_meta\nwhere simulation = '$simulation';", "refresh": 2, "regex": "", "sort": 1, @@ -2349,13 +2349,13 @@ "type": "postgres", "uid": "P7B13B9DF907EC40C" }, - "definition": "SELECT index\nFROM demand_meta\nwhere simulation = '$simulation';", + "definition": "SELECT unit_id\nFROM demand_meta\nwhere simulation = '$simulation';", "description": "Can choose which units we want to display ", "includeAll": false, "multi": true, "name": "Demand_Units", "options": [], - "query": "SELECT index\nFROM demand_meta\nwhere simulation = '$simulation';", + "query": "SELECT unit_id\nFROM demand_meta\nwhere simulation = '$simulation';", "refresh": 2, "regex": "", "sort": 1, @@ -2370,13 +2370,13 @@ "type": "postgres", "uid": "P7B13B9DF907EC40C" }, - "definition": "SELECT index\nFROM storage_meta\nwhere simulation = '$simulation';", + "definition": "SELECT unit_id\nFROM storage_meta\nwhere simulation = '$simulation';", "description": "Can choose which storage units we want to display ", "includeAll": false, "multi": true, "name": "Storage_Units", "options": [], - "query": "SELECT index\nFROM storage_meta\nwhere simulation = '$simulation';", + "query": "SELECT unit_id\nFROM storage_meta\nwhere simulation = '$simulation';", "refresh": 2, "regex": "", "sort": 1, diff --git a/docker_configs/dashboard-definitions/ASSUME-Support.json b/docker_configs/dashboard-definitions/ASSUME-Support.json deleted file mode 100644 index 022223497..000000000 --- a/docker_configs/dashboard-definitions/ASSUME-Support.json +++ /dev/null @@ -1,5280 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": 23, - "links": [], - "panels": [ - { - "description": "", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 4, - "options": { - "code": { - "language": "plaintext", - "showLineNumbers": false, - "showMiniMap": false - }, - "content": "# Welcome to our ASSUME demo\n\nThis is the Grafana Dashboard which makes interacting with the simulation data very easy.", - "mode": "markdown" - }, - "pluginVersion": "11.4.0", - "title": "Overview Dashboard", - "type": "text" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 4 - }, - "id": 30, - "panels": [], - "title": "Market Data", - "type": "row" - }, - { - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 24, - "x": 0, - "y": 5 - }, - "id": 17, - "options": { - "code": { - "language": "plaintext", - "showLineNumbers": false, - "showMiniMap": false - }, - "content": "# Market-specific Data\n\nData specific for the market depending on the choice made at te top of the panel\n\n", - "mode": "markdown" - }, - "pluginVersion": "11.4.0", - "title": "", - "type": "text" - }, - { - "datasource": { - "type": "postgres", - "uid": "P7B13B9DF907EC40C" - }, - "description": "Overview of market results for the chossen market", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "left", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 1, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "stepAfter", - "lineStyle": { - "fill": "solid" - }, - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "megwatt" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/Price.*/" - }, - "properties": [ - { - "id": "unit", - "value": "€/MWh" - }, - { - "id": "custom.axisPlacement", - "value": "auto" - } - ] - } - ] - }, - "gridPos": { - "h": 12, - "w": 24, - "x": 0, - "y": 8 - }, - "id": 11, - "maxPerRow": 4, - "options": { - "legend": { - "calcs": [ - "min", - "max", - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "timezone": [ - "" - ], - "tooltip": { - "maxHeight": 600, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.4.0", - "targets": [ - { - "datasource": { - "type": "postgres", - "uid": "P7B13B9DF907EC40C" - }, - "editorMode": "code", - "format": "time_series", - "group": [], - "metricColumn": "none", - "rawQuery": true, - "rawSql": "SELECT\n $__timeGroupAlias(product_start, $__interval),\n avg(demand_volume) AS \"Demand volume\",\n avg(supply_volume) AS \"Supply volume\",\n avg(price) AS \"Price\",\n node\nFROM market_meta\nWHERE (\"simulation\" LIKE '$simulation') AND \"market_id\" ='$market' AND $__timeFilter(product_start)\nGROUP BY 1, market_id, simulation, node\nORDER BY 1, node;\n", - "refId": "Volume", - "select": [ - [ - { - "params": [ - "supply_volume" - ], - "type": "column" - } - ] - ], - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - }, - "table": "market_meta", - "timeColumn": "product_start", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "title": "Market Summary $market", - "type": "timeseries" - }, - { - "datasource": { - "type": "postgres", - "uid": "P7B13B9DF907EC40C" - }, - "description": "Big Point Size is demand match. Unused generation after that point", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "fillOpacity": 50, - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineStyle": { - "fill": "solid" - }, - "pointShape": "circle", - "pointSize": { - "fixed": 2, - "max": 20, - "min": 1 - }, - "pointStrokeWidth": 1, - "scaleDistribution": { - "type": "linear" - }, - "show": "points+lines" - }, - "fieldMinMax": false, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "megwatt" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Price" - }, - "properties": [ - { - "id": "unit", - "value": "€/MWh" - } - ] - } - ] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 20 - }, - "id": 100, - "maxPerRow": 4, - "options": { - "legend": { - "calcs": [ - "max", - "min" - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "mapping": "manual", - "series": [ - { - "frame": { - "matcher": { - "id": "byIndex", - "options": 1 - } - }, - "name": { - "fixed": "Demand" - }, - "size": { - "matcher": { - "id": "byName", - "options": "size" - } - }, - "x": { - "matcher": { - "id": "byName", - "options": "Volume" - } - }, - "y": { - "matcher": { - "id": "byName", - "options": "Price" - } - } - }, - { - "frame": { - "matcher": { - "id": "byIndex", - "options": 0 - } - }, - "name": { - "fixed": "Generation" - }, - "size": { - "matcher": { - "id": "byName", - "options": "size" - } - }, - "x": { - "matcher": { - "id": "byName", - "options": "Volume" - } - }, - "y": { - "matcher": { - "id": "byName", - "options": "Price" - } - } - } - ], - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.4.0", - "targets": [ - { - "datasource": { - "type": "postgres", - "uid": "P7B13B9DF907EC40C" - }, - "editorMode": "code", - "format": "table", - "group": [], - "metricColumn": "none", - "rawQuery": true, - "rawSql": "SELECT\n \n sum(volume::float) OVER (PARTITION BY market_id ORDER BY price asc, unit_id) AS \"Volume\",\n price::float AS \"Price\", unit_id,\n 1/(volume - accepted_volume +1)/2 as size\n --'rgba(255,255,255,1)' as delta\nFROM market_orders\nWHERE\n \n start_time = (select min(start_time) from market_orders where market_id = '$market' AND\n simulation = '$simulation' and $__timeFilter(start_time))\n AND\n volume > 0 AND\n --accepted_volume != 0 AND\n market_id = '$market' AND\n simulation = '$simulation'\nGROUP BY unit_id, volume, accepted_volume, price, market_id --, bid_id\nORDER BY price asc\n\n-- ", - "refId": "Volume", - "select": [ - [ - { - "params": [ - "supply_volume" - ], - "type": "column" - } - ] - ], - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - }, - "table": "market_meta", - "timeColumn": "product_start", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - }, - { - "datasource": { - "type": "grafana-postgresql-datasource", - "uid": "P7B13B9DF907EC40C" - }, - "editorMode": "code", - "format": "table", - "hide": false, - "rawQuery": true, - "rawSql": "SELECT\n -sum(volume::float) OVER (PARTITION BY market_id ORDER BY accepted_price asc, unit_id) AS \"Volume\",\n accepted_price::float AS \"Price\", unit_id,\n 10 as size\nFROM market_orders\nWHERE\n \n start_time = (select min(start_time) from market_orders where market_id = '$market' AND\n simulation = '$simulation' and $__timeFilter(start_time))\n AND\n volume < 0 AND\n --accepted_volume != 0 AND\n market_id = '$market' AND\n simulation = '$simulation'\nGROUP BY unit_id, volume, accepted_volume, accepted_price, market_id --, bid_id\nORDER BY accepted_price asc\n\n-- ", - "refId": "Demand", - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - } - } - ], - "title": "Merit Order $market at $min_time", - "type": "xychart" - }, - { - "datasource": { - "type": "postgres", - "uid": "P7B13B9DF907EC40C" - }, - "description": "Dispatch by technology", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 76, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "megwatt" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "volume solar" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "yellow", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "volume wind_offshore" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "dark-blue", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "volume wind_onshore" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "super-light-blue", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "volume biomass" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "green", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "volume gas" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "orange", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "volume hard coal" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "text", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "volume hydro" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "blue", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "volume lignite" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#a52a2a", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "volume nuclear" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "light-red", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "volume oil" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#120c12", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 20 - }, - "id": 101, - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "maxHeight": 600, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.4.0", - "targets": [ - { - "datasource": { - "type": "postgres", - "uid": "P7B13B9DF907EC40C" - }, - "editorMode": "code", - "format": "time_series", - "group": [ - { - "params": [ - "$__interval", - "none" - ], - "type": "time" - }, - { - "params": [ - "unit_id" - ], - "type": "column" - } - ], - "metricColumn": "none", - "rawQuery": true, - "rawSql": "select $__timeGroupAlias(time,$__interval), avg(volume) as \"volume\", tech FROM \n(\nSELECT\n $__timeGroupAlias(start_time,'1h'),\n sum(accepted_volume) AS \"volume\", pm.technology as tech\nFROM market_orders mo\njoin power_plant_meta pm on pm.index=mo.unit_id and pm.simulation=mo.simulation\nWHERE\n\n $__timeFilter(start_time) AND\n market_id = '$market' AND\n mo.simulation = '$simulation'\n\nGROUP BY 1, pm.technology\nORDER BY 1\n\n) a\ngroup by 1, tech\nORDER BY 1, tech desc", - "refId": "A", - "select": [ - [ - { - "params": [ - "volume" - ], - "type": "column" - }, - { - "params": [ - "avg" - ], - "type": "aggregate" - }, - { - "params": [ - "volume" - ], - "type": "alias" - } - ], - [ - { - "params": [ - "unit_id" - ], - "type": "column" - }, - { - "params": [ - "unit_id" - ], - "type": "alias" - } - ] - ], - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - }, - "table": "market_orders", - "timeColumn": "start_time", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - }, - { - "datatype": "text", - "name": "", - "params": [ - "market_id", - "=", - "'$market'" - ], - "type": "expression" - }, - { - "datatype": "text", - "name": "", - "params": [ - "simulation", - "=", - "'$simulation'" - ], - "type": "expression" - } - ] - } - ], - "title": "Market result by technology", - "type": "timeseries" - }, - { - "datasource": { - "type": "postgres", - "uid": "P7B13B9DF907EC40C" - }, - "description": "Bid prices of accepted bids per unit in the chosen market", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "€/MW" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 29 - }, - "id": 19, - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "maxHeight": 600, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.4.0", - "targets": [ - { - "datasource": { - "type": "postgres", - "uid": "P7B13B9DF907EC40C" - }, - "editorMode": "code", - "format": "time_series", - "group": [ - { - "params": [ - "$__interval", - "none" - ], - "type": "time" - }, - { - "params": [ - "unit_id" - ], - "type": "column" - } - ], - "metricColumn": "none", - "rawQuery": true, - "rawSql": "SELECT\n $__timeGroupAlias(start_time,$__interval),\n avg(accepted_price::float) AS \"Price\", unit_id\nFROM market_orders\nWHERE\n $__timeFilter(start_time) AND\n market_id = '$market' AND\n simulation = '$simulation'\nGROUP BY 1, unit_id --, bid_id\nORDER BY 1", - "refId": "A", - "select": [ - [ - { - "params": [ - "price" - ], - "type": "column" - }, - { - "params": [ - "avg" - ], - "type": "aggregate" - }, - { - "params": [ - "price" - ], - "type": "alias" - } - ], - [ - { - "params": [ - "unit_id" - ], - "type": "column" - }, - { - "params": [ - "unit_id" - ], - "type": "alias" - } - ] - ], - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - }, - "table": "market_orders", - "timeColumn": "start_time", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - }, - { - "datatype": "text", - "name": "", - "params": [ - "market_id", - "=", - "'$market'" - ], - "type": "expression" - }, - { - "datatype": "text", - "name": "", - "params": [ - "simulation", - "=", - "'$simulation'" - ], - "type": "expression" - } - ] - } - ], - "title": "Average Accepted Bid Price", - "type": "timeseries" - }, - { - "datasource": { - "type": "postgres", - "uid": "P7B13B9DF907EC40C" - }, - "description": "Accepted Volume per unit in chosen market", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "megwatt" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 29 - }, - "id": 20, - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "maxHeight": 600, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.4.0", - "targets": [ - { - "datasource": { - "type": "postgres", - "uid": "P7B13B9DF907EC40C" - }, - "editorMode": "code", - "format": "time_series", - "group": [ - { - "params": [ - "$__interval", - "none" - ], - "type": "time" - }, - { - "params": [ - "unit_id" - ], - "type": "column" - } - ], - "metricColumn": "none", - "rawQuery": true, - "rawSql": "SELECT\n $__timeGroupAlias(start_time,$__interval),\n avg(accepted_volume) AS \"Volume\", unit_id\nFROM market_orders\nWHERE\n $__timeFilter(start_time) AND\n market_id = '$market' AND\n simulation = '$simulation'\nGROUP BY 1, unit_id, bid_id\nORDER BY 1", - "refId": "A", - "select": [ - [ - { - "params": [ - "volume" - ], - "type": "column" - }, - { - "params": [ - "avg" - ], - "type": "aggregate" - }, - { - "params": [ - "volume" - ], - "type": "alias" - } - ], - [ - { - "params": [ - "unit_id" - ], - "type": "column" - }, - { - "params": [ - "unit_id" - ], - "type": "alias" - } - ] - ], - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - }, - "table": "market_orders", - "timeColumn": "start_time", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - }, - { - "datatype": "text", - "name": "", - "params": [ - "market_id", - "=", - "'$market'" - ], - "type": "expression" - }, - { - "datatype": "text", - "name": "", - "params": [ - "simulation", - "=", - "'$simulation'" - ], - "type": "expression" - } - ] - } - ], - "title": "Average Accepted Bid Volume", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 38 - }, - "id": 74, - "panels": [], - "title": "Key performance indicators", - "type": "row" - }, - { - "datasource": { - "type": "postgres", - "uid": "P7B13B9DF907EC40C" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "fillOpacity": 80, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineWidth": 1, - "scaleDistribution": { - "type": "linear" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "currencyEUR" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Total volume" - }, - "properties": [ - { - "id": "unit", - "value": "megwatt" - }, - { - "id": "custom.axisPlacement", - "value": "right" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total cost" - }, - "properties": [ - { - "id": "custom.axisPlacement", - "value": "left" - }, - { - "id": "unit", - "value": "currencyEUR" - } - ] - } - ] - }, - "gridPos": { - "h": 12, - "w": 10, - "x": 0, - "y": 39 - }, - "id": 72, - "options": { - "barRadius": 0, - "barWidth": 0.97, - "fullHighlight": false, - "groupWidth": 0.7, - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "orientation": "vertical", - "showValue": "never", - "stacking": "none", - "tooltip": { - "maxHeight": 600, - "mode": "multi", - "sort": "none" - }, - "xField": "simulation", - "xTickLabelRotation": 0, - "xTickLabelSpacing": 0 - }, - "pluginVersion": "11.4.0", - "targets": [ - { - "datasource": { - "type": "postgres", - "uid": "P7B13B9DF907EC40C" - }, - "format": "table", - "group": [], - "hide": false, - "metricColumn": "none", - "rawQuery": true, - "rawSql": "SELECT\n simulation,\n sum(round(CAST(value AS numeric), 2)) FILTER (WHERE variable = 'total_cost') as \"Total cost\",\n sum(round(CAST(value AS numeric), 2)) FILTER (WHERE variable = 'total_volume') as \"Total volume\"\nFROM kpis\nWHERE simulation = '$simulation'\ngroup by simulation", - "refId": "Cost", - "select": [ - [ - { - "params": [ - "power" - ], - "type": "column" - } - ] - ], - "table": "market_dispatch", - "timeColumn": "datetime", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "title": "Total Cost and Volume", - "type": "barchart" - }, - { - "datasource": { - "type": "postgres", - "uid": "P7B13B9DF907EC40C" - }, - "fieldConfig": { - "defaults": { - "custom": { - "align": "auto", - "cellOptions": { - "type": "auto" - }, - "inspect": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "total_volume" - }, - "properties": [ - { - "id": "unit", - "value": "kwatth" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "average_cost" - }, - "properties": [ - { - "id": "unit", - "value": "€/MW" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "total_costs" - }, - "properties": [ - { - "id": "unit", - "value": "currencyEUR" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "simulation" - }, - "properties": [ - { - "id": "custom.width", - "value": 372 - } - ] - } - ] - }, - "gridPos": { - "h": 12, - "w": 9, - "x": 10, - "y": 39 - }, - "id": 76, - "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "showHeader": true, - "sortBy": [ - { - "desc": true, - "displayName": "simulation" - } - ] - }, - "pluginVersion": "11.4.0", - "targets": [ - { - "datasource": { - "type": "postgres", - "uid": "P7B13B9DF907EC40C" - }, - "editorMode": "code", - "format": "table", - "group": [], - "hide": false, - "metricColumn": "none", - "rawQuery": true, - "rawSql": "SELECT\n simulation,\n avg(round(CAST(value AS numeric), 2)) FILTER (WHERE variable = 'avg_price') as avg_price,\n sum(round(CAST(value AS numeric), 2)) FILTER (WHERE variable = 'total_cost') as total_cost,\n sum(round(CAST(value AS numeric), 2)*1000) FILTER (WHERE variable = 'total_volume') as total_volume\nFROM kpis\ngroup by simulation\nORDER BY simulation", - "refId": "A", - "select": [ - [ - { - "params": [ - "power" - ], - "type": "column" - } - ] - ], - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - }, - "table": "market_dispatch", - "timeColumn": "datetime", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "title": "Key indicators", - "type": "table" - }, - { - "datasource": { - "type": "postgres", - "uid": "P7B13B9DF907EC40C" - }, - "description": "Shows installed capacities in the simulation based on the fuel type", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - } - }, - "mappings": [], - "unit": "megwatt" - }, - "overrides": [] - }, - "gridPos": { - "h": 12, - "w": 5, - "x": 19, - "y": 39 - }, - "id": 7, - "options": { - "displayLabels": [ - "value", - "name" - ], - "legend": { - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "pieType": "pie", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "/^sum$/", - "values": true - }, - "tooltip": { - "maxHeight": 600, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.4.0", - "targets": [ - { - "datasource": { - "type": "postgres", - "uid": "P7B13B9DF907EC40C" - }, - "format": "table", - "group": [ - { - "params": [ - "fuel_type" - ], - "type": "column" - } - ], - "hide": false, - "metricColumn": "none", - "rawQuery": true, - "rawSql": "SELECT \n NOW() as time_sec,\n SUM(max_power),\n technology\nFROM power_plant_meta\nWHERE \"simulation\" LIKE '$simulation'\nGROUP BY technology, simulation;\n\n\n", - "refId": "Generation", - "select": [ - [ - { - "params": [ - "max_power" - ], - "type": "column" - }, - { - "params": [ - "avg" - ], - "type": "aggregate" - }, - { - "params": [ - "max_power" - ], - "type": "alias" - } - ] - ], - "table": "power_plant_meta", - "timeColumn": "fuel_type", - "timeColumnType": "text", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "title": "Installed Generation Capacities", - "type": "piechart" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 51 - }, - "id": 41, - "panels": [], - "title": "Generation units $Gen_Units", - "type": "row" - }, - { - "description": "", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 24, - "x": 0, - "y": 52 - }, - "id": 22, - "options": { - "code": { - "language": "plaintext", - "showLineNumbers": false, - "showMiniMap": false - }, - "content": "# Unit Specific Data\n\nFor the chosen market and the chosen unit here the dispatch is displayed.", - "mode": "markdown" - }, - "pluginVersion": "11.4.0", - "title": "", - "type": "text" - }, - { - "datasource": { - "type": "postgres", - "uid": "P7B13B9DF907EC40C" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineStyle": { - "fill": "solid" - }, - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "always", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "megwatt" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 18, - "x": 0, - "y": 55 - }, - "id": 24, - "interval": "15m", - "options": { - "legend": { - "calcs": [ - "min", - "max", - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "maxHeight": 600, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.4.0", - "targets": [ - { - "datasource": { - "type": "postgres", - "uid": "P7B13B9DF907EC40C" - }, - "editorMode": "code", - "format": "time_series", - "group": [], - "hide": false, - "metricColumn": "none", - "rawQuery": true, - "rawSql": "SELECT\n $__timeGroupAlias(datetime,$__interval),\n avg(power) AS \"Market dispatch\",\n concat(unit_id, ' - ', market_id) as \"unit\"\nFROM market_dispatch\nWHERE\n $__timeFilter(datetime) AND\n simulation = '$simulation' AND\n unit_id in ($Gen_Units)\nGROUP BY 1, unit_id, power, market_id\nORDER BY 1", - "refId": "A", - "select": [ - [ - { - "params": [ - "volume" - ], - "type": "column" - } - ] - ], - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - }, - "table": "demand_meta", - "timeColumn": "\"Timestamp\"", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - }, - { - "datasource": { - "type": "grafana-postgresql-datasource", - "uid": "P7B13B9DF907EC40C" - }, - "editorMode": "code", - "format": "time_series", - "hide": false, - "rawQuery": true, - "rawSql": "SELECT\n $__timeGroupAlias(time,$__interval),\n avg(power) AS \"Actual dispatch\",\n unit\nFROM unit_dispatch\nWHERE\n $__timeFilter(time) AND\n simulation = '$simulation' AND\n unit in ($Gen_Units)\nGROUP BY 1, unit\nORDER BY 1", - "refId": "B", - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - } - } - ], - "title": "Unitwise Dispatch", - "type": "timeseries" - }, - { - "datasource": { - "type": "postgres", - "uid": "P7B13B9DF907EC40C" - }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 6, - "x": 18, - "y": 55 - }, - "id": 26, - "options": { - "afterRender": "", - "content": "### General Information\n\nName: {{index}}
\nTechnology: {{technology}}
\n\n### Technical Specifications\nEmissions: {{emission_factor}} t/MWh
\nMaximum Power: {{max_power}} MW
\nMinimum Power: {{min_power}} MW
\nEfficiency: {{efficiency}}
\n\n##### Unit Operator: {{unit_operator}}\n ", - "contentPartials": [], - "defaultContent": "The query didn't return any results.", - "editor": { - "format": "auto", - "height": 200, - "language": "markdown" - }, - "editors": [], - "externalScripts": [], - "externalStyles": [], - "helpers": "", - "renderMode": "everyRow", - "styles": "", - "wrap": true - }, - "pluginVersion": "5.6.0", - "targets": [ - { - "datasource": { - "type": "postgres", - "uid": "P7B13B9DF907EC40C" - }, - "editorMode": "code", - "format": "table", - "group": [], - "metricColumn": "none", - "rawQuery": true, - "rawSql": "SELECT * FROM power_plant_meta\nWHERE index in ($Gen_Units) and simulation = '$simulation'\n", - "refId": "A", - "select": [ - [ - { - "params": [ - "volume" - ], - "type": "column" - } - ] - ], - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - }, - "table": "demand_meta", - "timeColumn": "\"Timestamp\"", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "title": "Chosen Unit Specifications", - "type": "marcusolsson-dynamictext-panel" - }, - { - "datasource": { - "type": "postgres", - "uid": "P7B13B9DF907EC40C" - }, - "description": "Bid prices of accepted bids per unit in the chosen market", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "currencyEUR" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "price .*" - }, - "properties": [ - { - "id": "unit", - "value": "€/MW" - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 64 - }, - "id": 70, - "options": { - "legend": { - "calcs": [ - "min", - "max", - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "maxHeight": 600, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.4.0", - "targets": [ - { - "datasource": { - "type": "postgres", - "uid": "P7B13B9DF907EC40C" - }, - "format": "time_series", - "group": [], - "hide": false, - "metricColumn": "none", - "rawQuery": true, - "rawSql": "SELECT\r\n $__timeGroupAlias(start_time,$__interval),\r\n avg(accepted_price::float) AS \"Accepted price:\",\r\n avg(price) AS \"Bid price:\",\r\n concat(unit_id, ' - ', market_id) as \"unit_id\"\r\nFROM market_orders\r\nWHERE\r\n $__timeFilter(start_time) AND\r\n unit_id in ($Gen_Units) AND\r\n simulation = '$simulation'\r\nGROUP BY 1, unit_id, market_id\r\nORDER BY 1\r\n", - "refId": "B", - "select": [ - [ - { - "params": [ - "power" - ], - "type": "column" - } - ] - ], - "table": "market_dispatch", - "timeColumn": "datetime", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - }, - { - "datasource": { - "type": "postgres", - "uid": "P7B13B9DF907EC40C" - }, - "format": "time_series", - "group": [ - { - "params": [ - "$__interval", - "none" - ], - "type": "time" - }, - { - "params": [ - "unit_id" - ], - "type": "column" - } - ], - "hide": true, - "metricColumn": "none", - "rawQuery": true, - "rawSql": "SELECT\r\n $__timeGroupAlias(start_time,$__interval),\r\n price AS \"Bid price:\",\r\n unit_id as \"unit_id\",\r\n bid_id as \"bid_id\"\r\nFROM market_orders\r\nWHERE\r\n $__timeFilter(start_time) AND\r\n unit_id in ($Gen_Units) AND\r\n simulation = '$simulation'\r\nGROUP BY 1, unit_id, market_id, price, bid_id\r\nORDER BY 1", - "refId": "A", - "select": [ - [ - { - "params": [ - "original_price" - ], - "type": "column" - }, - { - "params": [ - "avg" - ], - "type": "aggregate" - }, - { - "params": [ - "price" - ], - "type": "alias" - } - ], - [ - { - "params": [ - "unit_id" - ], - "type": "column" - }, - { - "params": [ - "unit_id" - ], - "type": "alias" - } - ] - ], - "table": "market_orders", - "timeColumn": "start_time", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - }, - { - "datatype": "text", - "name": "", - "params": [ - "market_id", - "=", - "'$market'" - ], - "type": "expression" - }, - { - "datatype": "text", - "name": "", - "params": [ - "simulation", - "=", - "'$simulation'" - ], - "type": "expression" - } - ] - } - ], - "title": "Bid Prices", - "type": "timeseries" - }, - { - "datasource": { - "type": "postgres", - "uid": "P7B13B9DF907EC40C" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "megwatt" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "volume_acceptance_ratio" - }, - "properties": [ - { - "id": "custom.axisPlacement", - "value": "right" - }, - { - "id": "min", - "value": 0 - }, - { - "id": "max", - "value": 1 - }, - { - "id": "custom.axisLabel", - "value": "Relative acceptance of bid volume" - }, - { - "id": "unit" - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 64 - }, - "id": 78, - "options": { - "legend": { - "calcs": [ - "min", - "max", - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "maxHeight": 600, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.4.0", - "targets": [ - { - "datasource": { - "type": "postgres", - "uid": "P7B13B9DF907EC40C" - }, - "format": "time_series", - "group": [], - "metricColumn": "none", - "rawQuery": true, - "rawSql": "SELECT\r\n $__timeGroupAlias(start_time,$__interval),\r\n sum(accepted_volume) AS \"Accepted volume:\",\r\n avg(volume) AS \"Bid volume:\",\r\n concat(unit_id, ' - ', market_id) as \"unit_id\"\r\nFROM market_orders\r\nWHERE\r\n $__timeFilter(start_time) AND\r\n unit_id in ($Gen_Units) AND\r\n simulation = '$simulation'\r\nGROUP BY 1, unit_id, market_id\r\nORDER BY 1\r\n", - "refId": "A", - "select": [ - [ - { - "params": [ - "power" - ], - "type": "column" - } - ] - ], - "table": "market_dispatch", - "timeColumn": "datetime", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - }, - { - "datasource": { - "type": "postgres", - "uid": "P7B13B9DF907EC40C" - }, - "format": "time_series", - "group": [], - "hide": true, - "metricColumn": "none", - "rawQuery": true, - "rawSql": "SELECT\r\n $__timeGroupAlias(start_time,$__interval),\r\n volume AS \"Bid volume:\",\r\n unit_id as \"unit_id\",\r\n bid_id as \"bid_id\"\r\nFROM market_orders\r\nWHERE\r\n $__timeFilter(start_time) AND\r\n unit_id in ($Gen_Units) AND\r\n simulation = '$simulation'\r\nGROUP BY 1, unit_id, market_id, volume, bid_id\r\nORDER BY 1", - "refId": "B", - "select": [ - [ - { - "params": [ - "power" - ], - "type": "column" - } - ] - ], - "table": "market_dispatch", - "timeColumn": "datetime", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "title": "Bid volume", - "transformations": [ - { - "id": "calculateField", - "options": { - "alias": "volume_acceptance_ratio", - "binary": { - "left": "accepted_volume Unit 1", - "operator": "/", - "reducer": "sum", - "right": "bid_volume Unit 1" - }, - "mode": "binary", - "reduce": { - "reducer": "sum" - } - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "postgres", - "uid": "P7B13B9DF907EC40C" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "currencyEUR" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "Profit .*" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "yellow", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "Production costs .*" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "purple", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 9, - "w": 24, - "x": 0, - "y": 72 - }, - "id": 93, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "sum" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "11.4.0", - "targets": [ - { - "datasource": { - "type": "postgres", - "uid": "P7B13B9DF907EC40C" - }, - "editorMode": "code", - "format": "time_series", - "group": [], - "metricColumn": "none", - "rawQuery": true, - "rawSql": "SELECT\r\n $__timeGroupAlias(time,$__interval),\r\n avg(energy_cashflow + coalesce(financial_support_cashflow,0)) AS \"Cashflow\",\r\n avg(energy_generation_costs) AS \"Production costs\",\r\n avg(energy_cashflow + coalesce(financial_support_cashflow, 0) - energy_generation_costs) AS \"Profit\",\r\n max(financial_support_cashflow) AS \"Support Cashflow\",\r\n unit\r\nFROM unit_dispatch\r\nWHERE\r\n $__timeFilter(time) AND\r\n simulation = '$simulation' AND\r\n unit in ($Gen_Units)\r\nGROUP BY 1, unit\r\nORDER BY 1\r\n", - "refId": "A", - "select": [ - [ - { - "params": [ - "power" - ], - "type": "column" - } - ] - ], - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - }, - "table": "market_dispatch", - "timeColumn": "datetime", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "title": "Financial Overview ${__from:date:YYYY-MM-DD} until ${__to:date:YYYY-MM-DD}", - "type": "stat" - }, - { - "datasource": { - "type": "postgres", - "uid": "P7B13B9DF907EC40C" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "currencyEUR" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "Profit .*" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "yellow", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "Production costs .*" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "purple", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 5, - "w": 24, - "x": 0, - "y": 81 - }, - "id": 99, - "interval": "1h", - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "maxHeight": 600, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.4.0", - "targets": [ - { - "datasource": { - "type": "postgres", - "uid": "P7B13B9DF907EC40C" - }, - "editorMode": "code", - "format": "time_series", - "group": [], - "metricColumn": "none", - "rawQuery": true, - "rawSql": "SELECT\r\n $__timeGroupAlias(time,$__interval),\r\n avg(energy_cashflow) AS \"Cashflow\",\r\n avg(energy_generation_costs) AS \"Production costs\",\r\n avg(energy_cashflow + coalesce(financial_support_cashflow,0) - energy_generation_costs) AS \"Profit\",\r\n max(financial_support_cashflow) AS \"Support Cashflow\",\r\n unit\r\nFROM unit_dispatch\r\nWHERE\r\n $__timeFilter(time) AND\r\n simulation = '$simulation' AND\r\n unit in ($Gen_Units)\r\nGROUP BY 1, unit\r\nORDER BY 1", - "refId": "A", - "select": [ - [ - { - "params": [ - "power" - ], - "type": "column" - } - ] - ], - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - }, - "table": "market_dispatch", - "timeColumn": "datetime", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "title": "Financial Overview ${__from:date:YYYY-MM-DD} until ${__to:date:YYYY-MM-DD}", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 86 - }, - "id": 39, - "panels": [], - "title": "Demand units data $Demand_Units", - "type": "row" - }, - { - "datasource": { - "type": "postgres", - "uid": "P7B13B9DF907EC40C" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "always", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "megwatt" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 18, - "x": 0, - "y": 87 - }, - "id": 36, - "options": { - "legend": { - "calcs": [ - "min", - "max", - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "maxHeight": 600, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.4.0", - "targets": [ - { - "datasource": { - "type": "postgres", - "uid": "P7B13B9DF907EC40C" - }, - "editorMode": "code", - "format": "time_series", - "group": [], - "metricColumn": "none", - "rawQuery": true, - "rawSql": "SELECT\n $__timeGroupAlias(datetime,$__interval),\n power AS \"Market dispatch\",\n concat(unit_id, ' - ', market_id) as unit\nFROM market_dispatch\nWHERE\n $__timeFilter(datetime) AND\n simulation = '$simulation' AND\n unit_id in ($Demand_Units)\nGROUP BY 1, unit_id, power, market_id\nORDER BY 1", - "refId": "A", - "select": [ - [ - { - "params": [ - "volume" - ], - "type": "column" - } - ] - ], - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - }, - "table": "demand_meta", - "timeColumn": "\"Timestamp\"", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - }, - { - "datasource": { - "type": "postgres", - "uid": "P7B13B9DF907EC40C" - }, - "editorMode": "code", - "format": "time_series", - "group": [], - "hide": false, - "metricColumn": "none", - "rawQuery": true, - "rawSql": "SELECT\n $__timeGroupAlias(time,$__interval),\n power AS \"Actual dispatch\",\n unit\nFROM unit_dispatch\nWHERE\n $__timeFilter(time) AND\n simulation = '$simulation' AND\n unit in ($Demand_Units)\nGROUP BY 1, unit, power\nORDER BY 1", - "refId": "B", - "select": [ - [ - { - "params": [ - "power" - ], - "type": "column" - } - ] - ], - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - }, - "table": "market_dispatch", - "timeColumn": "datetime", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "title": "Unitwise Dispatch", - "type": "timeseries" - }, - { - "datasource": { - "type": "postgres", - "uid": "P7B13B9DF907EC40C" - }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 6, - "x": 18, - "y": 87 - }, - "id": 37, - "options": { - "afterRender": "", - "content": "### General Information\n\nName: {{index}}
\nTechnology: {{technology}}
\n\n### Technical Specifications\nEmissions: {{emission_factor}} t/MWh
\nMaximum Power: {{max_power}} MW
\nMinimum Power: {{min_power}} MW
\nEfficiency: {{efficiency}}
\n\n##### Unit Operator: {{unit_operator}}\n ", - "contentPartials": [], - "defaultContent": "The query didn't return any results.", - "editor": { - "format": "auto", - "height": 200, - "language": "markdown" - }, - "editors": [], - "externalScripts": [], - "externalStyles": [], - "helpers": "", - "renderMode": "everyRow", - "styles": "", - "wrap": true - }, - "pluginVersion": "5.6.0", - "targets": [ - { - "datasource": { - "type": "postgres", - "uid": "P7B13B9DF907EC40C" - }, - "editorMode": "code", - "format": "table", - "group": [], - "metricColumn": "none", - "rawQuery": true, - "rawSql": "SELECT * FROM demand_meta\nWHERE index in ($Demand_Units) and simulation = '$simulation'\n", - "refId": "A", - "select": [ - [ - { - "params": [ - "volume" - ], - "type": "column" - } - ] - ], - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - }, - "table": "demand_meta", - "timeColumn": "\"Timestamp\"", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "title": "Chosen Unit Specifications", - "type": "marcusolsson-dynamictext-panel" - }, - { - "datasource": { - "type": "postgres", - "uid": "P7B13B9DF907EC40C" - }, - "description": "Bid prices of accepted bids per unit in the chosen market", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "currencyEUR" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "price .*" - }, - "properties": [ - { - "id": "unit", - "value": "€/MW" - } - ] - } - ] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 96 - }, - "id": 89, - "options": { - "legend": { - "calcs": [ - "min", - "max", - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "maxHeight": 600, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.4.0", - "targets": [ - { - "datasource": { - "type": "postgres", - "uid": "P7B13B9DF907EC40C" - }, - "format": "time_series", - "group": [ - { - "params": [ - "$__interval", - "none" - ], - "type": "time" - }, - { - "params": [ - "unit_id" - ], - "type": "column" - } - ], - "metricColumn": "none", - "rawQuery": true, - "rawSql": "SELECT\r\n $__timeGroupAlias(start_time,$__interval),\r\n avg(accepted_price::float) AS \"Accepted price:\",\r\n price AS \"Bid price:\",\r\n concat(unit_id, ' - ', market_id) as \"unit_id\"\r\nFROM market_orders\r\nWHERE\r\n $__timeFilter(start_time) AND\r\n unit_id in ($Demand_Units) AND\r\n simulation = '$simulation'\r\nGROUP BY 1, unit_id, market_id, price\r\nORDER BY 1", - "refId": "A", - "select": [ - [ - { - "params": [ - "original_price" - ], - "type": "column" - }, - { - "params": [ - "avg" - ], - "type": "aggregate" - }, - { - "params": [ - "price" - ], - "type": "alias" - } - ], - [ - { - "params": [ - "unit_id" - ], - "type": "column" - }, - { - "params": [ - "unit_id" - ], - "type": "alias" - } - ] - ], - "table": "market_orders", - "timeColumn": "start_time", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - }, - { - "datatype": "text", - "name": "", - "params": [ - "market_id", - "=", - "'$market'" - ], - "type": "expression" - }, - { - "datatype": "text", - "name": "", - "params": [ - "simulation", - "=", - "'$simulation'" - ], - "type": "expression" - } - ] - } - ], - "title": "Bid Prices", - "type": "timeseries" - }, - { - "datasource": { - "type": "postgres", - "uid": "P7B13B9DF907EC40C" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "currencyEUR" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "energy_generation_costs demand_EOM" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "blue", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "Volume .*" - }, - "properties": [ - { - "id": "unit", - "value": "megwatt" - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "Price .*" - }, - "properties": [ - { - "id": "unit", - "value": "€/MW" - } - ] - } - ] - }, - "gridPos": { - "h": 5, - "w": 4, - "x": 12, - "y": 96 - }, - "id": 95, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "sum" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "11.4.0", - "targets": [ - { - "datasource": { - "type": "postgres", - "uid": "P7B13B9DF907EC40C" - }, - "editorMode": "code", - "format": "time_series", - "group": [], - "hide": false, - "metricColumn": "none", - "rawQuery": true, - "rawSql": "SELECT\r\n $__timeGroupAlias(time,$__interval),\r\n -energy_cashflow AS \"Cashflow\",\r\n unit\r\nFROM unit_dispatch d\r\nWHERE\r\n $__timeFilter(time) AND\r\n d.simulation = '$simulation' AND\r\n unit in ($Demand_Units)\r\nGROUP BY 1, unit, energy_cashflow, power\r\nORDER BY 1", - "refId": "A", - "select": [ - [ - { - "params": [ - "power" - ], - "type": "column" - } - ] - ], - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - }, - "table": "market_dispatch", - "timeColumn": "datetime", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "title": "Total Cost of Energy Dispatch", - "type": "stat" - }, - { - "datasource": { - "type": "postgres", - "uid": "P7B13B9DF907EC40C" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "megwatt" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "volume_acceptance_ratio" - }, - "properties": [ - { - "id": "custom.axisPlacement", - "value": "right" - }, - { - "id": "min", - "value": 0 - }, - { - "id": "max", - "value": 1 - }, - { - "id": "custom.axisLabel", - "value": "Relative acceptance of bid volume" - }, - { - "id": "unit" - } - ] - } - ] - }, - "gridPos": { - "h": 5, - "w": 8, - "x": 16, - "y": 96 - }, - "id": 90, - "options": { - "legend": { - "calcs": [ - "min", - "max", - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "maxHeight": 600, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.4.0", - "targets": [ - { - "datasource": { - "type": "postgres", - "uid": "P7B13B9DF907EC40C" - }, - "format": "time_series", - "group": [], - "metricColumn": "none", - "rawQuery": true, - "rawSql": "SELECT\r\n $__timeGroupAlias(start_time,$__interval),\r\n -avg(accepted_volume) AS \"Accepted volume:\",\r\n -avg(volume) AS \"Bid volume:\",\r\n concat(unit_id, ' - ', market_id) as \"unit_id\"\r\nFROM market_orders\r\nWHERE\r\n $__timeFilter(start_time) AND\r\n unit_id in ($Demand_Units) AND\r\n simulation = '$simulation'\r\nGROUP BY 1, unit_id, market_id\r\nORDER BY 1\r\n", - "refId": "A", - "select": [ - [ - { - "params": [ - "power" - ], - "type": "column" - } - ] - ], - "table": "market_dispatch", - "timeColumn": "datetime", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "title": "Bid volume", - "transformations": [ - { - "id": "calculateField", - "options": { - "alias": "volume_acceptance_ratio", - "binary": { - "left": "accepted_volume Unit 1", - "operator": "/", - "reducer": "sum", - "right": "bid_volume Unit 1" - }, - "mode": "binary", - "reduce": { - "reducer": "sum" - } - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "postgres", - "uid": "P7B13B9DF907EC40C" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "€/MW" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 12, - "y": 101 - }, - "id": 96, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "11.4.0", - "targets": [ - { - "datasource": { - "type": "postgres", - "uid": "P7B13B9DF907EC40C" - }, - "editorMode": "code", - "format": "table", - "group": [], - "hide": false, - "metricColumn": "none", - "rawQuery": true, - "rawSql": "SELECT\n sum(energy_cashflow)/sum(power) as \"Price per MW\",\n unit\nFROM unit_dispatch d\nWHERE\n $__timeFilter(time) AND\n d.simulation = '$simulation' AND\n unit in ($Demand_Units)\nGROUP BY unit\nORDER BY 1", - "refId": "A", - "select": [ - [ - { - "params": [ - "power" - ], - "type": "column" - } - ] - ], - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - }, - "table": "market_dispatch", - "timeColumn": "datetime", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "title": "weighted average Price in €/MW for Demand $Demand_Units", - "type": "stat" - }, - { - "datasource": { - "type": "postgres", - "uid": "P7B13B9DF907EC40C" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "kwatth" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 8, - "x": 16, - "y": 101 - }, - "id": 94, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "sum" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "11.4.0", - "targets": [ - { - "datasource": { - "type": "postgres", - "uid": "P7B13B9DF907EC40C" - }, - "editorMode": "code", - "format": "time_series", - "group": [], - "hide": false, - "metricColumn": "none", - "rawQuery": true, - "rawSql": "SELECT\r\n $__timeGroupAlias(time,$__interval),\r\n -power*1e3 as \"Volume\",\r\n unit\r\nFROM unit_dispatch d\r\nWHERE\r\n $__timeFilter(time) AND\r\n d.simulation = '$simulation' AND\r\n unit in ($Demand_Units)\r\nGROUP BY 1, unit, energy_cashflow, power\r\nORDER BY 1", - "refId": "A", - "select": [ - [ - { - "params": [ - "power" - ], - "type": "column" - } - ] - ], - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - }, - "table": "market_dispatch", - "timeColumn": "datetime", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "title": "Total dispatched Volume", - "type": "stat" - }, - { - "datasource": { - "type": "postgres", - "uid": "P7B13B9DF907EC40C" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "currencyEUR" - }, - "overrides": [ - { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "Support Cashflow demand1" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": false, - "tooltip": false, - "viz": true - } - } - ] - } - ] - }, - "gridPos": { - "h": 9, - "w": 24, - "x": 0, - "y": 105 - }, - "id": 86, - "options": { - "legend": { - "calcs": [ - "min", - "max", - "mean", - "sum" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "maxHeight": 600, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.4.0", - "targets": [ - { - "datasource": { - "type": "postgres", - "uid": "P7B13B9DF907EC40C" - }, - "editorMode": "code", - "format": "time_series", - "group": [], - "metricColumn": "none", - "rawQuery": true, - "rawSql": "SELECT\r\n $__timeGroupAlias(time,$__interval),\r\n avg(energy_cashflow) AS \"Cashflow:\",\r\n -avg(energy_generation_costs) AS \"Production costs:\",\r\n avg(energy_cashflow + energy_generation_costs) AS \"Profit\",\r\n avg(financial_support_cashflow) as \"Support Cashflow\",\r\n unit\r\nFROM unit_dispatch\r\nWHERE\r\n $__timeFilter(time) AND\r\n simulation = '$simulation' AND\r\n unit in ($Demand_Units)\r\nGROUP BY 1, unit\r\nORDER BY 1", - "refId": "A", - "select": [ - [ - { - "params": [ - "power" - ], - "type": "column" - } - ] - ], - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - }, - "table": "market_dispatch", - "timeColumn": "datetime", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "title": "Energy Related Cashflow", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 114 - }, - "id": 44, - "panels": [], - "title": "Storage units data $Storage_Units", - "type": "row" - }, - { - "datasource": { - "type": "postgres", - "uid": "P7B13B9DF907EC40C" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "always", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "megwatt" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 18, - "x": 0, - "y": 115 - }, - "id": 65, - "options": { - "legend": { - "calcs": [ - "min", - "max", - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "maxHeight": 600, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.4.0", - "targets": [ - { - "datasource": { - "type": "postgres", - "uid": "P7B13B9DF907EC40C" - }, - "editorMode": "code", - "format": "time_series", - "group": [], - "metricColumn": "none", - "rawQuery": true, - "rawSql": "SELECT\n $__timeGroupAlias(datetime,$__interval),\n power AS \"Market dispatch\",\n unit_id,\n market_id\nFROM market_dispatch\nWHERE\n $__timeFilter(datetime) AND\n simulation = '$simulation' AND\n unit_id in ($Storage_Units)\nGROUP BY 1, unit_id, power, market_id\nORDER BY 1", - "refId": "A", - "select": [ - [ - { - "params": [ - "volume" - ], - "type": "column" - } - ] - ], - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - }, - "table": "demand_meta", - "timeColumn": "\"Timestamp\"", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - }, - { - "datasource": { - "type": "postgres", - "uid": "P7B13B9DF907EC40C" - }, - "editorMode": "code", - "format": "time_series", - "group": [], - "hide": false, - "metricColumn": "none", - "rawQuery": true, - "rawSql": "SELECT\n $__timeGroupAlias(time,$__interval),\n power AS \"Actual dispatch\",\n unit\nFROM unit_dispatch\nWHERE\n $__timeFilter(time) AND\n simulation = '$simulation' AND\n unit in ($Storage_Units)\nGROUP BY 1, unit, power\nORDER BY 1", - "refId": "B", - "select": [ - [ - { - "params": [ - "power" - ], - "type": "column" - } - ] - ], - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - }, - "table": "market_dispatch", - "timeColumn": "datetime", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "title": "Storage Dispatch", - "type": "timeseries" - }, - { - "datasource": { - "type": "postgres", - "uid": "P7B13B9DF907EC40C" - }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 6, - "x": 18, - "y": 115 - }, - "id": 47, - "options": { - "afterRender": "", - "content": "### General Information\n\nName: {{index}}
\nTechnology: {{technology}}
\n\n### Technical Specifications\nEmissions: {{emission_factor}} t/MWh
\nMaximum Power Discharge: {{max_power_discharge}} MW
\nMinimum Power Discharge: {{min_power_discharge}} MW
\nEfficiency Discharge: {{efficiency_discharge}}
\n\nMaximum Power Charge: {{max_power_charge}} MW
\nMinimum Power Charge: {{min_power_charge}} MW
\nEfficiency Charge: {{efficiency_charge}}
\n\n##### Unit Operator: {{unit_operator}}\n ", - "contentPartials": [], - "defaultContent": "The query didn't return any results.", - "editor": { - "format": "auto", - "height": 200, - "language": "markdown" - }, - "editors": [], - "externalScripts": [], - "externalStyles": [], - "helpers": "", - "renderMode": "everyRow", - "styles": "", - "wrap": true - }, - "pluginVersion": "5.6.0", - "targets": [ - { - "datasource": { - "type": "postgres", - "uid": "P7B13B9DF907EC40C" - }, - "format": "table", - "group": [], - "metricColumn": "none", - "rawQuery": true, - "rawSql": "SELECT * FROM storage_meta\nWHERE index in ($Storage_Units) and simulation = '$simulation'\n", - "refId": "A", - "select": [ - [ - { - "params": [ - "volume" - ], - "type": "column" - } - ] - ], - "table": "demand_meta", - "timeColumn": "\"Timestamp\"", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "title": "Chosen Unit Specifications", - "type": "marcusolsson-dynamictext-panel" - }, - { - "datasource": { - "default": true, - "type": "grafana-postgresql-datasource", - "uid": "P7B13B9DF907EC40C" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "mwatth" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 24, - "x": 0, - "y": 124 - }, - "id": 80, - "options": { - "legend": { - "calcs": [ - "min", - "max", - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "maxHeight": 600, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.4.0", - "targets": [ - { - "datasource": { - "type": "postgres", - "uid": "P7B13B9DF907EC40C" - }, - "editorMode": "code", - "format": "time_series", - "group": [], - "metricColumn": "none", - "rawQuery": true, - "rawSql": "SELECT\r\n $__timeGroupAlias(time,$__interval),\r\n SOC AS \"State of Charge\",\r\n unit\r\nFROM unit_dispatch\r\nWHERE\r\n $__timeFilter(time) AND\r\n simulation = '$simulation' AND\r\n unit in ($Storage_Units)\r\nGROUP BY 1, unit, SOC\r\nORDER BY 1", - "refId": "A", - "select": [ - [ - { - "params": [ - "power" - ], - "type": "column" - } - ] - ], - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - }, - "table": "market_dispatch", - "timeColumn": "datetime", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "title": "State of Charge", - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-postgresql-datasource", - "uid": "P7B13B9DF907EC40C" - }, - "description": "Bid prices of accepted bids per unit in the chosen market", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "currencyEUR" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "price .*" - }, - "properties": [ - { - "id": "unit", - "value": "€/MW" - } - ] - } - ] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 133 - }, - "id": 91, - "options": { - "legend": { - "calcs": [ - "min", - "max", - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "maxHeight": 600, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.4.0", - "targets": [ - { - "datasource": { - "type": "postgres", - "uid": "P7B13B9DF907EC40C" - }, - "editorMode": "code", - "format": "time_series", - "group": [ - { - "params": [ - "$__interval", - "none" - ], - "type": "time" - }, - { - "params": [ - "unit_id" - ], - "type": "column" - } - ], - "metricColumn": "none", - "rawQuery": true, - "rawSql": "SELECT\r\n $__timeGroupAlias(start_time,$__interval),\r\n CASE WHEN avg(accepted_volume) >= 0 THEN avg(accepted_price::float) END AS \"Accepted price Discharge\",\r\n CASE WHEN avg(accepted_volume) < 0 THEN avg(accepted_price::float) END AS \"Accepted price Charge\",\r\n CASE WHEN avg(volume) >= 0 THEN avg(price) END AS \"Bid price Discharge\",\r\n CASE WHEN avg(volume) < 0 THEN avg(price) END AS \"Bid price Charge\",\r\n concat(unit_id, ' - ', market_id) as \"unit_id\"\r\nFROM market_orders\r\nWHERE\r\n $__timeFilter(start_time) AND\r\n unit_id in ($Storage_Units) AND\r\n simulation = '$simulation'\r\nGROUP BY 1, unit_id, market_id\r\nORDER BY 1\r\n", - "refId": "A", - "select": [ - [ - { - "params": [ - "original_price" - ], - "type": "column" - }, - { - "params": [ - "avg" - ], - "type": "aggregate" - }, - { - "params": [ - "price" - ], - "type": "alias" - } - ], - [ - { - "params": [ - "unit_id" - ], - "type": "column" - }, - { - "params": [ - "unit_id" - ], - "type": "alias" - } - ] - ], - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - }, - "table": "market_orders", - "timeColumn": "start_time", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - }, - { - "datatype": "text", - "name": "", - "params": [ - "market_id", - "=", - "'$market'" - ], - "type": "expression" - }, - { - "datatype": "text", - "name": "", - "params": [ - "simulation", - "=", - "'$simulation'" - ], - "type": "expression" - } - ] - } - ], - "title": "Bid Prices", - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-postgresql-datasource", - "uid": "P7B13B9DF907EC40C" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "megwatt" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "volume_acceptance_ratio" - }, - "properties": [ - { - "id": "custom.axisPlacement", - "value": "right" - }, - { - "id": "min", - "value": 0 - }, - { - "id": "max", - "value": 1 - }, - { - "id": "custom.axisLabel", - "value": "Relative acceptance of bid volume" - }, - { - "id": "unit" - } - ] - } - ] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 133 - }, - "id": 92, - "options": { - "legend": { - "calcs": [ - "min", - "max", - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "maxHeight": 600, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.4.0", - "targets": [ - { - "datasource": { - "type": "postgres", - "uid": "P7B13B9DF907EC40C" - }, - "editorMode": "code", - "format": "time_series", - "group": [], - "metricColumn": "none", - "rawQuery": true, - "rawSql": "SELECT\r\n $__timeGroupAlias(start_time,$__interval),\r\n avg(accepted_volume) AS \"Accepted volume:\",\r\n CASE WHEN avg(volume) < 0 THEN avg(volume) END AS \"Bid volume charge:\",\r\n CASE WHEN avg(volume) >= 0 THEN avg(volume) END AS \"Bid volume discharge:\",\r\n concat(unit_id, ' - ', market_id) as \"unit_id\"\r\nFROM market_orders\r\nWHERE\r\n $__timeFilter(start_time) AND\r\n unit_id in ($Storage_Units) AND\r\n simulation = '$simulation'\r\nGROUP BY 1, unit_id, market_id\r\nORDER BY 1\r\n", - "refId": "A", - "select": [ - [ - { - "params": [ - "power" - ], - "type": "column" - } - ] - ], - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - }, - "table": "market_dispatch", - "timeColumn": "datetime", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - }, - { - "datasource": { - "type": "postgres", - "uid": "P7B13B9DF907EC40C" - }, - "format": "time_series", - "group": [], - "hide": true, - "metricColumn": "none", - "rawQuery": true, - "rawSql": "SELECT\r\n $__timeGroupAlias(start_time,$__interval),\r\n avg(volume) AS \"Bid volume:\",\r\n unit_id\r\nFROM market_orders\r\nWHERE\r\n $__timeFilter(start_time) AND\r\n unit_id in ($Storage_Units) AND\r\n simulation = '$simulation'\r\nGROUP BY 1, unit_id\r\nORDER BY 1", - "refId": "B", - "select": [ - [ - { - "params": [ - "power" - ], - "type": "column" - } - ] - ], - "table": "market_dispatch", - "timeColumn": "datetime", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "title": "Bid volume", - "transformations": [ - { - "id": "calculateField", - "options": { - "alias": "volume_acceptance_ratio", - "binary": { - "left": "accepted_volume Unit 1", - "operator": "/", - "reducer": "sum", - "right": "bid_volume Unit 1" - }, - "mode": "binary", - "reduce": { - "reducer": "sum" - } - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "postgres", - "uid": "P7B13B9DF907EC40C" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "currencyEUR" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "Profit .*" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "yellow", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "Production costs .*" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "purple", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 9, - "w": 24, - "x": 0, - "y": 142 - }, - "id": 97, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "sum" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "11.4.0", - "targets": [ - { - "datasource": { - "type": "postgres", - "uid": "P7B13B9DF907EC40C" - }, - "editorMode": "code", - "format": "time_series", - "group": [], - "metricColumn": "none", - "rawQuery": true, - "rawSql": "SELECT\r\n $__timeGroupAlias(time,$__interval),\r\n energy_cashflow AS \"Cashflow\",\r\n energy_generation_costs AS \"Production costs\",\r\n energy_cashflow - energy_generation_costs AS \"Profit\",\r\n unit\r\nFROM unit_dispatch\r\nWHERE\r\n $__timeFilter(time) AND\r\n simulation = '$simulation' AND\r\n unit in ($Storage_Units)\r\nGROUP BY 1, unit, energy_cashflow, energy_generation_costs\r\nORDER BY 1", - "refId": "A", - "select": [ - [ - { - "params": [ - "power" - ], - "type": "column" - } - ] - ], - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - }, - "table": "market_dispatch", - "timeColumn": "datetime", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "title": "Financial Overview ${__from:date:YYYY-MM-DD} until ${__to:date:YYYY-MM-DD}", - "type": "stat" - }, - { - "datasource": { - "type": "postgres", - "uid": "P7B13B9DF907EC40C" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "currencyEUR" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "Profit .*" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "yellow", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "Production costs .*" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "purple", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 9, - "w": 24, - "x": 0, - "y": 151 - }, - "id": 98, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "maxHeight": 600, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.4.0", - "targets": [ - { - "datasource": { - "type": "postgres", - "uid": "P7B13B9DF907EC40C" - }, - "editorMode": "code", - "format": "time_series", - "group": [], - "metricColumn": "none", - "rawQuery": true, - "rawSql": "SELECT\r\n $__timeGroupAlias(time,$__interval),\r\n energy_cashflow AS \"Cashflow\",\r\n energy_generation_costs AS \"Production costs\",\r\n energy_cashflow - energy_generation_costs AS \"Profit\",\r\n unit\r\nFROM unit_dispatch\r\nWHERE\r\n $__timeFilter(time) AND\r\n simulation = '$simulation' AND\r\n unit in ($Storage_Units)\r\nGROUP BY 1, unit, energy_cashflow, energy_generation_costs\r\nORDER BY 1", - "refId": "A", - "select": [ - [ - { - "params": [ - "power" - ], - "type": "column" - } - ] - ], - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - }, - "table": "market_dispatch", - "timeColumn": "datetime", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "title": "Financial Overview ${__from:date:YYYY-MM-DD} until ${__to:date:YYYY-MM-DD}", - "type": "timeseries" - } - ], - "preload": false, - "refresh": "", - "schemaVersion": 40, - "tags": [], - "templating": { - "list": [ - { - "current": { - "text": "world_script_policy", - "value": "world_script_policy" - }, - "datasource": { - "type": "postgres", - "uid": "P7B13B9DF907EC40C" - }, - "definition": "SELECT simulation\nFROM market_meta", - "description": "Can choose which simulation we want to show ", - "includeAll": false, - "name": "simulation", - "options": [], - "query": "SELECT simulation\nFROM market_meta", - "refresh": 2, - "regex": "", - "sort": 1, - "type": "query" - }, - { - "current": { - "text": "EOM", - "value": "EOM" - }, - "datasource": { - "type": "postgres", - "uid": "P7B13B9DF907EC40C" - }, - "definition": "SELECT \n market_id\nFROM market_meta\nwhere simulation='$simulation'\ngroup by market_id ;", - "description": "Choose for which market the data is displayed", - "includeAll": false, - "name": "market", - "options": [], - "query": "SELECT \n market_id\nFROM market_meta\nwhere simulation='$simulation'\ngroup by market_id ;", - "refresh": 2, - "regex": "", - "sort": 1, - "type": "query" - }, - { - "current": { - "text": [ - "nuclear1" - ], - "value": [ - "nuclear1" - ] - }, - "datasource": { - "type": "postgres", - "uid": "P7B13B9DF907EC40C" - }, - "definition": "SELECT index\nFROM power_plant_meta\nwhere simulation = '$simulation';", - "description": "Can choose which units we want to display ", - "includeAll": false, - "multi": true, - "name": "Gen_Units", - "options": [], - "query": "SELECT index\nFROM power_plant_meta\nwhere simulation = '$simulation';", - "refresh": 2, - "regex": "", - "sort": 1, - "type": "query" - }, - { - "current": { - "text": [ - "demand1" - ], - "value": [ - "demand1" - ] - }, - "datasource": { - "type": "postgres", - "uid": "P7B13B9DF907EC40C" - }, - "definition": "SELECT index\nFROM demand_meta\nwhere simulation = '$simulation';", - "description": "Can choose which units we want to display ", - "includeAll": false, - "multi": true, - "name": "Demand_Units", - "options": [], - "query": "SELECT index\nFROM demand_meta\nwhere simulation = '$simulation';", - "refresh": 2, - "regex": "", - "sort": 1, - "type": "query" - }, - { - "current": { - "text": [], - "value": [] - }, - "datasource": { - "type": "postgres", - "uid": "P7B13B9DF907EC40C" - }, - "definition": "SELECT index\nFROM storage_meta\nwhere simulation = '$simulation';", - "description": "Can choose which storage units we want to display ", - "includeAll": false, - "multi": true, - "name": "Storage_Units", - "options": [], - "query": "SELECT index\nFROM storage_meta\nwhere simulation = '$simulation';", - "refresh": 2, - "regex": "", - "sort": 1, - "type": "query" - }, - { - "current": { - "text": "2019-01-01 03:00:00", - "value": "2019-01-01 03:00:00" - }, - "definition": "select min(start_time)::text from market_orders where market_id = '$market' AND\n simulation = '$simulation' and $__timeFilter(start_time)", - "description": "The minimum time we have in the time range.\nUsed for the merit order plot title", - "hide": 2, - "name": "min_time", - "options": [], - "query": "select min(start_time)::text from market_orders where market_id = '$market' AND\n simulation = '$simulation' and $__timeFilter(start_time)", - "refresh": 1, - "regex": "", - "type": "query" - } - ] - }, - "time": { - "from": "2018-12-27T09:09:17.102Z", - "to": "2019-03-04T20:54:17.918Z" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "1m", - "5m", - "15m", - "30m", - "1h" - ] - }, - "timezone": "utc", - "title": "ASSUME: Main Support Overview", - "uid": "ceaqgovvukef4b", - "version": 15, - "weekStart": "" - } diff --git a/docker_configs/dashboard-definitions/ASSUME-Support.json.license b/docker_configs/dashboard-definitions/ASSUME-Support.json.license deleted file mode 100644 index a6ae06366..000000000 --- a/docker_configs/dashboard-definitions/ASSUME-Support.json.license +++ /dev/null @@ -1,3 +0,0 @@ -SPDX-FileCopyrightText: ASSUME Developers - -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/docker_configs/dashboard-definitions/ASSUME.json b/docker_configs/dashboard-definitions/ASSUME.json index b6d95a538..7d0a054fc 100644 --- a/docker_configs/dashboard-definitions/ASSUME.json +++ b/docker_configs/dashboard-definitions/ASSUME.json @@ -757,7 +757,7 @@ ], "metricColumn": "none", "rawQuery": true, - "rawSql": "select time, sum(volume) as \"volume\", tech FROM \n(\nSELECT\n $__timeGroupAlias(start_time,$__interval),\n sum(accepted_volume) AS \"volume\", unit_id, pm.technology as tech\nFROM market_orders mo\njoin power_plant_meta pm on pm.index=mo.unit_id and pm.simulation=mo.simulation\nWHERE\n\n $__timeFilter(start_time) AND\n market_id = '$market' AND\n mo.simulation = '$simulation'\n\nGROUP BY 1, unit_id, bid_id, pm.technology\nORDER BY 1\n\n) a\ngroup by 1, tech\nORDER BY 1, tech desc", + "rawSql": "SELECT\n time,\n SUM(volume) AS volume,\n tech\nFROM (\n SELECT\n $__timeGroupAlias(mo.start_time, $__interval),\n SUM(mo.accepted_volume) AS volume,\n mo.unit_id AS unit_id,\n pm.technology AS tech\n FROM market_orders mo\n JOIN power_plant_meta pm\n ON pm.unit_id = mo.unit_id\n AND pm.simulation = mo.simulation\n WHERE\n $__timeFilter(mo.start_time)\n AND mo.market_id = '$market'\n AND mo.simulation = '$simulation'\n GROUP BY 1, mo.unit_id, mo.bid_id, pm.technology\n) AS a\nGROUP BY 1, tech\nORDER BY 1, tech DESC;\n", "refId": "A", "select": [ [ @@ -1932,7 +1932,7 @@ "id": 26, "options": { "afterRender": "", - "content": "### General Information\n\nName: {{index}}
\nTechnology: {{technology}}
\n\n### Technical Specifications\nEmissions: {{emission_factor}} t/MWh
\nMaximum Power: {{max_power}} MW
\nMinimum Power: {{min_power}} MW
\nEfficiency: {{efficiency}}
\n\n##### Unit Operator: {{unit_operator}}\n ", + "content": "### General Information\n\nName: {{unit_id}}
\nTechnology: {{technology}}
\n\n### Technical Specifications\nEmissions: {{emission_factor}} t/MWh
\nMaximum Power: {{max_power}} MW
\nMinimum Power: {{min_power}} MW
\nEfficiency: {{efficiency}}
\n\n##### Unit Operator: {{unit_operator}}\n ", "contentPartials": [], "defaultContent": "The query didn't return any results.", "editor": { @@ -1948,7 +1948,7 @@ "styles": "", "wrap": true }, - "pluginVersion": "5.6.0", + "pluginVersion": "5.7.0", "targets": [ { "datasource": { @@ -1960,7 +1960,7 @@ "group": [], "metricColumn": "none", "rawQuery": true, - "rawSql": "SELECT * FROM power_plant_meta\nWHERE index in ($Gen_Units) and simulation = '$simulation'\n", + "rawSql": "SELECT * FROM power_plant_meta\nWHERE unit_id in ($Gen_Units) and simulation = '$simulation'\n", "refId": "A", "select": [ [ @@ -2627,8 +2627,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2808,8 +2807,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2963,8 +2961,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2984,7 +2981,7 @@ "id": 37, "options": { "afterRender": "", - "content": "### General Information\n\nName: {{index}}
\nTechnology: {{technology}}
\n\n### Technical Specifications\nEmissions: {{emission_factor}} t/MWh
\nMaximum Power: {{max_power}} MW
\nMinimum Power: {{min_power}} MW
\nEfficiency: {{efficiency}}
\n\n##### Unit Operator: {{unit_operator}}\n ", + "content": "### General Information\n\nName: {{unit_id}}
\nTechnology: {{technology}}
\n\n### Technical Specifications\nEmissions: {{emission_factor}} t/MWh
\nMaximum Power: {{max_power}} MW
\nMinimum Power: {{min_power}} MW
\nEfficiency: {{efficiency}}
\n\n##### Unit Operator: {{unit_operator}}\n ", "contentPartials": [], "defaultContent": "The query didn't return any results.", "editor": { @@ -3012,7 +3009,7 @@ "group": [], "metricColumn": "none", "rawQuery": true, - "rawSql": "SELECT * FROM demand_meta\nWHERE index in ($Demand_Units) and simulation = '$simulation'\n", + "rawSql": "SELECT * FROM demand_meta\nWHERE unit_id in ($Demand_Units) and simulation = '$simulation'\n", "refId": "A", "select": [ [ @@ -3105,8 +3102,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3268,8 +3264,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3451,8 +3446,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3587,8 +3581,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3696,8 +3689,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3838,8 +3830,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4009,8 +4000,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4164,8 +4154,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4185,7 +4174,7 @@ "id": 47, "options": { "afterRender": "", - "content": "### General Information\n\nName: {{index}}
\nTechnology: {{technology}}
\nUnit Operator: {{unit_operator}}\n\n### Technical Specifications\nMaximum Capacity: {{max_soc}} MWh
\nMinimum Capacity: {{min_soc}} MWh
\n\nMaximum Power Discharge: {{max_power_discharge}} MW
\nMinimum Power Discharge: {{min_power_discharge}} MW
\nEfficiency Discharge: {{efficiency_discharge}}
\n\nMaximum Power Charge: {{max_power_charge}} MW
\nMinimum Power Charge: {{min_power_charge}} MW
\nEfficiency Charge: {{efficiency_charge}} %
", + "content": "### General Information\n\nName: {{unit_id}}
\nTechnology: {{technology}}
\nUnit Operator: {{unit_operator}}\n\n### Technical Specifications\nMaximum Capacity: {{max_soc}} MWh
\nMinimum Capacity: {{min_soc}} MWh
\n\nMaximum Power Discharge: {{max_power_discharge}} MW
\nMinimum Power Discharge: {{min_power_discharge}} MW
\nEfficiency Discharge: {{efficiency_discharge}}
\n\nMaximum Power Charge: {{max_power_charge}} MW
\nMinimum Power Charge: {{min_power_charge}} MW
\nEfficiency Charge: {{efficiency_charge}} %
", "contentPartials": [], "defaultContent": "The query didn't return any results.", "editor": { @@ -4212,7 +4201,7 @@ "group": [], "metricColumn": "none", "rawQuery": true, - "rawSql": "SELECT * FROM storage_meta\nWHERE index in ($Storage_Units) and simulation = '$simulation'\n", + "rawSql": "SELECT * FROM storage_meta\nWHERE unit_id in ($Storage_Units) and simulation = '$simulation'\n", "refId": "A", "select": [ [ @@ -4288,8 +4277,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4430,8 +4418,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4645,8 +4632,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4833,8 +4819,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -5006,8 +4991,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -5186,13 +5170,13 @@ "type": "postgres", "uid": "P7B13B9DF907EC40C" }, - "definition": "SELECT index\nFROM power_plant_meta\nwhere simulation = '$simulation';", + "definition": "SELECT unit_id\nFROM power_plant_meta\nwhere simulation = '$simulation';", "description": "Can choose which units we want to display ", "includeAll": false, "multi": true, "name": "Gen_Units", "options": [], - "query": "SELECT index\nFROM power_plant_meta\nwhere simulation = '$simulation';", + "query": "SELECT unit_id\nFROM power_plant_meta\nwhere simulation = '$simulation';", "refresh": 2, "regex": "", "sort": 1, @@ -5201,23 +5185,23 @@ { "current": { "text": [ - "demand_CRM_neg" + "demand_EOM" ], "value": [ - "demand_CRM_neg" + "demand_EOM" ] }, "datasource": { "type": "postgres", "uid": "P7B13B9DF907EC40C" }, - "definition": "SELECT index\nFROM demand_meta\nwhere simulation = '$simulation';", + "definition": "SELECT unit_id\nFROM demand_meta\nwhere simulation = '$simulation';", "description": "Can choose which units we want to display ", "includeAll": false, "multi": true, "name": "Demand_Units", "options": [], - "query": "SELECT index\nFROM demand_meta\nwhere simulation = '$simulation';", + "query": "SELECT unit_id\nFROM demand_meta\nwhere simulation = '$simulation';", "refresh": 2, "regex": "", "sort": 1, @@ -5236,13 +5220,13 @@ "type": "postgres", "uid": "P7B13B9DF907EC40C" }, - "definition": "SELECT index\nFROM storage_meta\nwhere simulation = '$simulation';", + "definition": "SELECT unit_id\nFROM storage_meta\nwhere simulation = '$simulation';", "description": "Can choose which storage units we want to display ", "includeAll": false, "multi": true, "name": "Storage_Units", "options": [], - "query": "SELECT index\nFROM storage_meta\nwhere simulation = '$simulation';", + "query": "SELECT unit_id\nFROM storage_meta\nwhere simulation = '$simulation';", "refresh": 2, "regex": "", "sort": 1, @@ -5250,8 +5234,8 @@ }, { "current": { - "text": "2019-01-01 10:00:00", - "value": "2019-01-01 10:00:00" + "text": "2019-01-01 01:00:00", + "value": "2019-01-01 01:00:00" }, "definition": "select min(start_time)::text from market_orders where market_id = '$market' AND\n simulation = '$simulation' and $__timeFilter(start_time)", "description": "The minimum time we have in the time range.\nUsed for the merit order plot title", diff --git a/docker_configs/db-init/assume_schema.sql b/docker_configs/db-init/assume_schema.sql index a09f1a471..28a604e8e 100644 --- a/docker_configs/db-init/assume_schema.sql +++ b/docker_configs/db-init/assume_schema.sql @@ -23,11 +23,13 @@ CREATE TABLE IF NOT EXISTS market_meta ( max_price REAL, min_price REAL, node TEXT, - PRIMARY KEY (simulation, market_id, time) + PRIMARY KEY (simulation, market_id, node, time) ) PARTITION BY LIST (simulation); CREATE INDEX ON market_meta (simulation, time); +CREATE INDEX idx_market_meta_sim_market_prod_start + ON market_meta (simulation, market_id, product_start); ---------------------------- -- 2) market_dispatch (partitioned by simulation) @@ -43,6 +45,8 @@ CREATE TABLE IF NOT EXISTS market_dispatch ( PARTITION BY LIST (simulation); CREATE INDEX ON market_dispatch (simulation, datetime); +CREATE INDEX idx_market_dispatch_sim_unit_datetime + ON market_dispatch (simulation, unit_id, datetime); ---------------------------- -- 3) market_orders (partitioned by simulation) @@ -60,11 +64,16 @@ CREATE TABLE IF NOT EXISTS market_orders ( unit_id TEXT, accepted_price REAL, accepted_volume REAL, + parent_bid_id TEXT, + min_acceptance_ratio REAL, PRIMARY KEY (simulation, market_id, start_time, bid_id) ) PARTITION BY LIST (simulation); -CREATE INDEX ON market_orders (simulation, start_time); +CREATE INDEX ON market_orders (simulation, market_id, start_time); +CREATE INDEX ON market_orders (simulation, unit_id); +CREATE INDEX idx_market_orders_sim_unit_start + ON market_orders (simulation, unit_id, start_time); ---------------------------- -- 4) unit_dispatch (partitioned by simulation) @@ -79,11 +88,13 @@ CREATE TABLE IF NOT EXISTS unit_dispatch ( energy_generation_costs REAL, energy_cashflow REAL, total_costs REAL, - PRIMARY KEY (simulation, unit, time) + PRIMARY KEY (simulation, time, unit) ) PARTITION BY LIST (simulation); CREATE INDEX ON unit_dispatch (simulation, time); +CREATE INDEX idx_unit_dispatch_sim_unit_time + ON unit_dispatch (simulation, unit, time); ---------------------------- -- 5.1) power_plant_meta (static) @@ -101,6 +112,9 @@ CREATE TABLE IF NOT EXISTS power_plant_meta ( PRIMARY KEY (simulation, unit_id) ); +CREATE INDEX idx_power_plant_meta_sim_tech + ON power_plant_meta (simulation, technology); + ---------------------------- -- 5.2) storage_meta (static) ---------------------------- @@ -166,7 +180,7 @@ CREATE TABLE IF NOT EXISTS rl_params ( total_grad_norm REAL, max_grad_norm REAL, learning_rate REAL, - PRIMARY KEY (simulation, episode, evaluation_mode, datetime) + PRIMARY KEY (simulation, episode, evaluation_mode, unit, datetime) ) PARTITION BY LIST (simulation); From bba502f6053b30ed8c0a4e5174f45a9112a23f86 Mon Sep 17 00:00:00 2001 From: Nick Harder Date: Tue, 6 May 2025 15:54:37 +0200 Subject: [PATCH 10/11] - fix upinserts --- assume/common/outputs.py | 124 +++++++++++++++++---------------------- 1 file changed, 55 insertions(+), 69 deletions(-) diff --git a/assume/common/outputs.py b/assume/common/outputs.py index f70a6c1da..dd2a0d0c2 100644 --- a/assume/common/outputs.py +++ b/assume/common/outputs.py @@ -17,9 +17,7 @@ from mango import Role from pandas.api.types import is_bool_dtype, is_numeric_dtype from psycopg2.errors import UndefinedColumn -from sqlalchemy import MetaData, Table, create_engine, inspect, text -from sqlalchemy.dialects.postgresql import insert as pg_insert -from sqlalchemy.dialects.sqlite import insert as sqlite_insert +from sqlalchemy import create_engine, inspect, text from sqlalchemy.exc import ( DataError, NoSuchTableError, @@ -729,13 +727,10 @@ def _copy_df_to_db(self, table: str, df: pd.DataFrame): # 2) Dispatch based on dialect and table if self.db.dialect.name == "sqlite": - if table == "unit_dispatch": - return self._copy_df_to_db_sqlite_upsert(table, df) - else: - return self._copy_df_to_db_sqlite_append(table, df) + return self._copy_df_to_db_sqlite_append(table, df) else: if table == "unit_dispatch": - return self._copy_df_to_db_postgres_upsert(table, df) + return self._copy_df_to_unit_dispatch(table, df) else: return self._copy_df_to_db_postgres_copy(table, df) @@ -761,36 +756,6 @@ def _copy_df_to_db_sqlite_append(self, table: str, df: pd.DataFrame): except Exception as e: logger.error("SQLite append failed for %s: %s", table, e) - def _copy_df_to_db_sqlite_upsert(self, table: str, df: pd.DataFrame): - """ - SQLite UPSERT for unit_dispatch: INSERT...ON CONFLICT DO UPDATE. - """ - # 1) Ensure table exists - if not self._inspector.has_table(table): - self._ensure_table(table, df) - - try: - meta = MetaData() - tbl = Table(table, meta, autoload_with=self.db) - records = df.to_dict(orient="records") - stmt = sqlite_insert(tbl).values(records) - - pk_cols = [c.name for c in tbl.primary_key.columns] - update_cols = [c for c in df.columns if c not in pk_cols] - - stmt = stmt.on_conflict_do_update( - index_elements=pk_cols, - set_={col: getattr(stmt.excluded, col) for col in update_cols}, - ) - with self.db.begin() as conn: - conn.execute(stmt) - - except (ProgrammingError, OperationalError, DataError, UndefinedColumn): - self._check_columns(table, df) - return self._copy_df_to_db_sqlite_upsert(table, df) - except Exception as e: - logger.error("SQLite upsert failed for %s: %s", table, e) - def _copy_df_to_db_postgres_copy(self, table: str, df: pd.DataFrame): """ Postgres COPY for all tables except unit_dispatch. @@ -825,35 +790,46 @@ def _copy_df_to_db_postgres_copy(self, table: str, df: pd.DataFrame): except Exception: pass - def _copy_df_to_db_postgres_upsert(self, table: str, df: pd.DataFrame): - """ - Postgres UPSERT for unit_dispatch: INSERT...ON CONFLICT DO UPDATE. - """ - # 1) Ensure table exists - if not self._inspector.has_table(table): - self._ensure_table(table, df) + def _copy_df_to_unit_dispatch(self, table: str, df: pd.DataFrame): + # drop duplicates in the staging DataFrame + df = df.sort_values(["simulation", "time", "unit"]).drop_duplicates( + subset=["simulation", "time", "unit"], keep="last" + ) - try: - meta = MetaData() - tbl = Table(table, meta, autoload_with=self.db) - records = df.to_dict(orient="records") + buf = StringIO() + df.to_csv(buf, index=False, header=True) + buf.seek(0) - stmt = pg_insert(tbl).values(records) - pk_cols = [c.name for c in tbl.primary_key.columns] - update_cols = [c for c in df.columns if c not in pk_cols] + cols = df.columns.tolist() + col_list = ",".join(f'"{c}"' for c in cols) - stmt = stmt.on_conflict_do_update( - index_elements=pk_cols, - set_={col: getattr(stmt.excluded, col) for col in update_cols}, + raw = self.db.raw_connection() + cur = raw.cursor() + try: + cur.execute(f""" + CREATE TEMP TABLE tmp_{table} ( + LIKE "{table}" INCLUDING ALL + ) ON COMMIT DROP; + """) + cur.copy_expert( + f"COPY tmp_{table} ({col_list}) FROM STDIN WITH (FORMAT CSV, HEADER TRUE)", + buf, ) - with self.db.begin() as conn: - conn.execute(stmt) - except (ProgrammingError, OperationalError, DataError, UndefinedColumn): - self._check_columns(table, df) - return self._copy_df_to_db_postgres_upsert(table, df) - except Exception as e: - logger.error("Postgres upsert failed for %s: %s", table, e) + pk_cols = ["simulation", "time", "unit"] + update_cols = [c for c in cols if c not in pk_cols] + update_clause = ", ".join(f'"{c}" = EXCLUDED."{c}"' for c in update_cols) + + cur.execute(f""" + INSERT INTO "{table}" ({col_list}) + SELECT {col_list} FROM tmp_{table} + ON CONFLICT (simulation, time, unit) + DO UPDATE SET {update_clause}; + """) + raw.commit() + + finally: + cur.close() def _create_partitions(self, simulation_id: str): """Create one child‐partition per parent table for this simulation_id.""" @@ -965,28 +941,38 @@ def _purge_simulation_data_postgresql(self, simulation_id: str): """ TimescaleDB/Postgres cleanup: 1) Drop any simulation-specific partitions. - 2) Delete all rows in the static tables for that simulation. + 2) Delete all rows in the static tables for that simulation, + but only if the table actually exists. """ + inspector = self._inspector + # 1) drop per-simulation partitions for table in PARTITIONED_TABLES: # only delete rl_params during the first episode of learning if table == "rl_params" and not (self.learning_mode and self.episode == 1): continue + partition_name = f"{table}_{simulation_id}" + sql_drop = text(f'DROP TABLE IF EXISTS "{partition_name}";') with self.db.begin() as conn: - conn.execute(text(f"DROP TABLE IF EXISTS {table}_{simulation_id};")) + conn.execute(sql_drop) + logger.debug("Dropped partition %s if it existed", partition_name) - # 2) delete from the static tables + # 2) delete from the static tables, skipping any that don’t exist static_tables = [t for t in ALL_SIM_TABLES if t not in PARTITIONED_TABLES] for table in static_tables: + # only delete rl_meta during the first episode of learning if table == "rl_meta" and not (self.learning_mode and self.episode == 1): continue + if not inspector.has_table(table): + logger.debug("Skipping purge: table %s does not exist", table) + continue + with self.db.begin() as conn: - conn.execute( - text(f"DELETE FROM {table} WHERE simulation = :sim;"), - {"sim": simulation_id}, - ) + sql_delete = text(f'DELETE FROM "{table}" WHERE simulation = :sim;') + res = conn.execute(sql_delete, {"sim": simulation_id}) + logger.debug("Deleted %s rows from %s", res.rowcount, table) class DatabaseMaintenance: From b23cff55fee4729f78dc7ba7d3c89e26a3d4d96f Mon Sep 17 00:00:00 2001 From: Nick Harder Date: Tue, 6 May 2025 16:33:03 +0200 Subject: [PATCH 11/11] - make the code more sleak --- assume/common/outputs.py | 336 ++++++++++++++++----------------------- 1 file changed, 139 insertions(+), 197 deletions(-) diff --git a/assume/common/outputs.py b/assume/common/outputs.py index dd2a0d0c2..1ec67b260 100644 --- a/assume/common/outputs.py +++ b/assume/common/outputs.py @@ -2,6 +2,7 @@ # # SPDX-License-Identifier: AGPL-3.0-or-later +import functools import logging import shutil from collections import defaultdict @@ -55,6 +56,29 @@ ] +# --- Decorator to retry on missing tables or columns --- +def retry_on_schema_errors(func): + @functools.wraps(func) + def wrapper(self, table: str, df: pd.DataFrame): + try: + return func(self, table, df) + except NoSuchTableError: + self._ensure_table(table, df) + return func(self, table, df) + except (ProgrammingError, OperationalError, DataError, UndefinedColumn): + self._check_columns(table, df) + return func(self, table, df) + except Exception as e: + logger.error("%s failed for %s: %s", func.__name__, table, e) + + return wrapper + + +# --- Helper for index normalization --- +def _prepare_df(df: pd.DataFrame) -> pd.DataFrame: + return df.reset_index() if df.index.name else df + + class OutputDef(TypedDict): name: str value: str @@ -710,253 +734,170 @@ def get_sum_reward(self, episode: int, evaluation_mode=True): return rewards_by_unit def _get_columns(self, table: str) -> list[str]: - if table not in self._column_cache: - self._column_cache[table] = [ - c["name"] for c in self._inspector.get_columns(table) - ] - return self._column_cache[table] + return self._column_cache.setdefault( + table, [c["name"] for c in self._inspector.get_columns(table)] + ) def _copy_df_to_db(self, table: str, df: pd.DataFrame): - """ - Main entry: normalize index, then dispatch to the correct backend - and the correct strategy (upsert only for unit_dispatch). - """ - # 1) If there’s a named index, turn it into a real column - if df.index.name: - df = df.reset_index() - - # 2) Dispatch based on dialect and table - if self.db.dialect.name == "sqlite": - return self._copy_df_to_db_sqlite_append(table, df) - else: - if table == "unit_dispatch": - return self._copy_df_to_unit_dispatch(table, df) - else: - return self._copy_df_to_db_postgres_copy(table, df) - - def _copy_df_to_db_sqlite_append(self, table: str, df: pd.DataFrame): - """ - SQLite append‐only: pandas.to_sql, with retry‐on‐missing‐schema. - """ - try: - with self.db.begin() as conn: - df.to_sql( - name=table, - con=conn, - if_exists="append", - index=df.index.name is not None, - index_label=df.index.name, - ) - except NoSuchTableError: - self._ensure_table(table, df) - return self._copy_df_to_db_sqlite_append(table, df) - except (ProgrammingError, OperationalError, DataError, UndefinedColumn): - self._check_columns(table, df) - return self._copy_df_to_db_sqlite_append(table, df) - except Exception as e: - logger.error("SQLite append failed for %s: %s", table, e) - - def _copy_df_to_db_postgres_copy(self, table: str, df: pd.DataFrame): - """ - Postgres COPY for all tables except unit_dispatch. - """ - try: - buf = StringIO() - df.to_csv(buf, index=False, header=True) - buf.seek(0) - - cols = df.columns.tolist() - col_list = ",".join(f'"{c}"' for c in cols) - sql = ( - f'COPY "{table}" ({col_list}) FROM STDIN WITH (FORMAT CSV, HEADER TRUE)' + df = _prepare_df(df) + key = (self.db.dialect.name, table == "unit_dispatch") + handler = { + ("sqlite", False): self._copy_sqlite, + ("sqlite", True): self._copy_sqlite_upsert, + ("postgresql", False): self._copy_postgres, + ("postgresql", True): self._copy_postgres_upsert, + }.get(key) + if not handler: + raise ValueError(f"No copy strategy for {key}") + return handler(table, df) + + @retry_on_schema_errors + def _copy_sqlite(self, table: str, df: pd.DataFrame): + with self.db.begin() as conn: + df.to_sql( + table, + conn, + if_exists="append", + index=df.index.name is not None, + index_label=df.index.name, ) - raw = self.db.raw_connection() - cur = raw.cursor() - cur.copy_expert(sql, buf) - raw.commit() - - except NoSuchTableError: - self._ensure_table(table, df) - return self._copy_df_to_db_postgres_copy(table, df) - except (ProgrammingError, OperationalError, DataError, UndefinedColumn): - self._check_columns(table, df) - return self._copy_df_to_db_postgres_copy(table, df) - except Exception as e: - logger.error("Postgres COPY failed for %s: %s", table, e) - finally: - try: - cur.close() - except Exception: - pass - - def _copy_df_to_unit_dispatch(self, table: str, df: pd.DataFrame): - # drop duplicates in the staging DataFrame + @retry_on_schema_errors + def _copy_sqlite_upsert(self, table: str, df: pd.DataFrame): df = df.sort_values(["simulation", "time", "unit"]).drop_duplicates( - subset=["simulation", "time", "unit"], keep="last" + ["simulation", "time", "unit"], keep="last" ) + cols = df.columns.tolist() + col_list = ", ".join(f'"{c}"' for c in cols) + placeholders = ", ".join(f":{c}" for c in cols) + stmt = text( + f'INSERT OR REPLACE INTO "{table}" ({col_list}) VALUES ({placeholders})' + ) + with self.db.begin() as conn: + conn.execute(stmt, df.to_dict("records")) + @retry_on_schema_errors + def _copy_postgres(self, table: str, df: pd.DataFrame): buf = StringIO() df.to_csv(buf, index=False, header=True) buf.seek(0) - cols = df.columns.tolist() col_list = ",".join(f'"{c}"' for c in cols) - + sql = f'COPY "{table}" ({col_list}) FROM STDIN WITH (FORMAT CSV, HEADER TRUE)' raw = self.db.raw_connection() cur = raw.cursor() - try: - cur.execute(f""" - CREATE TEMP TABLE tmp_{table} ( - LIKE "{table}" INCLUDING ALL - ) ON COMMIT DROP; - """) - cur.copy_expert( - f"COPY tmp_{table} ({col_list}) FROM STDIN WITH (FORMAT CSV, HEADER TRUE)", - buf, - ) - - pk_cols = ["simulation", "time", "unit"] - update_cols = [c for c in cols if c not in pk_cols] - update_clause = ", ".join(f'"{c}" = EXCLUDED."{c}"' for c in update_cols) + cur.copy_expert(sql, buf) + raw.commit() + cur.close() - cur.execute(f""" - INSERT INTO "{table}" ({col_list}) - SELECT {col_list} FROM tmp_{table} - ON CONFLICT (simulation, time, unit) - DO UPDATE SET {update_clause}; - """) - raw.commit() - - finally: - cur.close() + @retry_on_schema_errors + def _copy_postgres_upsert(self, table: str, df: pd.DataFrame): + df = df.sort_values(["simulation", "time", "unit"]).drop_duplicates( + ["simulation", "time", "unit"], keep="last" + ) + buf = StringIO() + df.to_csv(buf, index=False, header=True) + buf.seek(0) + cols = df.columns.tolist() + col_list = ",".join(f'"{c}"' for c in cols) + raw = self.db.raw_connection() + cur = raw.cursor() + cur.execute("SELECT 1") # ensure connection + # create temp table + cur.execute(f""" + CREATE TEMP TABLE tmp_{table} (LIKE "{table}" INCLUDING ALL) ON COMMIT DROP; + """) + cur.copy_expert( + f"COPY tmp_{table} ({col_list}) FROM STDIN WITH (FORMAT CSV, HEADER TRUE)", + buf, + ) + pk = ["simulation", "time", "unit"] + updates = ", ".join(f'"{c}"=EXCLUDED."{c}"' for c in cols if c not in pk) + cur.execute(f""" + INSERT INTO "{table}" ({col_list}) + SELECT {col_list} FROM tmp_{table} + ON CONFLICT ({','.join(pk)}) DO UPDATE SET {updates}; + """) + raw.commit() + cur.close() def _create_partitions(self, simulation_id: str): - """Create one child‐partition per parent table for this simulation_id.""" with self.db.begin() as conn: for table in PARTITIONED_TABLES: partition = f"{table}_{simulation_id}" - sql = f""" - CREATE TABLE IF NOT EXISTS {partition} - PARTITION OF {table} - FOR VALUES IN ('{simulation_id}'); - """ - conn.execute(text(sql)) + conn.execute( + text( + f"CREATE TABLE IF NOT EXISTS {partition} " + f"PARTITION OF {table} FOR VALUES IN ('{simulation_id}');" + ) + ) logger.debug("created partition %s", partition) def _ensure_table(self, table: str, df: pd.DataFrame): - """ - If `table` doesn’t exist in the DB yet, create it using df.head(0).to_sql(), - so that future copies will succeed. - """ if not self._inspector.has_table(table): - # Use zero‐row to create the right columns & types df.head(0).to_sql( name=table, con=self.db, - if_exists="append", # create if missing, then do nothing + if_exists="append", index=bool(df.index.name), method=None, ) def _check_columns(self, table: str, df: pd.DataFrame, index: bool = True): - """ - Checks and adds columns to the database table if necessary. - - Args: - table (str): The name of the database table. - df (pandas.DataFrame): The DataFrame to be checked. - index (bool): Whether to also ensure the index name exists as a column. - """ - # 1) Fetch current columns from the database with self.db.begin() as conn: - query = f"SELECT * FROM {table} WHERE 1=0" - db_columns = pd.read_sql(query, conn).columns - - # Normalize to lowercase for robust comparison - db_cols_lower = [c.lower() for c in db_columns] - - # 2) Add any missing DataFrame columns - for column in df.columns: - if column.lower() not in db_cols_lower: - try: - if is_bool_dtype(df[column]): - column_type = "BOOLEAN" - elif is_numeric_dtype(df[column]): - column_type = "DOUBLE PRECISION" - else: - column_type = "TEXT" - alter = f'ALTER TABLE {table} ADD COLUMN "{column}" {column_type}' - with self.db.begin() as conn: - conn.execute(text(alter)) - logger.debug("Added column %s to table %s", column, table) - # update our lowercase cache list immediately - db_cols_lower.append(column.lower()) - except Exception: - logger.exception("Error adding column %s to %s", column, table) - - # 3) Optionally add the index name as a column + db_columns = pd.read_sql(f"SELECT * FROM {table} WHERE 1=0", conn).columns + db_cols_lower = {c.lower() for c in db_columns} + for col in df.columns: + if col.lower() not in db_cols_lower: + dtype = ( + "BOOLEAN" + if is_bool_dtype(df[col]) + else "DOUBLE PRECISION" + if is_numeric_dtype(df[col]) + else "TEXT" + ) + alter = text(f'ALTER TABLE {table} ADD COLUMN "{col}" {dtype}') + with self.db.begin() as conn: + conn.execute(alter) + db_cols_lower.add(col.lower()) if index and df.index.name: - idx = df.index.name.lower() - if idx not in db_cols_lower: - try: - column_type = ( - "DOUBLE PRECISION" if is_numeric_dtype(df.index) else "TEXT" - ) - alter = f'ALTER TABLE {table} ADD COLUMN "{idx}" {column_type}' - with self.db.begin() as conn: - conn.execute(text(alter)) - logger.info("Added index-column %s to table %s", idx, table) - db_cols_lower.append(idx) - except Exception: - logger.exception("Error adding index column %s to %s", idx, table) - - # 4) Invalidate the cached column list so _get_columns() will re-fetch next time + idx = df.index.name + if idx.lower() not in db_cols_lower: + dtype = "DOUBLE PRECISION" if is_numeric_dtype(df.index) else "TEXT" + alter = text(f'ALTER TABLE {table} ADD COLUMN "{idx}" {dtype}') + with self.db.begin() as conn: + conn.execute(alter) self._column_cache.pop(table, None) def _purge_simulation_data_sqlite(self, simulation_id: str): - """ - SQLite‐only fallback for clearing out old simulation data. - Deletes rows from every table where simulation = simulation_id, - but skips any tables that don’t yet exist. - """ for table in ALL_SIM_TABLES: - # do not delete rl table if in learning mode if not self._inspector.has_table(table): - logger.debug("Skipping purge: table %s does not exist", table) continue - - # only delete rl_params and rl_meta during the first episode of learning - if table in ["rl_params", "rl_meta"] and not ( + if table in {"rl_params", "rl_meta"} and not ( self.learning_mode and self.episode == 1 ): continue - with self.db.begin() as conn: - sql = text(f'DELETE FROM "{table}" WHERE simulation = :sim') - res = conn.execute(sql, {"sim": simulation_id}) - logger.debug("Deleted %s rows from %s", res.rowcount, table) + conn.execute( + text(f'DELETE FROM "{table}" WHERE simulation = :sim'), + {"sim": simulation_id}, + ) def _purge_simulation_data_postgresql(self, simulation_id: str): """ TimescaleDB/Postgres cleanup: 1) Drop any simulation-specific partitions. 2) Delete all rows in the static tables for that simulation, - but only if the table actually exists. + but only if the table actually exists. """ - inspector = self._inspector - # 1) drop per-simulation partitions for table in PARTITIONED_TABLES: # only delete rl_params during the first episode of learning if table == "rl_params" and not (self.learning_mode and self.episode == 1): continue - - partition_name = f"{table}_{simulation_id}" - sql_drop = text(f'DROP TABLE IF EXISTS "{partition_name}";') with self.db.begin() as conn: - conn.execute(sql_drop) - logger.debug("Dropped partition %s if it existed", partition_name) + conn.execute(text(f'DROP TABLE IF EXISTS "{table}_{simulation_id}";')) + logger.debug("Dropped partition %s", f"{table}_{simulation_id}") # 2) delete from the static tables, skipping any that don’t exist static_tables = [t for t in ALL_SIM_TABLES if t not in PARTITIONED_TABLES] @@ -964,15 +905,16 @@ def _purge_simulation_data_postgresql(self, simulation_id: str): # only delete rl_meta during the first episode of learning if table == "rl_meta" and not (self.learning_mode and self.episode == 1): continue - - if not inspector.has_table(table): - logger.debug("Skipping purge: table %s does not exist", table) + if not self._inspector.has_table(table): continue - with self.db.begin() as conn: - sql_delete = text(f'DELETE FROM "{table}" WHERE simulation = :sim;') - res = conn.execute(sql_delete, {"sim": simulation_id}) - logger.debug("Deleted %s rows from %s", res.rowcount, table) + conn.execute( + text(f'DELETE FROM "{table}" WHERE simulation = :sim;'), + {"sim": simulation_id}, + ) + logger.debug( + "Deleted rows from %s for simulation %s", table, simulation_id + ) class DatabaseMaintenance: