From a4699e50abc0dd0c72d7a069fbabefde79b4bd69 Mon Sep 17 00:00:00 2001 From: kim-mskw Date: Fri, 30 Jan 2026 17:11:37 +0100 Subject: [PATCH 1/2] Add smooth sigmoid-based reward calculation for energy learning strategy --- assume/strategies/learning_strategies.py | 37 ++++++++++++++++++++++-- 1 file changed, 35 insertions(+), 2 deletions(-) diff --git a/assume/strategies/learning_strategies.py b/assume/strategies/learning_strategies.py index a55db0cb0..be9ad60f7 100644 --- a/assume/strategies/learning_strategies.py +++ b/assume/strategies/learning_strategies.py @@ -612,14 +612,27 @@ def calculate_reward( duration = (end - start) / timedelta(hours=1) income = 0.0 + income_smooth = 0.0 operational_cost = 0.0 + operational_costs_smooth = 0.0 accepted_volume_total = 0 + accepted_volume_total_smooth = 0 offered_volume_total = 0 + # Glatte Annahme-Wahrscheinlichkeit statt harter Schwelle + # k steuert die Steilheit (größer = härter, kleiner = weicher) + k = 2 + # Iterate over all orders in the orderbook to calculate order-specific profit. for order in orderbook: + accept_prob_dynamic = 1 / ( + 1 + np.exp(k * (order.get("price", 0) - market_clearing_price)) + ) + accepted_volume = order.get("accepted_volume", 0) + accepted_volume_smooth = accepted_volume * accept_prob_dynamic + accepted_volume_total_smooth += accepted_volume_smooth accepted_volume_total += accepted_volume offered_volume_total += order["volume"] @@ -628,10 +641,19 @@ def calculate_reward( order_income = market_clearing_price * accepted_volume * duration order_cost = marginal_cost * accepted_volume * duration + # calculate smooth values + order_income_smooth = ( + market_clearing_price * accepted_volume_smooth * duration + ) + order_cost_smooth = marginal_cost * accepted_volume_smooth * duration + # Accumulate income and operational cost for all orders. income += order_income operational_cost += order_cost + income_smooth += order_income_smooth + operational_costs_smooth += order_cost_smooth + # Consideration of start-up costs, divided evenly between upward and downward regulation events. if ( unit.outputs[product_type].at[start] != 0 @@ -646,6 +668,9 @@ def calculate_reward( profit = income - operational_cost + # TODO: add start costs here + profit_smooth = income_smooth - operational_costs_smooth + # Stabilizing learning: Limit positive profit to 10% of its absolute value. # This reduces variance in rewards and avoids extreme profit-seeking behavior. # However, this does NOT prevent the agent from exploiting market inefficiencies if they exist. @@ -658,6 +683,7 @@ def calculate_reward( profit_scale = 1 profit = min(profit, profit_scale * abs(profit)) + profit_smooth = min(profit_smooth, profit_scale * abs(profit_smooth)) # Opportunity cost: The income lost due to not operating at full capacity. opportunity_cost = ( @@ -666,8 +692,15 @@ def calculate_reward( * duration ) + opportunity_cost_smooth = ( + (market_clearing_price - marginal_cost) + * (unit.max_power - accepted_volume_smooth) + * duration + ) + # If opportunity cost is negative, no income was lost, so we set it to zero. opportunity_cost = max(opportunity_cost, 0) + opportunity_cost_smooth = max(opportunity_cost_smooth, 0) # Dynamic regret scaling: # - If accepted volume is positive, apply lower regret (0.1) to avoid punishment for being on the edge of the merit order. @@ -681,8 +714,8 @@ def calculate_reward( # scaling factor to normalize the reward to the range [-1,1] scaling = 1 / (self.max_bid_price * unit.max_power) - regret = regret_scale * opportunity_cost - reward = scaling * (profit - regret) + regret = regret_scale * opportunity_cost_smooth + reward = scaling * (profit_smooth - regret) # Store results in unit outputs # Note: these are not learning-specific results but stored for all units for analysis From fa6d66e69c5fc9e123de41de96a54e2b793cda3b Mon Sep 17 00:00:00 2001 From: mthede Date: Mon, 9 Feb 2026 00:05:00 +0100 Subject: [PATCH 2/2] fix regret for smoothed function --- assume/strategies/learning_strategies.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/assume/strategies/learning_strategies.py b/assume/strategies/learning_strategies.py index be9ad60f7..e7a6d978e 100644 --- a/assume/strategies/learning_strategies.py +++ b/assume/strategies/learning_strategies.py @@ -694,7 +694,7 @@ def calculate_reward( opportunity_cost_smooth = ( (market_clearing_price - marginal_cost) - * (unit.max_power - accepted_volume_smooth) + * (unit.max_power - accepted_volume_total_smooth) * duration ) @@ -706,6 +706,9 @@ def calculate_reward( # - If accepted volume is positive, apply lower regret (0.1) to avoid punishment for being on the edge of the merit order. # - If no dispatch happens, apply higher regret (0.5) to discourage idle behavior, if it could have been profitable. regret_scale = 0.1 if accepted_volume_total > unit.min_power else 0.5 + regret_scale_smooth = ( + 0.1 if accepted_volume_total_smooth > unit.min_power else 0.5 + ) # -------------------- # 4.1 Calculate Reward @@ -714,7 +717,7 @@ def calculate_reward( # scaling factor to normalize the reward to the range [-1,1] scaling = 1 / (self.max_bid_price * unit.max_power) - regret = regret_scale * opportunity_cost_smooth + regret = regret_scale_smooth * opportunity_cost_smooth reward = scaling * (profit_smooth - regret) # Store results in unit outputs