From 80ada70ce26f44f0c04c7a1a134615706f15418d Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Wed, 17 Jun 2026 20:43:21 -0400 Subject: [PATCH] #1141: per-event notification mode for deadlock/blocking alerts (global, both apps) Adds an opt-in Per-event delivery mode (default stays Summary) that sends one notification per distinct incident instead of the batched per-cycle card, so downstream automation can open/track one ticket per incident and count recurrences via the #1140 fingerprint. - PerEventNotification.Split (shared): one message per incident, capped at the configured max-per-cycle, with a trailing "+N more" message that still carries the remaining fingerprints so none are silently dropped. Recurrence handling is left to the existing edge-triggered gating + the consumer's fingerprint dedup. - Settings: AlertDeliveryMode (Summary|PerEvent) + AlertPerEventMaxPerCycle (default 10) in both apps (Lite App statics + JSON; Dashboard UserPreferences), with load/save/reset. - Settings UI: a delivery-mode dropdown + per-cycle cap in both SettingsWindows. - Firing: a SendDetectedAlertAsync helper in each MainWindow routes the "Blocking Detected" and "Deadlocks Detected" sends through Per-event when enabled; alert-history recording is unchanged (one row per fire). Scope: GLOBAL setting (per-server override is a tracked fast-follow). 5 unit tests for the split helper. Lite 497 + Dashboard 487 tests green; both apps build 0-warnings. Refs #1141 Co-Authored-By: Claude Opus 4.8 (1M context) --- Dashboard/MainWindow.xaml.cs | 27 ++++++- Dashboard/Models/UserPreferences.cs | 11 +++ Dashboard/SettingsWindow.xaml | 11 +++ Dashboard/SettingsWindow.xaml.cs | 10 +++ Lite.Tests/PerEventNotificationTests.cs | 69 ++++++++++++++++ Lite/App.xaml.cs | 8 ++ Lite/MainWindow.xaml.cs | 37 +++++++-- Lite/Windows/SettingsWindow.xaml | 14 ++++ Lite/Windows/SettingsWindow.xaml.cs | 11 +++ .../PerEventNotification.cs | 80 +++++++++++++++++++ 10 files changed, 270 insertions(+), 8 deletions(-) create mode 100644 Lite.Tests/PerEventNotificationTests.cs create mode 100644 PerformanceMonitor.Notifications/PerEventNotification.cs diff --git a/Dashboard/MainWindow.xaml.cs b/Dashboard/MainWindow.xaml.cs index 81d9dcd0..0184479b 100644 --- a/Dashboard/MainWindow.xaml.cs +++ b/Dashboard/MainWindow.xaml.cs @@ -1603,7 +1603,7 @@ either suppresses alerts (offset went back) or bypasses the cooldown (offset wen if (!isMuted) { - await _emailAlertService.TrySendAlertEmailAsync( + await SendDetectedAlertAsync(prefs, "Blocking Detected", serverName, $"{(int)health.TotalBlocked} session(s), longest {(int)health.LongestBlockedSeconds}s", @@ -1670,7 +1670,7 @@ Falls back to the raw delta when no databases are excluded. */ if (!isMuted) { - await _emailAlertService.TrySendAlertEmailAsync( + await SendDetectedAlertAsync(prefs, "Deadlocks Detected", serverName, effectiveDeadlockDelta.ToString(), @@ -2181,6 +2181,29 @@ private static string Truncate(string text, int maxLength = 300) return text.Length <= maxLength ? text : text.Substring(0, maxLength) + "..."; } + /* #1141: in Per-event mode, deliver one notification per distinct incident (capped at + AlertPerEventMaxPerCycle, with a trailing "+N more" carrying the remaining fingerprints) + instead of one batched summary card. Falls back to the single summary send in Summary mode + or when there are no incidents. Recording to alert history is left to the one RecordAlert + call at the firing site; this only shapes the outbound send. */ + private async Task SendDetectedAlertAsync( + UserPreferences prefs, string metricName, string serverName, string summaryCurrentValue, + string thresholdValue, string serverId, AlertContext? context) + { + if (prefs.AlertDeliveryMode == AlertNotificationMode.PerEvent && context?.Incidents is { Count: > 0 }) + { + foreach (var msg in PerEventNotification.Split(context, prefs.AlertPerEventMaxPerCycle)) + { + await _emailAlertService.TrySendAlertEmailAsync( + metricName, serverName, msg.CurrentValue, thresholdValue, serverId, msg.Context); + } + return; + } + + await _emailAlertService.TrySendAlertEmailAsync( + metricName, serverName, summaryCurrentValue, thresholdValue, serverId, context); + } + private static string? ContextToDetailText(AlertContext? context) { if (context == null || context.Details.Count == 0) return null; diff --git a/Dashboard/Models/UserPreferences.cs b/Dashboard/Models/UserPreferences.cs index 46726e9a..7dc8b4af 100644 --- a/Dashboard/Models/UserPreferences.cs +++ b/Dashboard/Models/UserPreferences.cs @@ -8,6 +8,7 @@ using System.Collections.Generic; using System.Text.Json.Serialization; using PerformanceMonitor.Ui; +using PerformanceMonitor.Notifications; namespace PerformanceMonitorDashboard.Models { @@ -126,6 +127,16 @@ public int EmailCooldownMinutes set => _emailCooldownMinutes = Math.Clamp(value, 1, 120); } + /* #1141: deadlock/blocking notification delivery — Summary (one batched card per cycle, the + default) or PerEvent (one notification per distinct incident, capped). */ + public AlertNotificationMode AlertDeliveryMode { get; set; } = AlertNotificationMode.Summary; + private int _alertPerEventMaxPerCycle = 10; + public int AlertPerEventMaxPerCycle + { + get => _alertPerEventMaxPerCycle; + set => _alertPerEventMaxPerCycle = Math.Clamp(value, 1, 100); + } + // SMTP email alert settings public bool SmtpEnabled { get; set; } = false; public string SmtpServer { get; set; } = ""; diff --git a/Dashboard/SettingsWindow.xaml b/Dashboard/SettingsWindow.xaml index c7f4b358..70ee6d13 100644 --- a/Dashboard/SettingsWindow.xaml +++ b/Dashboard/SettingsWindow.xaml @@ -320,6 +320,17 @@ + + + + + + + + + + + diff --git a/Dashboard/SettingsWindow.xaml.cs b/Dashboard/SettingsWindow.xaml.cs index f2659724..ef49d728 100644 --- a/Dashboard/SettingsWindow.xaml.cs +++ b/Dashboard/SettingsWindow.xaml.cs @@ -199,6 +199,8 @@ private void LoadSettings() FailedJobLookbackTextBox.Text = prefs.FailedJobLookbackMinutes.ToString(CultureInfo.InvariantCulture); AlertCooldownTextBox.Text = prefs.AlertCooldownMinutes.ToString(CultureInfo.InvariantCulture); EmailCooldownTextBox.Text = prefs.EmailCooldownMinutes.ToString(CultureInfo.InvariantCulture); + AlertDeliveryModeCombo.SelectedIndex = prefs.AlertDeliveryMode == AlertNotificationMode.PerEvent ? 1 : 0; + AlertPerEventMaxTextBox.Text = prefs.AlertPerEventMaxPerCycle.ToString(CultureInfo.InvariantCulture); MuteRuleDefaultExpirationCombo.SelectedIndex = prefs.MuteRuleDefaultExpiration switch { "1 hour" => 0, @@ -391,6 +393,8 @@ private void RestoreAlertDefaultsButton_Click(object sender, RoutedEventArgs e) FailedJobLookbackTextBox.Text = "60"; AlertCooldownTextBox.Text = "5"; EmailCooldownTextBox.Text = "15"; + AlertDeliveryModeCombo.SelectedIndex = 0; + AlertPerEventMaxTextBox.Text = "10"; AlertExcludedDatabasesTextBox.Text = ""; MuteRuleDefaultExpirationCombo.SelectedIndex = 1; // 24 hours UpdateAlertPreviewText(); @@ -758,6 +762,12 @@ private async void OkButton_Click(object sender, RoutedEventArgs e) else validationErrors.Add("Email alert cooldown must be between 1 and 120 minutes"); + prefs.AlertDeliveryMode = AlertDeliveryModeCombo.SelectedIndex == 1 ? AlertNotificationMode.PerEvent : AlertNotificationMode.Summary; + if (int.TryParse(AlertPerEventMaxTextBox.Text, out int perEventMax) && perEventMax >= 1 && perEventMax <= 100) + prefs.AlertPerEventMaxPerCycle = perEventMax; + else + validationErrors.Add("Per-event max-per-cycle must be between 1 and 100"); + prefs.MuteRuleDefaultExpiration = (MuteRuleDefaultExpirationCombo.SelectedItem as ComboBoxItem)?.Content?.ToString() ?? "24 hours"; MuteRuleDialog.DefaultExpiration = prefs.MuteRuleDefaultExpiration; prefs.LogAlertDismissals = LogAlertDismissalsCheckBox.IsChecked == true; diff --git a/Lite.Tests/PerEventNotificationTests.cs b/Lite.Tests/PerEventNotificationTests.cs new file mode 100644 index 00000000..018a1d7d --- /dev/null +++ b/Lite.Tests/PerEventNotificationTests.cs @@ -0,0 +1,69 @@ +using System.Collections.Generic; +using System.Linq; +using PerformanceMonitor.Notifications; +using Xunit; + +namespace PerformanceMonitorLite.Tests; + +/// +/// Guards (#1141): one message per incident, capped, with a +/// trailing overflow batch so no #1140 fingerprint is ever dropped. +/// +public class PerEventNotificationTests +{ + private static AlertContext WithIncidents(int n) + { + var incidents = new List(); + for (int i = 0; i < n; i++) + incidents.Add(new AlertIncident($"key{i}", new[] { $"db.dbo.T{i}" }, OccurrenceCount: i + 1)); + var ctx = new AlertContext(); + AlertIncidentRenderer.Apply(ctx, incidents); + return ctx; + } + + [Fact] + public void Split_NoIncidents_ReturnsEmpty() + { + Assert.Empty(PerEventNotification.Split(new AlertContext(), 10)); + } + + [Fact] + public void Split_WithinCap_OneMessagePerIncident() + { + var messages = PerEventNotification.Split(WithIncidents(3), 10); + Assert.Equal(3, messages.Count); + Assert.All(messages, m => Assert.False(m.IsOverflow)); + Assert.All(messages, m => Assert.Single(m.Context.Incidents!)); + Assert.Equal("db.dbo.T0", messages[0].CurrentValue); + } + + [Fact] + public void Split_OverCap_CapsIndividualAndBatchesOverflow() + { + var messages = PerEventNotification.Split(WithIncidents(5), 2); + Assert.Equal(3, messages.Count); // 2 individual + 1 overflow + Assert.False(messages[0].IsOverflow); + Assert.False(messages[1].IsOverflow); + Assert.True(messages[2].IsOverflow); + Assert.Equal(3, messages[2].Context.Incidents!.Count); // overflow carries the remaining 3 + Assert.Contains("+3 more", messages[2].CurrentValue); + } + + [Fact] + public void Split_PreservesEveryFingerprint() + { + var source = WithIncidents(5); + var messages = PerEventNotification.Split(source, 2); + var emitted = messages.SelectMany(m => m.Context.Incidents!).Select(i => i.DedupKey).ToHashSet(); + var expected = source.Incidents!.Select(i => i.DedupKey).ToHashSet(); + Assert.Equal(expected, emitted); + } + + [Fact] + public void Split_CapZero_TreatedAsOne() + { + var messages = PerEventNotification.Split(WithIncidents(3), 0); + Assert.Equal(2, messages.Count); // 1 individual + overflow of 2 + Assert.True(messages[1].IsOverflow); + } +} diff --git a/Lite/App.xaml.cs b/Lite/App.xaml.cs index a904287e..93af6088 100644 --- a/Lite/App.xaml.cs +++ b/Lite/App.xaml.cs @@ -14,6 +14,7 @@ using System.Threading; using System.Threading.Tasks; using System.Windows; +using PerformanceMonitor.Notifications; using System.Windows.Threading; using PerformanceMonitorLite.Services; using PerformanceMonitor.Ui; @@ -114,6 +115,10 @@ public partial class App : Application public static int AlertFailedJobLookbackMinutes { get; set; } = 60; // Look back this many minutes for failed Agent job runs public static int AlertCooldownMinutes { get; set; } = 5; // Tray notification cooldown between repeated alerts public static int EmailCooldownMinutes { get; set; } = 15; // Email cooldown between repeated alerts + /* #1141: deadlock/blocking notification delivery — Summary (one batched card per cycle, the default) + or PerEvent (one notification per distinct incident, capped, for per-incident ticketing). */ + public static AlertNotificationMode AlertDeliveryMode { get; set; } = AlertNotificationMode.Summary; + public static int AlertPerEventMaxPerCycle { get; set; } = 10; // Max per-event notifications per cycle before "+N more" public static string MuteRuleDefaultExpiration { get; set; } = "24 hours"; // Default expiration for new mute rules public static bool LogAlertDismissals { get; set; } = true; // Log alert dismiss/mute actions to file @@ -499,6 +504,9 @@ public static void LoadAlertSettings() if (root.TryGetProperty("alert_failed_job_lookback_minutes", out v)) AlertFailedJobLookbackMinutes = (int)Math.Clamp(v.GetInt64(), 1, 1440); if (root.TryGetProperty("alert_cooldown_minutes", out v)) AlertCooldownMinutes = (int)Math.Clamp(v.GetInt64(), 1, 120); if (root.TryGetProperty("email_cooldown_minutes", out v)) EmailCooldownMinutes = (int)Math.Clamp(v.GetInt64(), 1, 120); + if (root.TryGetProperty("alert_delivery_mode", out v) && Enum.TryParse(v.GetString(), out var deliveryMode)) + AlertDeliveryMode = deliveryMode; + if (root.TryGetProperty("alert_per_event_max_per_cycle", out v)) AlertPerEventMaxPerCycle = (int)Math.Clamp(v.GetInt64(), 1, 100); if (root.TryGetProperty("mute_rule_default_expiration", out v)) { var exp = v.GetString(); diff --git a/Lite/MainWindow.xaml.cs b/Lite/MainWindow.xaml.cs index a9413270..d188c435 100644 --- a/Lite/MainWindow.xaml.cs +++ b/Lite/MainWindow.xaml.cs @@ -1668,15 +1668,15 @@ await _emailAlertService.TrySendAlertEmailAsync( var blockingContext = await BuildBlockingContextAsync(summary.ServerId, summary.DisplayName); var detailText = ContextToDetailText(blockingContext); - await _emailAlertService.TrySendAlertEmailAsync( + await SendDetectedAlertAsync( "Blocking Detected", summary.DisplayName, effectiveBlockingCount.ToString(), App.AlertBlockingThreshold.ToString(), summary.ServerId, blockingContext, - muted: isMuted, - detailText: detailText); + isMuted, + detailText); } else if (!blockingDecision.Active && wasBlockingActive) { @@ -1739,15 +1739,15 @@ await _emailAlertService.TrySendAlertEmailAsync( var deadlockContext = await BuildDeadlockContextAsync(summary.ServerId, summary.DisplayName); var detailText = ContextToDetailText(deadlockContext); - await _emailAlertService.TrySendAlertEmailAsync( + await SendDetectedAlertAsync( "Deadlocks Detected", summary.DisplayName, effectiveDeadlockCount.ToString(), App.AlertDeadlockThreshold.ToString(), summary.ServerId, deadlockContext, - muted: isMuted, - detailText: detailText); + isMuted, + detailText); } else if (!deadlockDecision.Active && wasDeadlockActive) { @@ -2235,6 +2235,31 @@ private static string TruncateText(string text, int maxLength = 300) return text.Length <= maxLength ? text : text.Substring(0, maxLength) + "..."; } + /* #1141: in Per-event mode, deliver one notification per distinct incident (capped at + AlertPerEventMaxPerCycle, with a trailing "+N more" that still carries the remaining + fingerprints) instead of one batched summary card. Falls back to the single summary send in + Summary mode or when there are no incidents. The existing edge-triggered gating still decides + whether to fire; this only shapes delivery. */ + private async Task SendDetectedAlertAsync( + string metricName, string serverName, string summaryCurrentValue, string thresholdValue, + int serverId, AlertContext? context, bool isMuted, string? summaryDetailText) + { + if (App.AlertDeliveryMode == AlertNotificationMode.PerEvent && context?.Incidents is { Count: > 0 }) + { + foreach (var msg in PerEventNotification.Split(context, App.AlertPerEventMaxPerCycle)) + { + await _emailAlertService.TrySendAlertEmailAsync( + metricName, serverName, msg.CurrentValue, thresholdValue, serverId, + msg.Context, muted: isMuted, detailText: ContextToDetailText(msg.Context)); + } + return; + } + + await _emailAlertService.TrySendAlertEmailAsync( + metricName, serverName, summaryCurrentValue, thresholdValue, serverId, + context, muted: isMuted, detailText: summaryDetailText); + } + private static string? ContextToDetailText(AlertContext? context) { if (context == null || context.Details.Count == 0) return null; diff --git a/Lite/Windows/SettingsWindow.xaml b/Lite/Windows/SettingsWindow.xaml index 0d7b5e10..b076ff54 100644 --- a/Lite/Windows/SettingsWindow.xaml +++ b/Lite/Windows/SettingsWindow.xaml @@ -260,6 +260,20 @@ + + + + + + + + + + + diff --git a/Lite/Windows/SettingsWindow.xaml.cs b/Lite/Windows/SettingsWindow.xaml.cs index 647efa16..734efd91 100644 --- a/Lite/Windows/SettingsWindow.xaml.cs +++ b/Lite/Windows/SettingsWindow.xaml.cs @@ -606,6 +606,8 @@ private void LoadAlertSettings() AlertFailedJobLookbackBox.Text = App.AlertFailedJobLookbackMinutes.ToString(); AlertCooldownBox.Text = App.AlertCooldownMinutes.ToString(); EmailCooldownBox.Text = App.EmailCooldownMinutes.ToString(); + AlertDeliveryModeBox.SelectedIndex = App.AlertDeliveryMode == AlertNotificationMode.PerEvent ? 1 : 0; + AlertPerEventMaxBox.Text = App.AlertPerEventMaxPerCycle.ToString(); MuteRuleDefaultExpirationCombo.SelectedIndex = App.MuteRuleDefaultExpiration switch { "1 hour" => 0, @@ -677,6 +679,11 @@ private bool SaveAlertSettings() App.EmailCooldownMinutes = emailCooldown; else validationErrors.Add("Email alert cooldown must be between 1 and 120 minutes."); + App.AlertDeliveryMode = AlertDeliveryModeBox.SelectedIndex == 1 ? AlertNotificationMode.PerEvent : AlertNotificationMode.Summary; + if (int.TryParse(AlertPerEventMaxBox.Text, out var perEventMax) && perEventMax >= 1 && perEventMax <= 100) + App.AlertPerEventMaxPerCycle = perEventMax; + else + validationErrors.Add("Per-event max-per-cycle must be between 1 and 100."); App.MuteRuleDefaultExpiration = (MuteRuleDefaultExpirationCombo.SelectedItem as ComboBoxItem)?.Content?.ToString() ?? "24 hours"; App.LogAlertDismissals = LogAlertDismissalsCheckBox.IsChecked == true; App.AnalysisEnabled = AnalysisEnabledCheckBox.IsChecked == true; @@ -739,6 +746,8 @@ private bool SaveAlertSettings() root["alert_failed_job_lookback_minutes"] = App.AlertFailedJobLookbackMinutes; root["alert_cooldown_minutes"] = App.AlertCooldownMinutes; root["email_cooldown_minutes"] = App.EmailCooldownMinutes; + root["alert_delivery_mode"] = App.AlertDeliveryMode.ToString(); + root["alert_per_event_max_per_cycle"] = App.AlertPerEventMaxPerCycle; root["mute_rule_default_expiration"] = App.MuteRuleDefaultExpiration; root["log_alert_dismissals"] = App.LogAlertDismissals; root["analysis_enabled"] = App.AnalysisEnabled; @@ -786,6 +795,8 @@ private void RestoreAlertDefaultsButton_Click(object sender, RoutedEventArgs e) AlertFailedJobLookbackBox.Text = "60"; AlertCooldownBox.Text = "5"; EmailCooldownBox.Text = "15"; + AlertDeliveryModeBox.SelectedIndex = 0; + AlertPerEventMaxBox.Text = "10"; AnalysisIntervalBox.Text = "30"; AnalysisNotifySeverityBox.Text = "1.5"; AlertExcludedDatabasesBox.Text = ""; diff --git a/PerformanceMonitor.Notifications/PerEventNotification.cs b/PerformanceMonitor.Notifications/PerEventNotification.cs new file mode 100644 index 00000000..ded07b62 --- /dev/null +++ b/PerformanceMonitor.Notifications/PerEventNotification.cs @@ -0,0 +1,80 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Collections.Generic; +using System.Linq; + +namespace PerformanceMonitor.Notifications; + +/// +/// How deadlock/blocking alerts are delivered (#1141). is the default — one +/// batched notification per alert cycle listing all incidents. sends one +/// notification per distinct incident so downstream automation (e.g. a Logic App) can open/track one +/// ticket per incident and count recurrences via the #1140 dedup fingerprint. +/// +public enum AlertNotificationMode +{ + Summary = 0, + PerEvent = 1 +} + +/// +/// Splits a built alert into per-incident messages for #1141 Per-event mode. +/// Each distinct incident (already grouped + fingerprinted by #1140) becomes one message carrying that +/// single incident; when the incident count exceeds the per-cycle cap, the overflow incidents are +/// batched into a final "+N more" message so no fingerprint is ever dropped (the requester's "don't +/// silently truncate"). Recurrence handling is left to the existing edge-triggered alert gating + the +/// consumer's fingerprint dedup — this helper only shapes delivery. +/// +public static class PerEventNotification +{ + /// One per-event notification to send: the single-incident (or overflow) context plus the + /// "current value" string the caller passes to its alert sender. + public sealed record Message(AlertContext Context, string CurrentValue, bool IsOverflow); + + /// + /// Produces one message per incident (capped at ), with a trailing + /// overflow message carrying any remaining incidents. Returns an empty list when the source has no + /// incidents — the caller then falls back to a single Summary send. Never mutates . + /// + public static List Split(AlertContext source, int maxPerCycle) + { + var messages = new List(); + if (source?.Incidents is not { Count: > 0 } incidents) + return messages; + + var cap = Math.Max(1, maxPerCycle); + + foreach (var incident in incidents.Take(cap)) + { + var ctx = new AlertContext { SeverityOverride = source.SeverityOverride }; + AlertIncidentRenderer.Apply(ctx, new[] { incident }); + messages.Add(new Message(ctx, DescribeIncident(incident), IsOverflow: false)); + } + + var overflow = incidents.Skip(cap).ToList(); + if (overflow.Count > 0) + { + var ctx = new AlertContext { SeverityOverride = source.SeverityOverride }; + AlertIncidentRenderer.Apply(ctx, overflow); + messages.Add(new Message(ctx, $"+{overflow.Count} more incident(s) this cycle", IsOverflow: true)); + } + + return messages; + } + + // The alert's "current value" string for a single-incident message: the involved objects, or the + // dedup key when no objects resolved, so the notification headline names what the incident is about. + private static string DescribeIncident(AlertIncident incident) + { + if (incident.InvolvedObjects.Count > 0) + return string.Join(", ", incident.InvolvedObjects); + return incident.DedupKey; + } +}