From 0095afe539a790701c2cd7b3fec9a1c123e93add Mon Sep 17 00:00:00 2001 From: John Oliver <1615532+johnoliver@users.noreply.github.com> Date: Wed, 27 May 2026 16:55:40 +0000 Subject: [PATCH 1/2] Add the ability to manually trigger profiles via 2 routes: - Creating/Touching a file at a well known location, by default "/applicationinsights-agent-profile-trigger" - Add a JMX mbean, can be called via "jcmd MBean.invoke com.microsoft:type=AI-alert,name=ProfilerControl triggerProfile" - Add a global cooldown on the profile to ensure that we dont get repeat profiles from multiple sources --- ...ertingProfileFileTriggerConfiguration.java | 91 +++++++++ .../alerting/AlertingSubsystem.java | 98 ++++++++- .../AlertingSubsystemFileTriggerTest.java | 186 ++++++++++++++++++ .../alerting/AlertingSubsystemTest.java | 13 +- .../alerting/TestTimeSource.java | 4 + .../internal/configuration/Configuration.java | 32 +++ .../PerformanceMonitoringService.java | 12 +- .../agent/internal/profiler/Profiler.java | 65 +++++- .../internal/profiler/ProfilerControl.java | 111 +++++++++++ .../profiler/ProfilerControlMBean.java | 34 ++++ .../triggers/AlertingSubsystemInit.java | 14 +- .../profiler/ProfilerControlTest.java | 98 +++++++++ .../profiler/ProfilerGlobalCooldownTest.java | 116 +++++++++++ .../AlertTriggerSpanProcessorTest.java | 7 +- .../profiler/triggers/GcEventInitTest.java | 6 +- docs/README.md | 68 ++++++- 16 files changed, 934 insertions(+), 21 deletions(-) create mode 100644 agent/agent-profiler/agent-alerting-api/src/main/java/com/microsoft/applicationinsights/alerting/config/AlertingProfileFileTriggerConfiguration.java create mode 100644 agent/agent-profiler/agent-alerting/src/test/java/com/microsoft/applicationinsights/alerting/AlertingSubsystemFileTriggerTest.java create mode 100644 agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/ProfilerControl.java create mode 100644 agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/ProfilerControlMBean.java create mode 100644 agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/profiler/ProfilerControlTest.java create mode 100644 agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/profiler/ProfilerGlobalCooldownTest.java diff --git a/agent/agent-profiler/agent-alerting-api/src/main/java/com/microsoft/applicationinsights/alerting/config/AlertingProfileFileTriggerConfiguration.java b/agent/agent-profiler/agent-alerting-api/src/main/java/com/microsoft/applicationinsights/alerting/config/AlertingProfileFileTriggerConfiguration.java new file mode 100644 index 00000000000..b79aa02172b --- /dev/null +++ b/agent/agent-profiler/agent-alerting-api/src/main/java/com/microsoft/applicationinsights/alerting/config/AlertingProfileFileTriggerConfiguration.java @@ -0,0 +1,91 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.microsoft.applicationinsights.alerting.config; + +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import java.io.File; + +/** + * Configuration for the file-based manual profile trigger. + * + *

When enabled, the alerting subsystem periodically checks for the existence of a trigger file. + * If the file exists and was recently modified (within {@link #MANUAL_TRIGGER_FILE_MAX_AGE_MS}), it + * is deleted and a manual profile recording is initiated. + * + *

This provides an operator-friendly mechanism for triggering on-demand profiles without + * requiring JMX access – simply {@code touch} the trigger file from a shell or orchestration tool. + */ +public class AlertingProfileFileTriggerConfiguration { + + /** + * Maximum age (in milliseconds) of the trigger file for it to be considered valid. Files older + * than this threshold are ignored to prevent stale trigger files from initiating unexpected + * recordings (e.g., after a restart). + */ + public static final long MANUAL_TRIGGER_FILE_MAX_AGE_MS = 60_000; // 1 minute + + // Whether the file-based manual trigger is enabled. + private final boolean enabled; + + // Path to the file that triggers a manual profile when created/touched. + // If relative, it is resolved against the agent's temp directory. + private final File filePath; + + /** The default duration (in seconds) used for profiles triggered via this file mechanism. */ + private final int defaultProfileDurationSeconds; + + private AlertingProfileFileTriggerConfiguration( + boolean enabled, File filePath, int defaultProfileDurationSeconds) { + this.enabled = enabled; + this.filePath = filePath; + this.defaultProfileDurationSeconds = defaultProfileDurationSeconds; + } + + /** + * Creates a file trigger configuration, resolving relative paths against the provided temp + * directory. + * + * @param enabled whether file-based triggering is active + * @param filePath path to the trigger file (absolute or relative to {@code tempDir}) + * @param defaultProfileDurationSeconds duration in seconds for the profile if no override is + * configured in the collection plan + * @param tempDir base directory used to resolve relative file paths + * @return a fully resolved configuration instance + */ + @SuppressFBWarnings( + value = "SECPTI", + justification = "File path is set by trusted user configuration (applicationinsights.json)") + public static AlertingProfileFileTriggerConfiguration create( + boolean enabled, String filePath, int defaultProfileDurationSeconds, File tempDir) { + + File manualTriggerFile = new File(filePath); + if (!manualTriggerFile.isAbsolute()) { + manualTriggerFile = new File(tempDir, filePath); + } + + return new AlertingProfileFileTriggerConfiguration( + enabled, manualTriggerFile, defaultProfileDurationSeconds); + } + + /** Creates a disabled configuration suitable for tests. */ + public static AlertingProfileFileTriggerConfiguration createDefault() { + return new AlertingProfileFileTriggerConfiguration( + false, new File("applicationinsights-agent-profile-trigger"), 120); + } + + /** Returns the default profile duration in seconds for file-triggered recordings. */ + public int getDefaultProfileDurationSeconds() { + return defaultProfileDurationSeconds; + } + + /** Returns the resolved path of the trigger file. */ + public File getFilePath() { + return filePath; + } + + /** Returns whether the file-based manual trigger is enabled. */ + public boolean isEnabled() { + return enabled; + } +} diff --git a/agent/agent-profiler/agent-alerting/src/main/java/com/microsoft/applicationinsights/alerting/AlertingSubsystem.java b/agent/agent-profiler/agent-alerting/src/main/java/com/microsoft/applicationinsights/alerting/AlertingSubsystem.java index f45719c797b..e49f09af028 100644 --- a/agent/agent-profiler/agent-alerting/src/main/java/com/microsoft/applicationinsights/alerting/AlertingSubsystem.java +++ b/agent/agent-profiler/agent-alerting/src/main/java/com/microsoft/applicationinsights/alerting/AlertingSubsystem.java @@ -11,9 +11,11 @@ import com.microsoft.applicationinsights.alerting.config.AlertConfiguration; import com.microsoft.applicationinsights.alerting.config.AlertMetricType; import com.microsoft.applicationinsights.alerting.config.AlertingConfiguration; +import com.microsoft.applicationinsights.alerting.config.AlertingProfileFileTriggerConfiguration; import com.microsoft.applicationinsights.alerting.config.CollectionPlanConfiguration; import com.microsoft.applicationinsights.alerting.config.CollectionPlanConfiguration.EngineMode; import com.microsoft.applicationinsights.alerting.config.DefaultConfiguration; +import java.io.File; import java.time.Instant; import java.util.ArrayList; import java.util.HashSet; @@ -33,7 +35,6 @@ public class AlertingSubsystem { private static final Logger logger = LoggerFactory.getLogger(AlertingSubsystem.class); - // Downstream observer of alerts produced by the alerting system private final Consumer alertHandler; @@ -46,25 +47,40 @@ public class AlertingSubsystem { // Current configuration of the alerting subsystem private AlertingConfiguration alertConfig; - private boolean enableRequestTriggerUpdates; + /** Configuration controlling the file-based manual profile trigger. */ + private final AlertingProfileFileTriggerConfiguration alertingProfileFileTriggerConfiguration; - protected AlertingSubsystem(Consumer alertHandler) { - this(alertHandler, TimeSource.DEFAULT, false); - } + private boolean enableRequestTriggerUpdates; protected AlertingSubsystem( Consumer alertHandler, TimeSource timeSource, - boolean enableRequestTriggerUpdates) { + boolean enableRequestTriggerUpdates, + AlertingProfileFileTriggerConfiguration alertingProfileFileTriggerConfiguration) { this.alertHandler = alertHandler; this.alertPipelines = new AlertPipelines(alertHandler); this.timeSource = timeSource; this.enableRequestTriggerUpdates = enableRequestTriggerUpdates; + this.alertingProfileFileTriggerConfiguration = alertingProfileFileTriggerConfiguration; } + /** + * Creates and initializes an {@link AlertingSubsystem} with an initially-disabled configuration. + * + * @param alertHandler downstream consumer that handles generated alert breaches + * @param timeSource time source used for alert evaluation windows + * @param alertingProfileFileTriggerConfiguration configuration for the file-based manual trigger + * @return a fully initialized alerting subsystem ready to receive configuration updates + */ public static AlertingSubsystem create( - Consumer alertHandler, TimeSource timeSource) { - AlertingSubsystem alertingSubsystem = new AlertingSubsystem(alertHandler, timeSource, true); + Consumer alertHandler, + TimeSource timeSource, + AlertingProfileFileTriggerConfiguration alertingProfileFileTriggerConfiguration) { + + AlertingSubsystem alertingSubsystem = + new AlertingSubsystem( + alertHandler, timeSource, true, alertingProfileFileTriggerConfiguration); + // init with disabled config alertingSubsystem.initialize( AlertingConfiguration.create( @@ -145,8 +161,17 @@ private void updateRequestPipelineConfig( } } - /** Determine if a manual alert has been requested. */ + /** + * Determine if a manual alert has been requested via any supported mechanism. Currently evaluates + * both the server-side collection plan and the local file-based trigger. + */ private void evaluateManualTrigger(AlertingConfiguration alertConfig) { + evaluateCollectionPlanTrigger(alertConfig); + evaluateFileTrigger(alertConfig); + } + + /** Check if the collection plan configuration requests a manual profile. */ + private void evaluateCollectionPlanTrigger(AlertingConfiguration alertConfig) { CollectionPlanConfiguration config = alertConfig.getCollectionPlanConfiguration(); boolean shouldTrigger = @@ -176,6 +201,61 @@ private void evaluateManualTrigger(AlertingConfiguration alertConfig) { } } + /** + * Check if a trigger file is present on the local file system and was recently modified. If so, + * delete the file and trigger a manual profile. The global cooldown in Profiler prevents + * overlapping profiles. + */ + private void evaluateFileTrigger(AlertingConfiguration alertConfig) { + if (!alertingProfileFileTriggerConfiguration.isEnabled()) { + return; + } + + File manualTriggerFile = alertingProfileFileTriggerConfiguration.getFilePath(); + if (manualTriggerFile == null || !manualTriggerFile.exists()) { + return; + } + + long lastModified = manualTriggerFile.lastModified(); + long age = timeSource.getNow().toEpochMilli() - lastModified; + + if (age > AlertingProfileFileTriggerConfiguration.MANUAL_TRIGGER_FILE_MAX_AGE_MS) { + return; + } + + // Delete the trigger file to prevent re-triggering + if (!manualTriggerFile.delete()) { + logger.warn( + "Failed to delete manual profile trigger file: {}", manualTriggerFile.getAbsolutePath()); + } + + logger.info("Manual profile trigger file detected, initiating profile recording"); + + // Use the collection plan's duration if configured, otherwise fall back to the + // file trigger's default duration setting. + CollectionPlanConfiguration collectionPlan = alertConfig.getCollectionPlanConfiguration(); + int durationSeconds = collectionPlan.getImmediateProfilingDurationSeconds(); + if (durationSeconds <= 0) { + durationSeconds = alertingProfileFileTriggerConfiguration.getDefaultProfileDurationSeconds(); + } + + AlertBreach alertBreach = + AlertBreach.builder() + .setType(AlertMetricType.MANUAL) + .setAlertValue(0.0) + .setAlertConfiguration( + AlertConfiguration.builder() + .setType(AlertMetricType.MANUAL) + .setEnabled(true) + .setProfileDurationSeconds(durationSeconds) + .build()) + .setProfileId(UUID.randomUUID().toString()) + .setCpuMetric(0) + .setMemoryUsage(0) + .build(); + alertHandler.accept(alertBreach); + } + public void setPipeline(AlertMetricType type, AlertPipeline alertPipeline) { alertPipelines.setAlertPipeline(type, alertPipeline); } diff --git a/agent/agent-profiler/agent-alerting/src/test/java/com/microsoft/applicationinsights/alerting/AlertingSubsystemFileTriggerTest.java b/agent/agent-profiler/agent-alerting/src/test/java/com/microsoft/applicationinsights/alerting/AlertingSubsystemFileTriggerTest.java new file mode 100644 index 00000000000..51cf4a4d9a8 --- /dev/null +++ b/agent/agent-profiler/agent-alerting/src/test/java/com/microsoft/applicationinsights/alerting/AlertingSubsystemFileTriggerTest.java @@ -0,0 +1,186 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.microsoft.applicationinsights.alerting; + +import static org.assertj.core.api.Assertions.assertThat; + +import com.microsoft.applicationinsights.alerting.alert.AlertBreach; +import com.microsoft.applicationinsights.alerting.config.AlertConfiguration; +import com.microsoft.applicationinsights.alerting.config.AlertMetricType; +import com.microsoft.applicationinsights.alerting.config.AlertingConfiguration; +import com.microsoft.applicationinsights.alerting.config.AlertingProfileFileTriggerConfiguration; +import com.microsoft.applicationinsights.alerting.config.CollectionPlanConfiguration; +import com.microsoft.applicationinsights.alerting.config.CollectionPlanConfiguration.EngineMode; +import com.microsoft.applicationinsights.alerting.config.DefaultConfiguration; +import java.io.File; +import java.io.IOException; +import java.time.Instant; +import java.util.ArrayList; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class AlertingSubsystemFileTriggerTest { + + @TempDir File tempDir; + + private static AlertingConfiguration defaultAlertingConfig() { + return AlertingConfiguration.create( + AlertConfiguration.builder() + .setType(AlertMetricType.CPU) + .setEnabled(false) + .setThreshold(80.0f) + .setProfileDurationSeconds(30) + .setCooldownSeconds(14400) + .build(), + AlertConfiguration.builder() + .setType(AlertMetricType.MEMORY) + .setEnabled(false) + .setThreshold(20.0f) + .setProfileDurationSeconds(120) + .setCooldownSeconds(14400) + .build(), + DefaultConfiguration.builder().build(), + CollectionPlanConfiguration.builder() + .setSingle(false) + .setMode(EngineMode.immediate) + .setExpiration(Instant.now()) + .setImmediateProfilingDurationSeconds(0) + .setSettingsMoniker("") + .build(), + new ArrayList<>()); + } + + @Test + void fileTriggerFiresWhenFileExistsAndIsRecent() throws IOException { + File triggerFile = new File(tempDir, "trigger"); + assertThat(triggerFile.createNewFile()).isTrue(); + + AtomicReference breach = new AtomicReference<>(); + Consumer consumer = breach::set; + + AlertingProfileFileTriggerConfiguration config = + AlertingProfileFileTriggerConfiguration.create(true, "trigger", 120, tempDir); + + AlertingSubsystem subsystem = AlertingSubsystem.create(consumer, new TestTimeSource(), config); + + subsystem.updateConfiguration(defaultAlertingConfig()); + + assertThat(breach.get()).isNotNull(); + assertThat(breach.get().getType()).isEqualTo(AlertMetricType.MANUAL); + assertThat(breach.get().getAlertConfiguration().getProfileDurationSeconds()).isEqualTo(120); + // trigger file should be deleted + assertThat(triggerFile.exists()).isFalse(); + } + + @Test + void fileTriggerDoesNotFireWhenDisabled() throws IOException { + File triggerFile = new File(tempDir, "trigger"); + assertThat(triggerFile.createNewFile()).isTrue(); + + AtomicReference breach = new AtomicReference<>(); + Consumer consumer = breach::set; + + AlertingProfileFileTriggerConfiguration config = + AlertingProfileFileTriggerConfiguration.create(false, "trigger", 120, tempDir); + + AlertingSubsystem subsystem = AlertingSubsystem.create(consumer, new TestTimeSource(), config); + + subsystem.updateConfiguration(defaultAlertingConfig()); + + assertThat(breach.get()).isNull(); + // file should not be deleted when disabled + assertThat(triggerFile.exists()).isTrue(); + } + + @Test + void fileTriggerDoesNotFireWhenFileDoesNotExist() { + AtomicReference breach = new AtomicReference<>(); + Consumer consumer = breach::set; + + AlertingProfileFileTriggerConfiguration config = + AlertingProfileFileTriggerConfiguration.create(true, "trigger", 120, tempDir); + + AlertingSubsystem subsystem = AlertingSubsystem.create(consumer, new TestTimeSource(), config); + + subsystem.updateConfiguration(defaultAlertingConfig()); + + assertThat(breach.get()).isNull(); + } + + @Test + void fileTriggerDoesNotFireWhenFileIsTooOld() throws IOException { + File triggerFile = new File(tempDir, "trigger"); + assertThat(triggerFile.createNewFile()).isTrue(); + + // Use a time source with a known "current" time, and set the file's lastModified to be + // older than the max age relative to that time. + long currentTimeMillis = System.currentTimeMillis(); + long oldLastModified = + currentTimeMillis + - AlertingProfileFileTriggerConfiguration.MANUAL_TRIGGER_FILE_MAX_AGE_MS + - 10_000; + assertThat(triggerFile.setLastModified(oldLastModified)).isTrue(); + + TestTimeSource timeSource = new TestTimeSource(); + timeSource.setNow(Instant.ofEpochMilli(currentTimeMillis)); + + AtomicReference breach = new AtomicReference<>(); + Consumer consumer = breach::set; + + AlertingProfileFileTriggerConfiguration config = + AlertingProfileFileTriggerConfiguration.create(true, "trigger", 120, tempDir); + + AlertingSubsystem subsystem = AlertingSubsystem.create(consumer, timeSource, config); + + subsystem.updateConfiguration(defaultAlertingConfig()); + + assertThat(breach.get()).isNull(); + // file should not be deleted when too old + assertThat(triggerFile.exists()).isTrue(); + } + + @Test + void fileTriggerDeletesFileOnSuccessfulTrigger() throws IOException { + File triggerFile = new File(tempDir, "trigger"); + assertThat(triggerFile.createNewFile()).isTrue(); + + AtomicReference breach = new AtomicReference<>(); + Consumer consumer = breach::set; + + AlertingProfileFileTriggerConfiguration config = + AlertingProfileFileTriggerConfiguration.create(true, "trigger", 120, tempDir); + + AlertingSubsystem subsystem = AlertingSubsystem.create(consumer, new TestTimeSource(), config); + + subsystem.updateConfiguration(defaultAlertingConfig()); + + assertThat(breach.get()).isNotNull(); + assertThat(triggerFile.exists()).isFalse(); + } + + @Test + void fileTriggerUsesDefaultDurationWhenCollectionPlanHasZero() throws IOException { + File triggerFile = new File(tempDir, "trigger"); + assertThat(triggerFile.createNewFile()).isTrue(); + + AtomicReference breach = new AtomicReference<>(); + Consumer consumer = breach::set; + + int expectedDefaultDuration = 90; + AlertingProfileFileTriggerConfiguration config = + AlertingProfileFileTriggerConfiguration.create( + true, "trigger", expectedDefaultDuration, tempDir); + + AlertingSubsystem subsystem = AlertingSubsystem.create(consumer, new TestTimeSource(), config); + + // The default alerting config has immediateProfilingDurationSeconds=0 + subsystem.updateConfiguration(defaultAlertingConfig()); + + assertThat(breach.get()).isNotNull(); + assertThat(breach.get().getAlertConfiguration().getProfileDurationSeconds()) + .isEqualTo(expectedDefaultDuration); + } +} diff --git a/agent/agent-profiler/agent-alerting/src/test/java/com/microsoft/applicationinsights/alerting/AlertingSubsystemTest.java b/agent/agent-profiler/agent-alerting/src/test/java/com/microsoft/applicationinsights/alerting/AlertingSubsystemTest.java index 99881d032e2..eaeedc45bbd 100644 --- a/agent/agent-profiler/agent-alerting/src/test/java/com/microsoft/applicationinsights/alerting/AlertingSubsystemTest.java +++ b/agent/agent-profiler/agent-alerting/src/test/java/com/microsoft/applicationinsights/alerting/AlertingSubsystemTest.java @@ -9,6 +9,7 @@ import com.microsoft.applicationinsights.alerting.config.AlertConfiguration; import com.microsoft.applicationinsights.alerting.config.AlertMetricType; import com.microsoft.applicationinsights.alerting.config.AlertingConfiguration; +import com.microsoft.applicationinsights.alerting.config.AlertingProfileFileTriggerConfiguration; import com.microsoft.applicationinsights.alerting.config.CollectionPlanConfiguration; import com.microsoft.applicationinsights.alerting.config.CollectionPlanConfiguration.EngineMode; import com.microsoft.applicationinsights.alerting.config.DefaultConfiguration; @@ -23,7 +24,9 @@ class AlertingSubsystemTest { private static AlertingSubsystem getAlertMonitor( Consumer consumer, TestTimeSource timeSource) { - AlertingSubsystem monitor = AlertingSubsystem.create(consumer, timeSource); + AlertingSubsystem monitor = + AlertingSubsystem.create( + consumer, timeSource, AlertingProfileFileTriggerConfiguration.createDefault()); monitor.updateConfiguration( AlertingConfiguration.create( @@ -82,7 +85,9 @@ void manualAlertWorks() { Consumer consumer = called::set; TestTimeSource timeSource = new TestTimeSource(); - AlertingSubsystem service = AlertingSubsystem.create(consumer, timeSource); + AlertingSubsystem service = + AlertingSubsystem.create( + consumer, timeSource, AlertingProfileFileTriggerConfiguration.createDefault()); service.updateConfiguration( AlertingConfiguration.create( @@ -123,7 +128,9 @@ void manualAlertDoesNotTriggerAfterExpired() { Consumer consumer = called::set; TestTimeSource timeSource = new TestTimeSource(); - AlertingSubsystem service = AlertingSubsystem.create(consumer, timeSource); + AlertingSubsystem service = + AlertingSubsystem.create( + consumer, timeSource, AlertingProfileFileTriggerConfiguration.createDefault()); service.updateConfiguration( AlertingConfiguration.create( diff --git a/agent/agent-profiler/agent-alerting/src/test/java/com/microsoft/applicationinsights/alerting/TestTimeSource.java b/agent/agent-profiler/agent-alerting/src/test/java/com/microsoft/applicationinsights/alerting/TestTimeSource.java index 90d81746c91..a2d41814605 100644 --- a/agent/agent-profiler/agent-alerting/src/test/java/com/microsoft/applicationinsights/alerting/TestTimeSource.java +++ b/agent/agent-profiler/agent-alerting/src/test/java/com/microsoft/applicationinsights/alerting/TestTimeSource.java @@ -15,6 +15,10 @@ public Instant getNow() { return now; } + void setNow(Instant now) { + this.now = now; + } + void increment(int milliseconds) { this.now = now.plusMillis(milliseconds); } diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/configuration/Configuration.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/configuration/Configuration.java index 26005a97585..722ecee251e 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/configuration/Configuration.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/configuration/Configuration.java @@ -1531,6 +1531,38 @@ public static class ProfilerConfiguration { public boolean enableRequestTriggering = false; public List requestTriggerEndpoints = new ArrayList<>(); @Nullable public String cgroupPath = null; + + /** Configuration for the file-based manual profile trigger mechanism. */ + public ManualProfileTriggerConfiguration manualTrigger = + new ManualProfileTriggerConfiguration(); + + // Global cooldown in seconds applied after any profile recording completes, regardless of + // trigger source. During cooldown, all trigger sources (CPU, memory, request, manual, periodic) + // are suppressed. Set to 0 to disable (individual trigger cooldowns still apply). + public int globalCooldownSeconds = 120; + + // Whether to register a JMX MBean that allows triggering profiles via jcmd or JMX tools. + public boolean enableProfilerControlMBean = false; + } + + /** + * Configuration for the file-based manual profile trigger. + * + *

When enabled, the agent watches for the creation of a trigger file. Touching or creating the + * file initiates an on-demand profile recording. The file is deleted after the trigger is + * processed to prevent repeated recordings. + */ + public static class ManualProfileTriggerConfiguration { + // Whether the file-based manual trigger is enabled. + public boolean enabled = true; + + // Path to the file that triggers a manual profile when created/touched. + // If relative, it is resolved against the agent's temp directory. + public String filePath = "applicationinsights-agent-profile-trigger"; + + // Default duration (in seconds) for profiles initiated by the file trigger when no override + // is specified in the collection plan configuration. + public int defaultProfileDurationSeconds = 120; } public static class GcEventConfiguration { diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/PerformanceMonitoringService.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/PerformanceMonitoringService.java index 8a4860ab594..18964731ce7 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/PerformanceMonitoringService.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/PerformanceMonitoringService.java @@ -15,6 +15,7 @@ import com.microsoft.applicationinsights.agent.internal.telemetry.TelemetryObservers; import com.microsoft.applicationinsights.alerting.AlertingSubsystem; import com.microsoft.applicationinsights.alerting.config.AlertingConfiguration; +import com.microsoft.applicationinsights.alerting.config.AlertingProfileFileTriggerConfiguration; import com.microsoft.applicationinsights.diagnostics.DiagnosticEngine; import com.microsoft.applicationinsights.diagnostics.DiagnosticEngineFactory; import com.microsoft.applicationinsights.diagnostics.appinsights.CodeOptimizerApplicationInsightFactoryJfr; @@ -102,6 +103,14 @@ synchronized void enableProfiler( profiler = new Profiler(configuration, tempDir); + // Build file-trigger configuration, resolving relative paths against the agent's temp directory + AlertingProfileFileTriggerConfiguration alertingProfileFileTriggerConfiguration = + AlertingProfileFileTriggerConfiguration.create( + configuration.manualTrigger.enabled, + configuration.manualTrigger.filePath, + configuration.manualTrigger.defaultProfileDurationSeconds, + tempDir); + alerting = AlertingSubsystemInit.create( configuration, @@ -110,7 +119,8 @@ synchronized void enableProfiler( profiler, telemetryClient, diagnosticEngine, - alertServiceExecutorService); + alertServiceExecutorService, + alertingProfileFileTriggerConfiguration); uploadService = new UploadService( diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/Profiler.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/Profiler.java index 5e877d9f1fb..3474782428b 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/Profiler.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/Profiler.java @@ -8,6 +8,7 @@ import com.microsoft.applicationinsights.agent.internal.profiler.upload.UploadListener; import com.microsoft.applicationinsights.agent.internal.profiler.upload.UploadService; import com.microsoft.applicationinsights.alerting.alert.AlertBreach; +import com.microsoft.applicationinsights.alerting.analysis.TimeSource; import com.microsoft.applicationinsights.alerting.config.AlertConfiguration; import com.microsoft.applicationinsights.alerting.config.AlertMetricType; import io.opentelemetry.contrib.jfr.connection.FlightRecorderConnection; @@ -60,6 +61,15 @@ public class Profiler { @Nullable private Recording activeRecording = null; @Nullable private File activeRecordingFile = null; + // Global cooldown: earliest time at which a new recording is allowed, regardless of trigger type. + // This prevents rapid successive profiles from different trigger sources (e.g., file trigger + // immediately followed by a CPU threshold breach). Reset after each recording completes. + private volatile Instant globalCooldownUntil = Instant.MIN; + + // Duration (in seconds) of the global cooldown period. A value of 0 disables the global cooldown + // (individual per-trigger cooldowns still apply). + private final int globalCooldownSeconds; + private final RecordingConfiguration memoryRecordingConfiguration; private final RecordingConfiguration cpuRecordingConfiguration; private final RecordingConfiguration spanRecordingConfiguration; @@ -67,7 +77,15 @@ public class Profiler { private final File temporaryDirectory; + private final TimeSource timeSource; + public Profiler(Configuration.ProfilerConfiguration config, File tempDir) { + this(config, tempDir, TimeSource.DEFAULT); + } + + public Profiler(Configuration.ProfilerConfiguration config, File tempDir, TimeSource timeSource) { + + this.timeSource = timeSource; periodicConfig = AlertConfiguration.builder() @@ -78,6 +96,8 @@ public Profiler(Configuration.ProfilerConfiguration config, File tempDir) { .setCooldownSeconds(config.periodicRecordingIntervalSeconds) .build(); + globalCooldownSeconds = config.globalCooldownSeconds; + memoryRecordingConfiguration = AlternativeJfrConfigurations.getMemoryProfileConfig(config); cpuRecordingConfiguration = AlternativeJfrConfigurations.getCpuProfileConfig(config); spanRecordingConfiguration = AlternativeJfrConfigurations.getSpanProfileConfig(config); @@ -109,6 +129,17 @@ public void initialize( } } + // visible for testing + void initialize( + UploadService uploadService, + ScheduledExecutorService scheduledExecutorService, + FlightRecorderConnection flightRecorderConnection) { + this.uploadService = uploadService; + this.scheduledExecutorService = scheduledExecutorService; + this.recordingOptionsBuilder = new RecordingOptions.Builder(); + this.flightRecorderConnection = flightRecorderConnection; + } + /** Apply new configuration settings obtained from Service Profiler. */ public void updateConfiguration(ProfilerConfiguration newConfig) { logger.debug("Received config {}", newConfig.getLastModified()); @@ -118,7 +149,7 @@ public void updateConfiguration(ProfilerConfiguration newConfig) { // visible for tests void profileAndUpload(AlertBreach alertBreach, Duration duration, UploadListener uploadListener) { - Instant recordingStart = Instant.now(); + Instant recordingStart = timeSource.getNow(); executeProfile( alertBreach.getType(), duration, @@ -133,6 +164,15 @@ private Recording startRecording(AlertMetricType alertType, Duration duration) { return null; } + // Enforce global cooldown across all trigger sources + if (globalCooldownSeconds > 0 && timeSource.getNow().isBefore(globalCooldownUntil)) { + logger.info( + "Alert received (type={}), but global cooldown is active until {}. Ignoring request.", + alertType, + globalCooldownUntil); + return null; + } + RecordingConfiguration recordingConfiguration; switch (alertType) { case REQUEST: @@ -278,10 +318,17 @@ private static void writeFileFromStream(Recording recording, File recordingFile) } } - private void clearActiveRecording() { + // visible for testing + void clearActiveRecording() { synchronized (activeRecordingLock) { activeRecording = null; + // Start global cooldown now that the recording is complete + if (globalCooldownSeconds > 0) { + globalCooldownUntil = timeSource.getNow().plusSeconds(globalCooldownSeconds); + logger.debug("Global profile cooldown active until {}", globalCooldownUntil); + } + // delete uploaded profile if (activeRecordingFile != null && activeRecordingFile.exists()) { if (!activeRecordingFile.delete()) { @@ -292,6 +339,18 @@ private void clearActiveRecording() { } } + // visible for testing + Instant getGlobalCooldownUntil() { + return globalCooldownUntil; + } + + // visible for testing + boolean isRecordingActive() { + synchronized (activeRecordingLock) { + return activeRecording != null; + } + } + /** Dump JFR profile to file. */ // visible for testing protected File createJfrFile(Duration duration) throws IOException { @@ -302,7 +361,7 @@ protected File createJfrFile(Duration duration) throws IOException { } } - Instant recordingStart = Instant.now(); + Instant recordingStart = timeSource.getNow(); Instant recordingEnd = recordingStart.plus(duration); return new File( diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/ProfilerControl.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/ProfilerControl.java new file mode 100644 index 00000000000..ed936a6b822 --- /dev/null +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/ProfilerControl.java @@ -0,0 +1,111 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.microsoft.applicationinsights.agent.internal.profiler; + +import com.microsoft.applicationinsights.alerting.alert.AlertBreach; +import com.microsoft.applicationinsights.alerting.config.AlertConfiguration; +import com.microsoft.applicationinsights.alerting.config.AlertMetricType; +import java.lang.management.ManagementFactory; +import java.util.UUID; +import java.util.function.Consumer; +import javax.management.InstanceAlreadyExistsException; +import javax.management.MBeanServer; +import javax.management.ObjectName; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * JMX MBean that exposes profile triggering via JMX tools. + * + *

Usage via jcmd (JDK 17+): + * + * 

+ *   jcmd <pid> MBean.invoke com.microsoft:type=AI-alert,name=ProfilerControl triggerProfile
+ *
+ * + *

Usage via jmxterm (any JDK): + * + * 

+ *   echo "run -b com.microsoft:type=AI-alert,name=ProfilerControl triggerProfile" | \
+ *     java -jar jmxterm.jar -l <pid>
+ *
+ */ +public class ProfilerControl implements ProfilerControlMBean { + + private static final Logger logger = LoggerFactory.getLogger(ProfilerControl.class); + + private static final String OBJECT_NAME = "com.microsoft:type=AI-alert,name=ProfilerControl"; + private static final int DEFAULT_DURATION_SECONDS = 120; + + private final Consumer alertHandler; + + /** + * Creates a new ProfilerControl instance. + * + * @param alertHandler consumer that processes the generated {@link AlertBreach}, typically wired + * to the profiler's recording logic + */ + ProfilerControl(Consumer alertHandler) { + this.alertHandler = alertHandler; + } + + @Override + public String triggerProfile() { + return triggerProfile(DEFAULT_DURATION_SECONDS); + } + + @Override + public String triggerProfile(int durationSeconds) { + if (durationSeconds <= 0) { + return "Error: duration must be positive, got " + durationSeconds; + } + + logger.info("Manual profile triggered via JMX, duration={}s", durationSeconds); + + AlertBreach alertBreach = + AlertBreach.builder() + .setType(AlertMetricType.MANUAL) + .setAlertValue(0.0) + .setAlertConfiguration( + AlertConfiguration.builder() + .setType(AlertMetricType.MANUAL) + .setEnabled(true) + .setProfileDurationSeconds(durationSeconds) + .build()) + .setProfileId(UUID.randomUUID().toString()) + .setCpuMetric(0) + .setMemoryUsage(0) + .build(); + + alertHandler.accept(alertBreach); + return "Profile recording started (duration=" + + durationSeconds + + "s, id=" + + alertBreach.getProfileId() + + ")"; + } + + /** + * Registers this MBean with the platform MBean server. Call during profiler initialization. + * + * @param alertHandler the alert handler (typically wired to Profiler.accept) + */ + public static void register(Consumer alertHandler) { + try { + MBeanServer beanServer = ManagementFactory.getPlatformMBeanServer(); + ObjectName objectName = new ObjectName(OBJECT_NAME); + ProfilerControl bean = new ProfilerControl(alertHandler); + beanServer.registerMBean(bean, objectName); + logger.info( + "Registered profiler control MBean: {}. " + + "Trigger profiles with: jcmd MBean.invoke {} triggerProfile", + OBJECT_NAME, + OBJECT_NAME); + } catch (InstanceAlreadyExistsException e) { + logger.debug("Profiler control MBean already registered"); + } catch (Exception e) { + logger.warn("Failed to register profiler control MBean", e); + } + } +} diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/ProfilerControlMBean.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/ProfilerControlMBean.java new file mode 100644 index 00000000000..afb456c525a --- /dev/null +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/ProfilerControlMBean.java @@ -0,0 +1,34 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.microsoft.applicationinsights.agent.internal.profiler; + +/** + * MBean interface for triggering Application Insights profiles via JMX. + * + *

Can be invoked via jcmd (JDK 17+): + * + * 

+ *   jcmd <pid> MBean.invoke com.microsoft:type=AI-alert,name=ProfilerControl triggerProfile
+ *   jcmd <pid> MBean.invoke com.microsoft:type=AI-alert,name=ProfilerControl triggerProfile 120
+ *
+ * + *

Or via jmxterm / JConsole on any JDK version. + */ +public interface ProfilerControlMBean { + + /** + * Trigger a manual profile with the default duration (120 seconds). + * + * @return a status message indicating whether the profile was started + */ + String triggerProfile(); + + /** + * Trigger a manual profile with the specified duration. + * + * @param durationSeconds the desired recording duration in seconds; must be positive + * @return a status message indicating whether the profile was started + */ + String triggerProfile(int durationSeconds); +} diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/triggers/AlertingSubsystemInit.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/triggers/AlertingSubsystemInit.java index 51f696750f9..dd5533e6472 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/triggers/AlertingSubsystemInit.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/triggers/AlertingSubsystemInit.java @@ -14,6 +14,7 @@ import com.microsoft.applicationinsights.agent.internal.configuration.Configuration; import com.microsoft.applicationinsights.agent.internal.configuration.GcReportingLevel; import com.microsoft.applicationinsights.agent.internal.profiler.Profiler; +import com.microsoft.applicationinsights.agent.internal.profiler.ProfilerControl; import com.microsoft.applicationinsights.agent.internal.profiler.upload.ServiceProfilerIndex; import com.microsoft.applicationinsights.agent.internal.telemetry.TelemetryClient; import com.microsoft.applicationinsights.agent.internal.telemetry.TelemetryObservers; @@ -23,6 +24,7 @@ import com.microsoft.applicationinsights.alerting.analysis.pipelines.AlertPipeline; import com.microsoft.applicationinsights.alerting.analysis.pipelines.AlertPipelineMultiplexer; import com.microsoft.applicationinsights.alerting.config.AlertMetricType; +import com.microsoft.applicationinsights.alerting.config.AlertingProfileFileTriggerConfiguration; import com.microsoft.applicationinsights.diagnostics.DiagnosticEngine; import java.util.List; import java.util.Map; @@ -47,14 +49,17 @@ public static AlertingSubsystem create( Profiler profiler, TelemetryClient telemetryClient, DiagnosticEngine diagnosticEngine, - ExecutorService executorService) { + ExecutorService executorService, + AlertingProfileFileTriggerConfiguration alertingProfileFileTriggerConfiguration) { // TODO (trask) delay creation of AlertingSubsystem until after Profiler is created and // initialized? Consumer alertAction = alert -> alertAction(alert, profiler, diagnosticEngine, telemetryClient); - alertingSubsystem = AlertingSubsystem.create(alertAction, TimeSource.DEFAULT); + alertingSubsystem = + AlertingSubsystem.create( + alertAction, TimeSource.DEFAULT, alertingProfileFileTriggerConfiguration); if (configuration.enableRequestTriggering) { if (!configuration.requestTriggerEndpoints.isEmpty()) { @@ -80,6 +85,11 @@ public static AlertingSubsystem create( executorService, fromGcEventMonitorConfiguration(reportingLevel)); + // Register JMX MBean for triggering profiles via jcmd / JMX tools + if (configuration.enableProfilerControlMBean) { + ProfilerControl.register(alertAction); + } + return alertingSubsystem; } diff --git a/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/profiler/ProfilerControlTest.java b/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/profiler/ProfilerControlTest.java new file mode 100644 index 00000000000..229affa8ae8 --- /dev/null +++ b/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/profiler/ProfilerControlTest.java @@ -0,0 +1,98 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.microsoft.applicationinsights.agent.internal.profiler; + +import static org.assertj.core.api.Assertions.assertThat; + +import com.microsoft.applicationinsights.alerting.alert.AlertBreach; +import com.microsoft.applicationinsights.alerting.config.AlertMetricType; +import java.lang.management.ManagementFactory; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; +import javax.management.MBeanServer; +import javax.management.ObjectName; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; + +class ProfilerControlTest { + + private static final String OBJECT_NAME = "com.microsoft:type=AI-alert,name=ProfilerControl"; + + @AfterEach + void cleanup() throws Exception { + // Unregister MBean if it was registered during the test + MBeanServer beanServer = ManagementFactory.getPlatformMBeanServer(); + ObjectName objectName = new ObjectName(OBJECT_NAME); + if (beanServer.isRegistered(objectName)) { + beanServer.unregisterMBean(objectName); + } + } + + @Test + void triggerProfileWithDefaultDuration() { + AtomicReference received = new AtomicReference<>(); + Consumer handler = received::set; + + ProfilerControl control = new ProfilerControl(handler); + String result = control.triggerProfile(); + + assertThat(result).startsWith("Profile recording started"); + assertThat(result).contains("duration=120s"); + + AlertBreach breach = received.get(); + assertThat(breach).isNotNull(); + assertThat(breach.getType()).isEqualTo(AlertMetricType.MANUAL); + assertThat(breach.getAlertConfiguration().getProfileDurationSeconds()).isEqualTo(120); + assertThat(breach.getProfileId()).isNotNull(); + } + + @Test + void registerCreatesAccessibleMBean() throws Exception { + AtomicReference received = new AtomicReference<>(); + Consumer handler = received::set; + + ProfilerControl.register(handler); + + MBeanServer beanServer = ManagementFactory.getPlatformMBeanServer(); + ObjectName objectName = new ObjectName(OBJECT_NAME); + + assertThat(beanServer.isRegistered(objectName)).isTrue(); + + // Invoke triggerProfile via JMX + Object result = beanServer.invoke(objectName, "triggerProfile", null, null); + + assertThat(result).isInstanceOf(String.class); + assertThat((String) result).contains("duration=120s"); + assertThat(received.get()).isNotNull(); + assertThat(received.get().getType()).isEqualTo(AlertMetricType.MANUAL); + } + + @Test + void registerWithCustomDurationViaMBean() throws Exception { + AtomicReference received = new AtomicReference<>(); + Consumer handler = received::set; + + ProfilerControl.register(handler); + + MBeanServer beanServer = ManagementFactory.getPlatformMBeanServer(); + ObjectName objectName = new ObjectName(OBJECT_NAME); + + Object result = + beanServer.invoke(objectName, "triggerProfile", new Object[] {45}, new String[] {"int"}); + + assertThat(result).isInstanceOf(String.class); + assertThat((String) result).contains("duration=45s"); + assertThat(received.get()).isNotNull(); + assertThat(received.get().getAlertConfiguration().getProfileDurationSeconds()).isEqualTo(45); + } + + @Test + void registerDoesNotThrowOnDoubleRegistration() { + Consumer handler = breach -> {}; + + ProfilerControl.register(handler); + // Should not throw + ProfilerControl.register(handler); + } +} diff --git a/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/profiler/ProfilerGlobalCooldownTest.java b/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/profiler/ProfilerGlobalCooldownTest.java new file mode 100644 index 00000000000..d034fe93618 --- /dev/null +++ b/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/profiler/ProfilerGlobalCooldownTest.java @@ -0,0 +1,116 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.microsoft.applicationinsights.agent.internal.profiler; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import com.microsoft.applicationinsights.agent.internal.configuration.Configuration; +import com.microsoft.applicationinsights.agent.internal.profiler.testutil.TestTimeSource; +import com.microsoft.applicationinsights.agent.internal.profiler.upload.UploadListener; +import com.microsoft.applicationinsights.agent.internal.profiler.upload.UploadService; +import com.microsoft.applicationinsights.alerting.alert.AlertBreach; +import com.microsoft.applicationinsights.alerting.config.AlertConfiguration; +import com.microsoft.applicationinsights.alerting.config.AlertMetricType; +import io.opentelemetry.contrib.jfr.connection.FlightRecorderConnection; +import io.opentelemetry.contrib.jfr.connection.Recording; +import io.opentelemetry.contrib.jfr.connection.RecordingConfiguration; +import io.opentelemetry.contrib.jfr.connection.RecordingOptions; +import java.io.File; +import java.time.Duration; +import java.time.Instant; +import java.util.UUID; +import java.util.concurrent.Executors; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class ProfilerGlobalCooldownTest { + @TempDir File tempDir; + + private final TestTimeSource timeSource = new TestTimeSource(); + + private static AlertBreach manualBreach(int durationSeconds) { + return AlertBreach.builder() + .setType(AlertMetricType.MANUAL) + .setAlertValue(0.0) + .setAlertConfiguration( + AlertConfiguration.builder() + .setType(AlertMetricType.MANUAL) + .setEnabled(true) + .setProfileDurationSeconds(durationSeconds) + .build()) + .setProfileId(UUID.randomUUID().toString()) + .setCpuMetric(0) + .setMemoryUsage(0) + .build(); + } + + private Profiler createProfiler(int globalCooldownSeconds) { + Configuration.ProfilerConfiguration config = new Configuration.ProfilerConfiguration(); + config.globalCooldownSeconds = globalCooldownSeconds; + Profiler profiler = + new Profiler(config, tempDir, timeSource) { + @Override + protected Recording createRecording(RecordingOptions opts, RecordingConfiguration cfg) { + return mock(Recording.class); + } + }; + + FlightRecorderConnection frc = mock(FlightRecorderConnection.class); + when(frc.newRecording(any(), any())).thenReturn(mock(Recording.class)); + + profiler.initialize(mock(UploadService.class), Executors.newScheduledThreadPool(1), frc); + return profiler; + } + + @Test + void globalCooldownIsSetAfterRecordingCompletes() { + Instant baseTime = Instant.parse("2025-01-01T00:00:00Z"); + timeSource.setNow(baseTime); + + Profiler profiler = createProfiler(120); + UploadListener noOpListener = index -> {}; + profiler.profileAndUpload(manualBreach(1), Duration.ofSeconds(1), noOpListener); + // Before clearing, cooldown should still be at MIN + assertThat(profiler.getGlobalCooldownUntil()).isEqualTo(Instant.MIN); + profiler.clearActiveRecording(); + // After clearing, cooldown should be exactly baseTime + 120s + assertThat(profiler.getGlobalCooldownUntil()).isEqualTo(baseTime.plusSeconds(120)); + } + + @Test + void globalCooldownNotSetWhenDisabled() { + timeSource.setNow(Instant.parse("2025-01-01T00:00:00Z")); + + Profiler profiler = createProfiler(0); + UploadListener noOpListener = index -> {}; + profiler.profileAndUpload(manualBreach(1), Duration.ofSeconds(1), noOpListener); + profiler.clearActiveRecording(); + assertThat(profiler.getGlobalCooldownUntil()).isEqualTo(Instant.MIN); + } + + @Test + void secondProfileRejectedDuringCooldown() { + Instant baseTime = Instant.parse("2025-01-01T00:00:00Z"); + timeSource.setNow(baseTime); + + Profiler profiler = createProfiler(600); + UploadListener noOpListener = index -> {}; + // First profile starts and completes + profiler.profileAndUpload(manualBreach(1), Duration.ofSeconds(1), noOpListener); + profiler.clearActiveRecording(); + // Cooldown should now be active (baseTime + 600s) + assertThat(profiler.getGlobalCooldownUntil()).isEqualTo(baseTime.plusSeconds(600)); + + // Advance time but stay within cooldown window + timeSource.setNow(baseTime.plusSeconds(300)); + + // Second profile should be silently rejected (startRecording returns null due to cooldown) + profiler.profileAndUpload(manualBreach(1), Duration.ofSeconds(1), noOpListener); + // activeRecording should still be null (second profile was rejected) + assertThat(profiler.isRecordingActive()).isFalse(); + } +} diff --git a/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/profiler/triggers/AlertTriggerSpanProcessorTest.java b/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/profiler/triggers/AlertTriggerSpanProcessorTest.java index 62747ce97a1..06004846bd5 100644 --- a/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/profiler/triggers/AlertTriggerSpanProcessorTest.java +++ b/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/profiler/triggers/AlertTriggerSpanProcessorTest.java @@ -10,6 +10,7 @@ import com.microsoft.applicationinsights.alerting.analysis.TimeSource; import com.microsoft.applicationinsights.alerting.analysis.pipelines.AlertPipelineMultiplexer; import com.microsoft.applicationinsights.alerting.config.AlertMetricType; +import com.microsoft.applicationinsights.alerting.config.AlertingProfileFileTriggerConfiguration; import io.opentelemetry.api.trace.Span; import io.opentelemetry.api.trace.SpanKind; import io.opentelemetry.api.trace.StatusCode; @@ -121,7 +122,11 @@ private static void run(Handle handle) throws InterruptedException { triggerConfig.filter.value = "foo.*"; triggerConfig.threshold.value = 0.75f; - AlertingSubsystem alertingSubsystem = AlertingSubsystem.create(alertAction, TimeSource.DEFAULT); + AlertingSubsystem alertingSubsystem = + AlertingSubsystem.create( + alertAction, + TimeSource.DEFAULT, + AlertingProfileFileTriggerConfiguration.createDefault()); TestTimeSource timeSource = new TestTimeSource(); timeSource.setNow(Instant.EPOCH); diff --git a/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/profiler/triggers/GcEventInitTest.java b/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/profiler/triggers/GcEventInitTest.java index 667d1ce7418..5b1257d5f16 100644 --- a/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/profiler/triggers/GcEventInitTest.java +++ b/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/profiler/triggers/GcEventInitTest.java @@ -12,6 +12,7 @@ import com.microsoft.applicationinsights.alerting.alert.AlertBreach; import com.microsoft.applicationinsights.alerting.analysis.TimeSource; import com.microsoft.applicationinsights.alerting.config.AlertingConfiguration; +import com.microsoft.applicationinsights.alerting.config.AlertingProfileFileTriggerConfiguration; import com.microsoft.gcmonitor.GcCollectionEvent; import com.microsoft.gcmonitor.GcEventConsumer; import com.microsoft.gcmonitor.GcMonitorFactory; @@ -73,7 +74,10 @@ public MemoryManagement monitor( private static AlertingSubsystem getAlertingSubsystem( CompletableFuture alertFuture, TimeSource timeSource) { AlertingSubsystem alertingSubsystem = - AlertingSubsystem.create(alertFuture::complete, timeSource); + AlertingSubsystem.create( + alertFuture::complete, + timeSource, + AlertingProfileFileTriggerConfiguration.createDefault()); AlertingConfiguration config = AlertConfigParser.parse( diff --git a/docs/README.md b/docs/README.md index 8fb8eddc3c2..714c6bf4ec1 100644 --- a/docs/README.md +++ b/docs/README.md @@ -92,7 +92,15 @@ Additionally, a number of parameters can be configured using environment variabl "profiler": { "enabled": true, "cpuTriggeredSettings": "profile-without-env-data", - "memoryTriggeredSettings": "profile-without-env-data" + "memoryTriggeredSettings": "profile-without-env-data", + "manualTriggeredSettings": "profile-without-env-data", + "globalCooldownSeconds": 120, + "enableProfilerControlMBean": false, + "manualTrigger": { + "enabled": true, + "filePath": "applicationinsights-agent-profile-trigger", + "defaultProfileDurationSeconds": 120 + } } } } @@ -114,3 +122,61 @@ This can be one of: [Warning](#Warning) section for details. - `profile`. Uses the `profile.jfc` jfc configuration that ships with JFR. - A path to a custom jfc configuration file on the file system, i.e `/tmp/myconfig.jfc`. + +`manualTriggeredSettings` - This configuration will be used for manually triggered profiles (via +file trigger or JMX MBean). Accepts the same values as `cpuTriggeredSettings`. + +`globalCooldownSeconds` - (default: 120) Cooldown period in seconds applied after any profile +recording completes, regardless of trigger source. During cooldown, all trigger sources (CPU, +memory, request, manual, periodic) are suppressed. Set to `0` to disable (individual trigger +cooldowns still apply). + +`enableProfilerControlMBean` - (default: false) Whether to register a JMX MBean +(`com.microsoft:type=AI-alert,name=ProfilerControl`) that allows triggering profiles via +`jcmd` or JMX tools. See [Manual Profile Triggering](#manual-profile-triggering) for usage. + +`manualTrigger` - Configuration for the file-based manual profile trigger: + +- `enabled` - (default: true) Whether the file-based manual trigger is enabled. +- `filePath` - (default: `applicationinsights-agent-profile-trigger`) Path to the trigger file. + If relative, it is resolved against the agent's temp directory. Creating or touching this file + triggers a profile recording. +- `defaultProfileDurationSeconds` - (default: 120) Duration in seconds for profiles initiated by + the file trigger when no override is specified in the collection plan. + +## Manual Profile Triggering + +In addition to automatic threshold-based triggers, profiles can be initiated manually using either +the file-based trigger or the JMX MBean. + +### File-based trigger + +When `manualTrigger.enabled` is `true` (the default), you can trigger a profile by creating or +touching the trigger file: + +```bash +touch /tmp/applicationinsights-agent-profile-trigger +``` + +The file must have been modified within the last 60 seconds to be considered valid (stale files +are ignored to prevent unintended recordings after restarts). After the trigger is detected, the +file is automatically deleted. + +### JMX MBean trigger + +When `enableProfilerControlMBean` is `true`, the agent registers a JMX MBean that can be invoked +to trigger profiles: + +**Via jcmd (JDK 17+):** +```bash +jcmd MBean.invoke com.microsoft:type=AI-alert,name=ProfilerControl triggerProfile +``` + +**Via jmxterm (any JDK):** +```bash +echo "run -b com.microsoft:type=AI-alert,name=ProfilerControl triggerProfile" | \ + java -jar jmxterm.jar -l +``` + +Both manual triggering mechanisms respect the `globalCooldownSeconds` setting — if a profile was +recently recorded, manual triggers will be suppressed until the cooldown expires. From 6e3854adf4038c641222107607efc60fe5c920a4 Mon Sep 17 00:00:00 2001 From: John Oliver <1615532+johnoliver@users.noreply.github.com> Date: Wed, 27 May 2026 18:49:58 +0000 Subject: [PATCH 2/2] Fix manual evaluation cycle --- .../alerting/AlertingSubsystem.java | 4 ++-- agent/agent-profiler/request-triggers.md | 5 +++++ .../internal/configuration/Configuration.java | 2 +- .../profiler/PerformanceMonitoringService.java | 6 ++++++ .../agent/internal/profiler/Profiler.java | 2 ++ .../internal/profiler/ProfilerControl.java | 12 ++++-------- .../profiler/ProfilerControlMBean.java | 11 ++++------- .../profiler/ProfilingInitializer.java | 12 +++++++++++- .../triggers/AlertingSubsystemInit.java | 2 +- docs/README.md | 18 +++++++++++------- 10 files changed, 47 insertions(+), 27 deletions(-) diff --git a/agent/agent-profiler/agent-alerting/src/main/java/com/microsoft/applicationinsights/alerting/AlertingSubsystem.java b/agent/agent-profiler/agent-alerting/src/main/java/com/microsoft/applicationinsights/alerting/AlertingSubsystem.java index e49f09af028..e5962294cab 100644 --- a/agent/agent-profiler/agent-alerting/src/main/java/com/microsoft/applicationinsights/alerting/AlertingSubsystem.java +++ b/agent/agent-profiler/agent-alerting/src/main/java/com/microsoft/applicationinsights/alerting/AlertingSubsystem.java @@ -167,7 +167,7 @@ private void updateRequestPipelineConfig( */ private void evaluateManualTrigger(AlertingConfiguration alertConfig) { evaluateCollectionPlanTrigger(alertConfig); - evaluateFileTrigger(alertConfig); + evaluateFileTrigger(); } /** Check if the collection plan configuration requests a manual profile. */ @@ -206,7 +206,7 @@ private void evaluateCollectionPlanTrigger(AlertingConfiguration alertConfig) { * delete the file and trigger a manual profile. The global cooldown in Profiler prevents * overlapping profiles. */ - private void evaluateFileTrigger(AlertingConfiguration alertConfig) { + public void evaluateFileTrigger() { if (!alertingProfileFileTriggerConfiguration.isEnabled()) { return; } diff --git a/agent/agent-profiler/request-triggers.md b/agent/agent-profiler/request-triggers.md index 6a2a0876aee..538b84912aa 100644 --- a/agent/agent-profiler/request-triggers.md +++ b/agent/agent-profiler/request-triggers.md @@ -140,6 +140,11 @@ Currently, we support: - `type` - Currently supports `fixed-duration-cooldown` - `value` - Time in seconds during which a profile will not be triggered +> **Note:** In addition to per-trigger throttling, a global cooldown (`globalCooldownSeconds`, +> default 120s) is applied after any profile recording completes — regardless of trigger source. +> During global cooldown, all triggers (CPU, memory, request, manual, periodic) are suppressed. +> See [Profiler Configuration](../../docs/README.md#configuration-file) for details. + ```json { "throttling": { diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/configuration/Configuration.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/configuration/Configuration.java index 722ecee251e..cb3deb0df68 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/configuration/Configuration.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/configuration/Configuration.java @@ -1541,7 +1541,7 @@ public static class ProfilerConfiguration { // are suppressed. Set to 0 to disable (individual trigger cooldowns still apply). public int globalCooldownSeconds = 120; - // Whether to register a JMX MBean that allows triggering profiles via jcmd or JMX tools. + // Whether to register a JMX MBean that allows triggering profiles via JMX tools. public boolean enableProfilerControlMBean = false; } diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/PerformanceMonitoringService.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/PerformanceMonitoringService.java index 18964731ce7..d8af15d22db 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/PerformanceMonitoringService.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/PerformanceMonitoringService.java @@ -201,4 +201,10 @@ public void updateConfiguration(AlertingConfiguration alertingConfig) { alerting.updateConfiguration(alertingConfig); } } + + public void evaluateFileTrigger() { + if (alerting != null) { + alerting.evaluateFileTrigger(); + } + } } diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/Profiler.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/Profiler.java index 3474782428b..6ad09d39bbb 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/Profiler.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/Profiler.java @@ -40,6 +40,8 @@ *

    *
  • Instantiates FlightRecorder subsystem *
  • Creates profiles on demand + *
  • Enforces a global cooldown between recordings to prevent rapid successive profiles from + * different trigger sources (CPU, memory, request, manual, periodic) *
*/ public class Profiler { diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/ProfilerControl.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/ProfilerControl.java index ed936a6b822..2f871266913 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/ProfilerControl.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/ProfilerControl.java @@ -18,18 +18,15 @@ /** * JMX MBean that exposes profile triggering via JMX tools. * - *

Usage via jcmd (JDK 17+): - * - * 

- *   jcmd <pid> MBean.invoke com.microsoft:type=AI-alert,name=ProfilerControl triggerProfile
- *
- * *

Usage via jmxterm (any JDK): * * 

  *   echo "run -b com.microsoft:type=AI-alert,name=ProfilerControl triggerProfile" | \
  *     java -jar jmxterm.jar -l <pid>
  *
+ * + *

Or connect with JConsole and invoke {@code triggerProfile()} on the {@code + * com.microsoft:type=AI-alert,name=ProfilerControl} MBean. */ public class ProfilerControl implements ProfilerControlMBean { @@ -99,8 +96,7 @@ public static void register(Consumer alertHandler) { beanServer.registerMBean(bean, objectName); logger.info( "Registered profiler control MBean: {}. " - + "Trigger profiles with: jcmd MBean.invoke {} triggerProfile", - OBJECT_NAME, + + "Trigger profiles via JMX tools (e.g. jmxterm or JConsole).", OBJECT_NAME); } catch (InstanceAlreadyExistsException e) { logger.debug("Profiler control MBean already registered"); diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/ProfilerControlMBean.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/ProfilerControlMBean.java index afb456c525a..7e807164d2a 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/ProfilerControlMBean.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/ProfilerControlMBean.java @@ -6,14 +6,11 @@ /** * MBean interface for triggering Application Insights profiles via JMX. * - *

Can be invoked via jcmd (JDK 17+): + *

Can be invoked via jmxterm, JConsole, or any JMX-compatible tool. The MBean is registered + * under {@code com.microsoft:type=AI-alert,name=ProfilerControl} when {@code + * enableProfilerControlMBean} is set to {@code true} in the profiler configuration. * - * 

- *   jcmd <pid> MBean.invoke com.microsoft:type=AI-alert,name=ProfilerControl triggerProfile
- *   jcmd <pid> MBean.invoke com.microsoft:type=AI-alert,name=ProfilerControl triggerProfile 120
- *
- * - *

Or via jmxterm / JConsole on any JDK version. + * @see ProfilerControl */ public interface ProfilerControlMBean { diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/ProfilingInitializer.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/ProfilingInitializer.java index b6fac1be160..1c500dce34b 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/ProfilingInitializer.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/ProfilingInitializer.java @@ -157,12 +157,22 @@ private void startPollingForConfigUpdates() { private void pullProfilerSettings(ConfigService configService) { try { - configService.pullSettings().subscribe(this::applyConfiguration, this::logProfilerPullError); + configService.pullSettings() + .doFinally(result -> evaluateFileTrigger()) + .subscribe(this::applyConfiguration, this::logProfilerPullError); } catch (Throwable t) { logProfilerPullError(t); } } + private void evaluateFileTrigger() { + if (currentlyEnabled.get()) { + if (performanceMonitoringService != null) { + performanceMonitoringService.evaluateFileTrigger(); + } + } + } + private void logProfilerPullError(Throwable e) { if (currentlyEnabled.get()) { logger.error("Error pulling service profiler settings", e); diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/triggers/AlertingSubsystemInit.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/triggers/AlertingSubsystemInit.java index dd5533e6472..b7c5b3a5b85 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/triggers/AlertingSubsystemInit.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/triggers/AlertingSubsystemInit.java @@ -85,7 +85,7 @@ public static AlertingSubsystem create( executorService, fromGcEventMonitorConfiguration(reportingLevel)); - // Register JMX MBean for triggering profiles via jcmd / JMX tools + // Register JMX MBean for triggering profiles via JMX tools (e.g. jmxterm, JConsole) if (configuration.enableProfilerControlMBean) { ProfilerControl.register(alertAction); } diff --git a/docs/README.md b/docs/README.md index 714c6bf4ec1..4df538708a3 100644 --- a/docs/README.md +++ b/docs/README.md @@ -133,7 +133,7 @@ cooldowns still apply). `enableProfilerControlMBean` - (default: false) Whether to register a JMX MBean (`com.microsoft:type=AI-alert,name=ProfilerControl`) that allows triggering profiles via -`jcmd` or JMX tools. See [Manual Profile Triggering](#manual-profile-triggering) for usage. +JMX tools. See [Manual Profile Triggering](#manual-profile-triggering) for usage. `manualTrigger` - Configuration for the file-based manual profile trigger: @@ -162,21 +162,25 @@ The file must have been modified within the last 60 seconds to be considered val are ignored to prevent unintended recordings after restarts). After the trigger is detected, the file is automatically deleted. +> **Note:** The file trigger is evaluated on the profiler's configuration polling cycle +> (default every 60 seconds), so there may be up to a one-minute delay between touching the file +> and the profile recording starting. + ### JMX MBean trigger When `enableProfilerControlMBean` is `true`, the agent registers a JMX MBean that can be invoked to trigger profiles: -**Via jcmd (JDK 17+):** -```bash -jcmd MBean.invoke com.microsoft:type=AI-alert,name=ProfilerControl triggerProfile -``` - -**Via jmxterm (any JDK):** +**Via jmxterm:** ```bash echo "run -b com.microsoft:type=AI-alert,name=ProfilerControl triggerProfile" | \ java -jar jmxterm.jar -l ``` +**Via JConsole:** + +Connect to the target JVM process, navigate to the MBeans tab, expand +`com.microsoft` → `AI-alert` → `ProfilerControl`, and invoke the `triggerProfile` operation. + Both manual triggering mechanisms respect the `globalCooldownSeconds` setting — if a profile was recently recorded, manual triggers will be suppressed until the cooldown expires.