diff --git a/tools/grafanactl/cmd/modify/cmd.go b/tools/grafanactl/cmd/modify/cmd.go index 3583bd3..5b952f9 100644 --- a/tools/grafanactl/cmd/modify/cmd.go +++ b/tools/grafanactl/cmd/modify/cmd.go @@ -16,6 +16,7 @@ package modify import ( "context" + "errors" "fmt" "strings" @@ -84,25 +85,135 @@ func (opts *RawAddDatasourceOptions) Run(ctx context.Context) error { return completed.Run(ctx) } -func (o *CompletedAddDatasourceOptions) getMatchingWorkspaceIDs(ctx context.Context, logger logr.Logger) (set.Set[string], error) { +func isTerminalFailureState(state armmonitor.ProvisioningState) bool { + switch state { + case armmonitor.ProvisioningStateFailed, armmonitor.ProvisioningStateCanceled: + return true + default: + return false + } +} + +func getMatchingWorkspaceIDs(workspaces []armmonitor.AzureMonitorWorkspaceResource, logger logr.Logger) set.Set[string] { validWorkspaceIDs := set.New[string]() - monitorWorkspaces, err := o.MonitorWorkspaceClient.GetAllMonitorWorkspaces(ctx) + for _, workspace := range workspaces { + if workspace.Properties == nil || workspace.Properties.ProvisioningState == nil || workspace.ID == nil { + continue + } + state := *workspace.Properties.ProvisioningState + if isTerminalFailureState(state) { + logger.Info("Skipping workspace in terminal failure state", "workspace-id", *workspace.ID, "provisioning-state", state) + continue + } + logger.Info("Found", "workspace-id", *workspace.ID, "provisioning-state", state) + validWorkspaceIDs.Insert(strings.ToLower(*workspace.ID)) + } + + return validWorkspaceIDs +} + +func getActiveWorkspaceNames(workspaces []armmonitor.AzureMonitorWorkspaceResource) set.Set[string] { + names := set.New[string]() + + for _, workspace := range workspaces { + if workspace.Name == nil { + continue + } + names.Insert(strings.ToLower(*workspace.Name)) + } + + return names +} + +func getWorkspaceEndpoints(workspaces []armmonitor.AzureMonitorWorkspaceResource, logger logr.Logger) map[string]string { + endpoints := make(map[string]string) + + for _, workspace := range workspaces { + if workspace.Name == nil || workspace.Properties == nil || + workspace.Properties.ProvisioningState == nil || + workspace.Properties.Metrics == nil || + workspace.Properties.Metrics.PrometheusQueryEndpoint == nil { + continue + } + if isTerminalFailureState(*workspace.Properties.ProvisioningState) { + continue + } + name := strings.ToLower(*workspace.Name) + endpoints[name] = *workspace.Properties.Metrics.PrometheusQueryEndpoint + logger.Info("Found workspace endpoint", "workspace-name", *workspace.Name, "endpoint", endpoints[name]) + } + + return endpoints +} + +func (o *CompletedAddDatasourceOptions) reconcileDatasources(ctx context.Context, logger logr.Logger, activeWorkspaceNames set.Set[string], workspaceEndpoints map[string]string) error { + datasources, err := o.GrafanaClient.ListDataSources(ctx) if err != nil { - return nil, fmt.Errorf("failed to list Azure Monitor Workspaces: %w", err) + return fmt.Errorf("failed to list Grafana datasources: %w", err) } - for _, workspace := range monitorWorkspaces { - if workspace.Properties == nil || workspace.Properties.ProvisioningState == nil || workspace.ID == nil { + var reconcileErrors error + for _, ds := range datasources { + if ds.Type != "prometheus" { + continue + } + + workspaceName := strings.TrimPrefix(ds.Name, "Managed_Prometheus_") + if workspaceName == ds.Name { + continue + } + + lowerName := strings.ToLower(workspaceName) + + if !activeWorkspaceNames.Has(lowerName) { + if o.DryRun { + logger.Info("Dry run - would delete orphaned datasource", "datasource-name", ds.Name) + continue + } + + logger.Info("Deleting orphaned datasource", "datasource-name", ds.Name) + if err := o.GrafanaClient.DeleteDataSource(ctx, ds.Name); err != nil { + reconcileErrors = errors.Join(reconcileErrors, fmt.Errorf("failed to delete datasource %q: %w", ds.Name, err)) + } + continue + } + + expectedEndpoint, ok := workspaceEndpoints[lowerName] + if !ok { + logger.Info("Workspace exists but has no Prometheus endpoint yet, skipping", "datasource-name", ds.Name) + continue + } + + if ds.URL == expectedEndpoint { + logger.Info("Datasource URL is current", "datasource-name", ds.Name, "url", ds.URL) continue } - if *workspace.Properties.ProvisioningState == armmonitor.ProvisioningStateSucceeded { - logger.Info("Found", "workspace-id", *workspace.ID, "provisioning-state", *workspace.Properties.ProvisioningState) - validWorkspaceIDs.Insert(strings.ToLower(*workspace.ID)) + + if o.DryRun { + logger.Info("Dry run - would update stale datasource URL", + "datasource-name", ds.Name, + "current-url", ds.URL, + "expected-url", expectedEndpoint) + continue } + + logger.Info("Updating stale datasource URL", + "datasource-name", ds.Name, + "current-url", ds.URL, + "expected-url", expectedEndpoint) + + ds.URL = expectedEndpoint + if err := o.GrafanaClient.UpdateDataSource(ctx, ds); err != nil { + reconcileErrors = errors.Join(reconcileErrors, fmt.Errorf("failed to update datasource %q URL: %w", ds.Name, err)) + } + } + + if reconcileErrors != nil { + return fmt.Errorf("failed to reconcile datasources: %w", reconcileErrors) } - return validWorkspaceIDs, nil + return nil } func (o *CompletedAddDatasourceOptions) Run(ctx context.Context) error { @@ -115,11 +226,13 @@ func (o *CompletedAddDatasourceOptions) Run(ctx context.Context) error { return fmt.Errorf("failed to get Grafana instance: %w", err) } - validWorkspaceIDs, err := o.getMatchingWorkspaceIDs(ctx, logger) + monitorWorkspaces, err := o.MonitorWorkspaceClient.GetAllMonitorWorkspaces(ctx) if err != nil { - return fmt.Errorf("failed to get valid workspace IDs: %w", err) + return fmt.Errorf("failed to list Azure Monitor Workspaces: %w", err) } + validWorkspaceIDs := getMatchingWorkspaceIDs(monitorWorkspaces, logger) + integrationList := set.New[string]() for _, integration := range grafana.Properties.GrafanaIntegrations.AzureMonitorWorkspaceIntegrations { if integration.AzureMonitorWorkspaceResourceID == nil { @@ -142,14 +255,21 @@ func (o *CompletedAddDatasourceOptions) Run(ctx context.Context) error { if o.DryRun { logger.Info("Dry run - would reconcile Azure Monitor Workspace integrations", "total-integrations", integrationList.Len()) - return nil + } else { + logger.Info("Reconciling Azure Monitor Workspace integrations", "total-integrations", integrationList.Len()) + + err = o.ManagedGrafanaClient.UpdateGrafanaIntegrations(ctx, o.ResourceGroup, o.GrafanaName, integrationList.UnsortedList()) + if err != nil { + return fmt.Errorf("failed to update Grafana integrations: %w", err) + } } - logger.Info("Reconciling Azure Monitor Workspace integrations", "total-integrations", integrationList.Len()) + activeWorkspaceNames := getActiveWorkspaceNames(monitorWorkspaces) + workspaceEndpoints := getWorkspaceEndpoints(monitorWorkspaces, logger) - err = o.ManagedGrafanaClient.UpdateGrafanaIntegrations(ctx, o.ResourceGroup, o.GrafanaName, integrationList.UnsortedList()) - if err != nil { - return fmt.Errorf("failed to update Grafana integrations: %w", err) + logger.Info("Reconciling datasources") + if err := o.reconcileDatasources(ctx, logger, activeWorkspaceNames, workspaceEndpoints); err != nil { + return fmt.Errorf("failed to reconcile datasources: %w", err) } return nil diff --git a/tools/grafanactl/cmd/modify/options.go b/tools/grafanactl/cmd/modify/options.go index 28cbd3d..47eec6e 100644 --- a/tools/grafanactl/cmd/modify/options.go +++ b/tools/grafanactl/cmd/modify/options.go @@ -23,6 +23,7 @@ import ( "github.com/Azure/ARO-Tools/tools/cmdutils" "github.com/Azure/ARO-Tools/tools/grafanactl/cmd/base" "github.com/Azure/ARO-Tools/tools/grafanactl/internal/azure" + "github.com/Azure/ARO-Tools/tools/grafanactl/internal/grafana" ) // RawAddDatasourceOptions represents the initial, unvalidated configuration for add datasource operations. @@ -48,6 +49,7 @@ type ValidatedAddDatasourceOptions struct { // for add datasource operations. type CompletedAddDatasourceOptions struct { *validatedAddDatasourceOptions + GrafanaClient *grafana.Client MonitorWorkspaceClient *azure.MonitorWorkspaceClient ManagedGrafanaClient *azure.ManagedGrafanaClient } @@ -107,6 +109,11 @@ func (o *ValidatedAddDatasourceOptions) Complete(ctx context.Context) (*Complete return nil, fmt.Errorf("failed to create managed Grafana client: %w", err) } + grafanaClient, err := grafana.NewClient(ctx, cred, managedGrafanaClient, o.SubscriptionID, o.ResourceGroup, o.GrafanaName) + if err != nil { + return nil, fmt.Errorf("failed to create Grafana client: %w", err) + } + monitorWorkspaceClient, err := azure.NewMonitorWorkspaceClient(o.SubscriptionID, cred, clientOpts) if err != nil { return nil, fmt.Errorf("failed to create monitor workspace client: %w", err) @@ -114,6 +121,7 @@ func (o *ValidatedAddDatasourceOptions) Complete(ctx context.Context) (*Complete return &CompletedAddDatasourceOptions{ validatedAddDatasourceOptions: o.validatedAddDatasourceOptions, + GrafanaClient: grafanaClient, MonitorWorkspaceClient: monitorWorkspaceClient, ManagedGrafanaClient: managedGrafanaClient, }, nil diff --git a/tools/grafanactl/internal/grafana/client.go b/tools/grafanactl/internal/grafana/client.go index 417d23f..dac992e 100644 --- a/tools/grafanactl/internal/grafana/client.go +++ b/tools/grafanactl/internal/grafana/client.go @@ -108,6 +108,16 @@ func (c *Client) DeleteDataSource(ctx context.Context, dataSourceName string) er return nil } +// UpdateDataSource updates a datasource in the Grafana instance. +func (c *Client) UpdateDataSource(ctx context.Context, ds sdk.Datasource) error { + _, err := c.grafanaClient.UpdateDatasource(ctx, ds) + if err != nil { + return fmt.Errorf("failed to update datasource %q (ID %d): %w", ds.Name, ds.ID, err) + } + + return nil +} + // ListFolders returns all folders in the Grafana instance. func (c *Client) ListFolders(ctx context.Context) ([]sdk.Folder, error) { folders, err := c.grafanaClient.GetAllFolders(ctx)