From 3236ac5a877f27e1426e1edec1e97bde77801395 Mon Sep 17 00:00:00 2001 From: Vladimir Belousov <89217163+vbelouso@users.noreply.github.com> Date: Mon, 19 Jan 2026 19:43:12 +0200 Subject: [PATCH 01/53] Merge pull request #233 from vbelouso/fix-pgsql-cron fix: enable pg_cron in PostgreSQL statefulset --- db/sql/cron.sql | 1 + .../helm/cluster-iq/templates/database/configmap-init.yaml | 3 +++ .../helm/cluster-iq/templates/database/statefulset.yaml | 5 +++++ 3 files changed, 9 insertions(+) diff --git a/db/sql/cron.sql b/db/sql/cron.sql index 73900a99..1ea002f3 100644 --- a/db/sql/cron.sql +++ b/db/sql/cron.sql @@ -1,6 +1,7 @@ \c postgres -- pg_cron task for updating the 'Terminated' elements in the inventory every 6 hours +CREATE EXTENSION IF NOT EXISTS pg_cron; SELECT cron.schedule_in_database( 'check_terminated_inventory', '0 */6 * * *', diff --git a/deployments/helm/cluster-iq/templates/database/configmap-init.yaml b/deployments/helm/cluster-iq/templates/database/configmap-init.yaml index 39cdc78b..e8282432 100644 --- a/deployments/helm/cluster-iq/templates/database/configmap-init.yaml +++ b/deployments/helm/cluster-iq/templates/database/configmap-init.yaml @@ -8,6 +8,9 @@ metadata: immutable: false data: cron.sql: | + \c postgres + CREATE EXTENSION IF NOT EXISTS pg_cron; + -- pg_cron task for updating the 'Terminated' elements in the inventory every 6 hours SELECT cron.schedule_in_database( 'check_terminated_inventory', diff --git a/deployments/helm/cluster-iq/templates/database/statefulset.yaml b/deployments/helm/cluster-iq/templates/database/statefulset.yaml index ca390e60..f849a33b 100644 --- a/deployments/helm/cluster-iq/templates/database/statefulset.yaml +++ b/deployments/helm/cluster-iq/templates/database/statefulset.yaml @@ -49,6 +49,11 @@ spec: envFrom: - secretRef: name: postgresql + env: + - name: POSTGRESQL_LIBRARIES + value: "pg_cron" + - name: POSTGRESQL_EXTENSIONS + value: "pg_cron" securityContext: {{- toYaml .Values.database.securityContext | nindent 12 }} image: "{{ .Values.database.image.repository }}:{{ .Values.database.image.tag | default .Chart.AppVersion }}" From da5197b87483d014a3b4867525f334a2f90db163 Mon Sep 17 00:00:00 2001 From: Vladimir Belousov <89217163+vbelouso@users.noreply.github.com> Date: Tue, 20 Jan 2026 13:31:14 +0200 Subject: [PATCH 02/53] fix: resolve logic bug causing skipped instances in AWS stocker (#240) Signed-off-by: Vladimir Belousov --- internal/stocker/aws_stocker_ec2.go | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/internal/stocker/aws_stocker_ec2.go b/internal/stocker/aws_stocker_ec2.go index 444d4ae3..b3e4d5c2 100644 --- a/internal/stocker/aws_stocker_ec2.go +++ b/internal/stocker/aws_stocker_ec2.go @@ -35,6 +35,7 @@ func (s *AWSStocker) processInstances(instances []inventory.Instance) { // Generating ClusterID for this instance based on its properties clusterName := inventory.GetClusterNameFromTags(instance.Tags) infraID := inventory.GetInfraIDFromTags(instance.Tags) + if s.skipNoOpenShiftInstances && clusterName == inventory.UnknownClusterNameCode { s.logger.Debug("Skipping instance because it's not associated to any cluster", zap.String("account_id", s.Account.AccountID), @@ -61,14 +62,14 @@ func (s *AWSStocker) processInstances(instances []inventory.Instance) { if !s.Account.IsClusterInAccount(cluster.ClusterID) { _ = s.Account.AddCluster(cluster) } + } - if err := s.Account.Clusters[clusterID].AddInstance(&instance); err != nil { - s.logger.Error("error adding instance to cluster during instance processing", - zap.String("account_id", s.Account.AccountID), - zap.String("cluster_id", clusterID), - zap.String("instance_id", instance.InstanceID), - zap.Error(err)) - } + if err := s.Account.Clusters[clusterID].AddInstance(&instance); err != nil { + s.logger.Error("error adding instance to cluster during instance processing", + zap.String("account_id", s.Account.AccountID), + zap.String("cluster_id", clusterID), + zap.String("instance_id", instance.InstanceID), + zap.Error(err)) } } } From d989adb9ff50e1455ab66d215ff190b10d7a776f Mon Sep 17 00:00:00 2001 From: Vladimir Belousov <89217163+vbelouso@users.noreply.github.com> Date: Tue, 20 Jan 2026 20:10:50 +0200 Subject: [PATCH 03/53] fix: initialize repository slices to return empty array instead of null (#235) Signed-off-by: Vladimir Belousov --- internal/repositories/account_repository.go | 6 +++--- internal/repositories/action_repository.go | 2 +- internal/repositories/cluster_repository.go | 6 +++--- internal/repositories/event_repository.go | 4 ++-- internal/repositories/expense_repository.go | 2 +- internal/repositories/instance_repository.go | 4 ++-- 6 files changed, 12 insertions(+), 12 deletions(-) diff --git a/internal/repositories/account_repository.go b/internal/repositories/account_repository.go index 76c77e5d..b013cf4a 100644 --- a/internal/repositories/account_repository.go +++ b/internal/repositories/account_repository.go @@ -69,7 +69,7 @@ func NewAccountRepository(db *dbclient.DBClient) AccountRepository { // - A slice of inventory.Account objects. // - An error if the query fails. func (r *accountRepositoryImpl) ListAccounts(ctx context.Context, opts models.ListOptions) ([]db.AccountDBResponse, int, error) { - var accounts []db.AccountDBResponse + accounts := []db.AccountDBResponse{} if err := r.db.SelectWithContext(ctx, &accounts, SelectAccountsMView, opts, "account_id", "*"); err != nil { return accounts, 0, fmt.Errorf("failed to list accounts: %w", err) @@ -130,7 +130,7 @@ func (r *accountRepositoryImpl) GetAccountByID(ctx context.Context, accountID st // - A slice of inventory.Account objects (usually containing one element). // - An error if the query fails. func (r *accountRepositoryImpl) GetAccountClustersByID(ctx context.Context, accountID string) ([]db.ClusterDBResponse, error) { - var clusters []db.ClusterDBResponse + clusters := []db.ClusterDBResponse{} opts := models.ListOptions{ PageSize: 0, @@ -157,7 +157,7 @@ func (r *accountRepositoryImpl) GetAccountClustersByID(ctx context.Context, acco // - A slice of inventory.Instance objects. // - An error if the query fails. func (r *accountRepositoryImpl) GetExpenseUpdateInstances(ctx context.Context, accountID string) ([]db.InstanceDBResponse, error) { - var instances []db.InstanceDBResponse + instances := []db.InstanceDBResponse{} opts := models.ListOptions{ PageSize: 0, diff --git a/internal/repositories/action_repository.go b/internal/repositories/action_repository.go index 7b88917c..0efd533e 100644 --- a/internal/repositories/action_repository.go +++ b/internal/repositories/action_repository.go @@ -126,7 +126,7 @@ func NewActionRepository(db *dbclient.DBClient) ActionRepository { // - An array of actions.Action with the scheduled actions declared on the DB // - An error if the query fails func (r *actionRepositoryImpl) List(ctx context.Context, opts models.ListOptions) ([]db.ActionDBResponse, int, error) { - var schedule []db.ActionDBResponse + schedule := []db.ActionDBResponse{} if err := r.db.SelectWithContext(ctx, &schedule, SelectScheduleFullView, opts, "id", "*"); err != nil { return schedule, 0, fmt.Errorf("failed to list schedule: %w", err) diff --git a/internal/repositories/cluster_repository.go b/internal/repositories/cluster_repository.go index 483e0a1e..0673f709 100644 --- a/internal/repositories/cluster_repository.go +++ b/internal/repositories/cluster_repository.go @@ -93,7 +93,7 @@ func NewClusterRepository(db *dbclient.DBClient) ClusterRepository { // - A slice of inventory.Cluster objects. // - An error if the query fails. func (r *clusterRepositoryImpl) ListClusters(ctx context.Context, opts models.ListOptions) ([]db.ClusterDBResponse, int, error) { - var clusters []db.ClusterDBResponse + clusters := []db.ClusterDBResponse{} if err := r.db.SelectWithContext(ctx, &clusters, SelectClustersFullMView, opts, "cluster_id", "*"); err != nil { return clusters, 0, fmt.Errorf("failed to list clusters: %w", err) @@ -188,7 +188,7 @@ func (r *clusterRepositoryImpl) GetClusterRegion(ctx context.Context, clusterID // - A slice of inventory.Tag objects representing the cluster's tags. // - An error if the query fails. func (r *clusterRepositoryImpl) GetClusterTags(ctx context.Context, clusterID string) ([]db.TagDBResponse, error) { - var result []db.TagDBResponse + result := []db.TagDBResponse{} opts := models.ListOptions{ PageSize: 0, @@ -239,7 +239,7 @@ func (r *clusterRepositoryImpl) GetClustersOnAccount(ctx context.Context, accoun // - A slice of inventory.Instance objects representing the instances in the cluster. // - An error if the query fails. func (r *clusterRepositoryImpl) GetInstancesOnCluster(ctx context.Context, clusterID string) ([]db.InstanceDBResponse, error) { - var instances []db.InstanceDBResponse + instances := []db.InstanceDBResponse{} opts := models.ListOptions{ PageSize: 0, diff --git a/internal/repositories/event_repository.go b/internal/repositories/event_repository.go index eadc92df..325835e2 100644 --- a/internal/repositories/event_repository.go +++ b/internal/repositories/event_repository.go @@ -70,7 +70,7 @@ func NewEventRepository(db *dbclient.DBClient) EventRepository { // ListSystemEvents retrieves system-wide events with extended metadata. func (r *eventRepositoryImpl) ListSystemEvents(ctx context.Context, opts models.ListOptions) ([]db.SystemEventDBResponse, int, error) { - var events []db.SystemEventDBResponse + events := []db.SystemEventDBResponse{} if err := r.db.SelectWithContext(ctx, &events, SelectSystemEventsView, opts, "event_timestamp", "*"); err != nil { return events, 0, fmt.Errorf("failed to list events: %w", err) @@ -81,7 +81,7 @@ func (r *eventRepositoryImpl) ListSystemEvents(ctx context.Context, opts models. // ListClusterEvents retrieves events for a specific resource (like a cluster). func (r *eventRepositoryImpl) ListClusterEvents(ctx context.Context, opts models.ListOptions) ([]db.ClusterEventDBResponse, int, error) { - var events []db.ClusterEventDBResponse + events := []db.ClusterEventDBResponse{} if err := r.db.SelectWithContext(ctx, &events, SelectClusterEventsView, opts, "event_timestamp", "*"); err != nil { return events, 0, fmt.Errorf("failed to list cluster events: %w", err) diff --git a/internal/repositories/expense_repository.go b/internal/repositories/expense_repository.go index af6edf34..17fd4f81 100644 --- a/internal/repositories/expense_repository.go +++ b/internal/repositories/expense_repository.go @@ -53,7 +53,7 @@ func NewExpenseRepository(db *dbclient.DBClient) ExpenseRepository { // - A slice of inventory.Expense objects. // - An error if the query fails. func (r *expenseRepositoryImpl) ListExpenses(ctx context.Context, opts models.ListOptions) ([]db.ExpenseDBResponse, int, error) { - var expenses []db.ExpenseDBResponse + expenses := []db.ExpenseDBResponse{} if err := r.db.SelectWithContext(ctx, &expenses, ExpensesTable, opts, "date", "*"); err != nil { return expenses, 0, fmt.Errorf("failed to list expenses: %w", err) diff --git a/internal/repositories/instance_repository.go b/internal/repositories/instance_repository.go index 2ecc63d6..2aa5a911 100644 --- a/internal/repositories/instance_repository.go +++ b/internal/repositories/instance_repository.go @@ -97,7 +97,7 @@ func NewInstanceRepository(db *dbclient.DBClient) InstanceRepository { // - A slice of inventory.Instance objects. // - An error if the query fails. func (r *instanceRepositoryImpl) ListInstances(ctx context.Context, opts models.ListOptions) ([]db.InstanceDBResponse, int, error) { - var instances []db.InstanceDBResponse + instances := []db.InstanceDBResponse{} if err := r.db.SelectWithContext(ctx, &instances, SelectInstancesFullMView, opts, "instance_id", "*"); err != nil { return instances, 0, fmt.Errorf("failed to list instances: %w", err) @@ -162,7 +162,7 @@ func (r *instanceRepositoryImpl) CreateInstances(ctx context.Context, instances return err } - var newTags []inventory.Tag + newTags := []inventory.Tag{} for _, instance := range instances { for _, tag := range instance.Tags { tag.InstanceID = instance.InstanceID From e919aaa617c93eab577f6d3bc433858b87074ce4 Mon Sep 17 00:00:00 2001 From: Vladimir Belousov <89217163+vbelouso@users.noreply.github.com> Date: Tue, 20 Jan 2026 20:11:32 +0200 Subject: [PATCH 04/53] chore: remove pagination size limit (#234) Signed-off-by: Vladimir Belousov --- internal/models/dto/pagination.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/models/dto/pagination.go b/internal/models/dto/pagination.go index edaa5f43..ec824a1b 100644 --- a/internal/models/dto/pagination.go +++ b/internal/models/dto/pagination.go @@ -4,5 +4,5 @@ package dto // allowing clients to specify page number and page size. type PaginationRequest struct { Page int `form:"page,default=1" binding:"gte=1"` - PageSize int `form:"page_size,default=10" binding:"gte=1,lte=100"` + PageSize int `form:"page_size,default=10" binding:"gte=1"` } // @name PaginationRequest From ce7f864704fc1c0ad97ecee444fa2de9f1b1a0d3 Mon Sep 17 00:00:00 2001 From: Alejandro Villegas Date: Thu, 22 Jan 2026 08:28:32 +0100 Subject: [PATCH 05/53] fix: Fixed cluster age calculation, unit tests and DTO transform function where 'age' parameter was missing (#244) Signed-off-by: r2dedios --- internal/inventory/cluster.go | 4 +- internal/inventory/cluster_test.go | 272 +++++++++++++++++----------- internal/models/dto/cluster_dto.go | 1 + internal/stocker/aws_stocker_ec2.go | 36 ++-- 4 files changed, 191 insertions(+), 122 deletions(-) diff --git a/internal/inventory/cluster.go b/internal/inventory/cluster.go index 25ffe677..120d9b4b 100644 --- a/internal/inventory/cluster.go +++ b/internal/inventory/cluster.go @@ -127,6 +127,8 @@ func (c *Cluster) Update() error { // UpdateAge updates cluster age based on the oldest instance creation timestamp. The cluster will be considered as old as the oldest instance func (c *Cluster) UpdateAge() error { creationTS := time.Now() + + // Getting oldest creation timestamp for _, instance := range c.Instances { if instance.CreatedAt.Before(creationTS) { creationTS = instance.CreatedAt @@ -135,7 +137,7 @@ func (c *Cluster) UpdateAge() error { c.CreatedAt = creationTS // Calculating Age in days since the cluster was created until last scraping - newAge := calculateAge(c.CreatedAt, c.LastScanTimestamp) + newAge := calculateAge(c.CreatedAt, time.Now()) if c.Age > newAge && c.Age != 0 { return fmt.Errorf("%s. Current age: %d, New estimated age: %d", ErrorNewClusterAge, c.Age, newAge) } diff --git a/internal/inventory/cluster_test.go b/internal/inventory/cluster_test.go index 095a0b12..02566d6d 100644 --- a/internal/inventory/cluster_test.go +++ b/internal/inventory/cluster_test.go @@ -128,136 +128,204 @@ func testIsClusterRunning_Terminated(t *testing.T) { assert.False(t, cluster.IsClusterRunning()) } -// TestUpdate tests the Update function including Age and Status updates -func TestUpdate(t *testing.T) { - t.Run("UpdateAge", func(t *testing.T) { testUpdate_Correct(t) }) - t.Run("UpdateAge Lower New Age", func(t *testing.T) { testUpdate_NewerAge(t) }) -} +// TestCluster_Update covers Cluster.Update() behavior, including status update, +// age update, and error propagation from UpdateAge(). +func TestCluster_Update(t *testing.T) { + t.Run("success updates status and age", testCluster_Update_SuccessUpdatesStatusAndAge) + t.Run("error from UpdateAge is returned and status is still updated", testCluster_Update_ErrorFromUpdateAgeIsReturnedAndStatusIsStillUpdated) +} + +// testCluster_Update_SuccessUpdatesStatusAndAge validates that Update() updates both status and age. +// It also verifies CreatedAt is set to the oldest instance creation timestamp. +func testCluster_Update_SuccessUpdatesStatusAndAge(t *testing.T) { + c := testClusterWithInstances( + Instance{Status: Stopped, CreatedAt: time.Now().Add(-48 * time.Hour)}, + Instance{Status: Terminated, CreatedAt: time.Now().Add(-24 * time.Hour)}, + ) + + // status must become Stopped (no Running, not all Terminated). + // CreatedAt must be the oldest instance timestamp. + expectedCreatedAt := c.Instances[0].CreatedAt + expectedAge := calculateAge(expectedCreatedAt, time.Now()) + + err := c.Update() + assert.NoError(t, err) + assert.Equal(t, Stopped, c.Status) + assert.True(t, c.CreatedAt.Equal(expectedCreatedAt)) + assert.Equal(t, expectedAge, c.Age) +} + +// testCluster_Update_ErrorFromUpdateAgeIsReturnedAndStatusIsStillUpdated validates that Update() +// returns UpdateAge() error, but still updates status beforehand. +func testCluster_Update_ErrorFromUpdateAgeIsReturnedAndStatusIsStillUpdated(t *testing.T) { + oldest := time.Now().Add(-48 * time.Hour) + + c := testClusterWithInstances( + Instance{Status: Running, CreatedAt: oldest}, // Forces early Running status + Instance{Status: Stopped, CreatedAt: time.Now().Add(-24 * time.Hour)}, + ) + + // force the "new estimated age is lower" error branch. + newAge := calculateAge(oldest, time.Now()) + c.Age = newAge + 10 + + err := c.Update() + assert.Error(t, err) + assert.ErrorContains(t, err, ErrorNewClusterAge.Error()) -func testUpdate_Correct(t *testing.T) { - cluster, err := NewCluster("testCluster", "i1", AWSProvider, "us-east-1", "https://console", "user") - assert.NotNil(t, cluster) - assert.Nil(t, err) + // UpdateStatus runs before UpdateAge, so status must be updated even on error. + assert.Equal(t, Running, c.Status) - err = cluster.Update() - assert.Nil(t, err) -} + // UpdateAge sets CreatedAt before comparing ages, so CreatedAt is still updated. + assert.True(t, c.CreatedAt.Equal(oldest)) -func testUpdate_NewerAge(t *testing.T) { - prevAge := 30 - cluster, err := NewCluster("testCluster", "i1", AWSProvider, "us-east-1", "https://console", "user") - assert.NotNil(t, cluster) - assert.Nil(t, err) - t1 := time.Now() - t2 := t1.Add(-12 * 24 * time.Hour) - cluster.Instances = []Instance{ - {CreatedAt: t1}, - {CreatedAt: t2}, - } + // on error, Age must remain unchanged (the assignment happens after the check). + assert.Equal(t, newAge+10, c.Age) +} - cluster.Age = prevAge - err = cluster.Update() - assert.Error(t, err) - assert.ErrorContains(t, err, ErrorNewClusterAge.Error()) +// TestCluster_UpdateAge covers Cluster.UpdateAge() behavior, including empty clusters, +// selecting the oldest instance timestamp, and the "age regression" error. +func TestCluster_UpdateAge(t *testing.T) { + t.Run("empty cluster sets CreatedAt to now and Age to 1", testCluster_UpdateAge_EmptyClusterSetsCreatedAtToNowAndAgeTo0) + t.Run("sets CreatedAt to oldest instance and updates Age", testCluster_UpdateAge_SetsCreatedAtToOldestInstanceAndUpdatesAge) + t.Run("returns error when current Age is greater than newly estimated Age and Age != 0", testCluster_UpdateAge_ReturnsErrorOnAgeRegression) } -// TestUpdateAge tests UpdateAge function -func TestUpdateAge(t *testing.T) { - t.Run("UpdateAge", func(t *testing.T) { testUpdateAge_Correct(t) }) - t.Run("UpdateAge Lower New Age", func(t *testing.T) { testUpdateAge_NewerAge(t) }) +// testCluster_UpdateAge_EmptyClusterSetsCreatedAtToNowAndAgeTo0 validates UpdateAge() behavior +// when the cluster has no instances. +func testCluster_UpdateAge_EmptyClusterSetsCreatedAtToNowAndAgeTo0(t *testing.T) { + c := &Cluster{} + + err := c.UpdateAge() + assert.NoError(t, err) + + // with no instances, CreatedAt starts as time.Now() and Age becomes 0 days. + assert.Equal(t, 1, c.Age) + assert.False(t, c.CreatedAt.IsZero()) } -func testUpdateAge_Correct(t *testing.T) { - cluster, err := NewCluster("testCluster", "i1", AWSProvider, "us-east-1", "https://console", "user") - assert.NotNil(t, cluster) - assert.Nil(t, err) - t1 := time.Now() - t2 := t1.Add(-12 * 24 * time.Hour) - cluster.Instances = []Instance{ - {CreatedAt: t1}, - {CreatedAt: t2}, - } +// testCluster_UpdateAge_SetsCreatedAtToOldestInstanceAndUpdatesAge validates that UpdateAge() +// uses the oldest instance creation timestamp and updates cluster age accordingly. +func testCluster_UpdateAge_SetsCreatedAtToOldestInstanceAndUpdatesAge(t *testing.T) { + oldest := time.Now().Add(-72 * time.Hour) + newer := time.Now().Add(-24 * time.Hour) - err = cluster.UpdateAge() - assert.Nil(t, err) - assert.Equal(t, cluster.CreatedAt, t2) - assert.Equal(t, cluster.Age, 11) + c := testClusterWithInstances( + Instance{Status: Stopped, CreatedAt: newer}, + Instance{Status: Stopped, CreatedAt: oldest}, + ) + + expectedAge := calculateAge(oldest, time.Now()) + + err := c.UpdateAge() + assert.NoError(t, err) + + // CreatedAt must reflect the oldest node in the cluster. + assert.True(t, c.CreatedAt.Equal(oldest)) + + // Age is recalculated using CreatedAt and the scrape time (time.Now()). + assert.Equal(t, expectedAge, c.Age) } -func testUpdateAge_NewerAge(t *testing.T) { - prevAge := 30 - cluster, err := NewCluster("testCluster", "i1", AWSProvider, "us-east-1", "https://console", "user") - assert.NotNil(t, cluster) - assert.Nil(t, err) - t1 := time.Now() - t2 := t1.Add(-12 * 24 * time.Hour) - cluster.Instances = []Instance{ - {CreatedAt: t1}, - {CreatedAt: t2}, - } - cluster.Age = prevAge +// testCluster_UpdateAge_ReturnsErrorOnAgeRegression validates that UpdateAge() fails when it detects +// a regression in age (newly estimated Age is lower than the current Age), and Age != 0. +func testCluster_UpdateAge_ReturnsErrorOnAgeRegression(t *testing.T) { + oldest := time.Now().Add(-48 * time.Hour) + + c := testClusterWithInstances( + Instance{Status: Stopped, CreatedAt: oldest}, + ) + + newAge := calculateAge(oldest, time.Now()) + c.Age = newAge + 1 // Force regression - err = cluster.UpdateAge() + err := c.UpdateAge() assert.Error(t, err) assert.ErrorContains(t, err, ErrorNewClusterAge.Error()) - assert.NotZero(t, cluster.CreatedAt) - assert.Equal(t, cluster.Age, prevAge) + + // CreatedAt is assigned before the regression check. + assert.True(t, c.CreatedAt.Equal(oldest)) + + // on error, Age must remain the previous value. + assert.Equal(t, newAge+1, c.Age) } -// TestUpdateStatus tests the UpdateStatus function -func TestUpdateStatus(t *testing.T) { - t.Run("UpdateStatus no Instances", func(t *testing.T) { testUpdateStatus_NoInstances(t) }) - t.Run("UpdateStatus at lease one Instance Running", func(t *testing.T) { testUpdateStatus_AtLeastOneInstanceRunning(t) }) - t.Run("UpdateStatus all Instances Terminated", func(t *testing.T) { testUpdateStatus_TerminatedInstances(t) }) - t.Run("UpdateStatus mix Stopped and Terminated", func(t *testing.T) { testUpdateStatus_NoInstancesRunning(t) }) +// TestCluster_UpdateStatus covers Cluster.UpdateStatus() logic for empty clusters, +// Running early return, all Terminated, and all/mixed Stopped/Terminated. +func TestCluster_UpdateStatus(t *testing.T) { + t.Run("empty cluster -> Terminated", testCluster_UpdateStatus_EmptyClusterTerminated) + t.Run("any instance Running -> Running (early return)", testCluster_UpdateStatus_AnyRunningEarlyReturnRunning) + t.Run("all instances Terminated -> Terminated", testCluster_UpdateStatus_AllTerminatedTerminated) + t.Run("mix of Stopped/Terminated (no Running) -> Stopped", testCluster_UpdateStatus_MixStoppedTerminatedStopped) + t.Run("all instances Stopped -> Stopped", testCluster_UpdateStatus_AllStoppedStopped) } -func testUpdateStatus_NoInstances(t *testing.T) { - cluster, err := NewCluster("testCluster", "i1", AWSProvider, "us-east-1", "https://console", "user") - assert.NotNil(t, cluster) - assert.Nil(t, err) +// testCluster_UpdateStatus_EmptyClusterTerminated validates that an empty cluster is marked as Terminated. +func testCluster_UpdateStatus_EmptyClusterTerminated(t *testing.T) { + c := &Cluster{Instances: nil} - cluster.UpdateStatus() - assert.Equal(t, cluster.Status, Terminated) + c.UpdateStatus() + assert.Equal(t, Terminated, c.Status) } -func testUpdateStatus_AtLeastOneInstanceRunning(t *testing.T) { - cluster, err := NewCluster("testCluster", "i1", AWSProvider, "us-east-1", "https://console", "user") - assert.NotNil(t, cluster) - assert.Nil(t, err) - cluster.Instances = []Instance{ - {Status: Running}, - {Status: Stopped}, - } +// testCluster_UpdateStatus_AnyRunningEarlyReturnRunning validates the early-return rule: +// any Running instance sets the cluster status to Running. +func testCluster_UpdateStatus_AnyRunningEarlyReturnRunning(t *testing.T) { + c := testClusterWithInstances( + Instance{Status: Terminated}, + Instance{Status: Running}, // Must win + Instance{Status: Stopped}, + ) - cluster.UpdateStatus() - assert.Equal(t, cluster.Status, Running) + c.UpdateStatus() + assert.Equal(t, Running, c.Status) } -func testUpdateStatus_TerminatedInstances(t *testing.T) { - cluster, err := NewCluster("testCluster", "i1", AWSProvider, "us-east-1", "https://console", "user") - assert.Nil(t, err) - assert.NotNil(t, cluster) - cluster.Instances = []Instance{ - {Status: Terminated}, - {Status: Terminated}, - } +// testCluster_UpdateStatus_AllTerminatedTerminated validates that all Terminated instances +// result in cluster status Terminated. +func testCluster_UpdateStatus_AllTerminatedTerminated(t *testing.T) { + c := testClusterWithInstances( + Instance{Status: Terminated}, + Instance{Status: Terminated}, + ) - cluster.UpdateStatus() - assert.Equal(t, cluster.Status, Terminated) + c.UpdateStatus() + assert.Equal(t, Terminated, c.Status) } -func testUpdateStatus_NoInstancesRunning(t *testing.T) { - cluster, err := NewCluster("testCluster", "i1", AWSProvider, "us-east-1", "https://console", "user") - assert.Nil(t, err) - assert.NotNil(t, cluster) - cluster.Instances = []Instance{ - {Status: Stopped}, - {Status: Terminated}, - } +// testCluster_UpdateStatus_MixStoppedTerminatedStopped validates that a mix of Stopped/Terminated +// (with no Running instances) results in cluster status Stopped. +func testCluster_UpdateStatus_MixStoppedTerminatedStopped(t *testing.T) { + c := testClusterWithInstances( + Instance{Status: Terminated}, + Instance{Status: Stopped}, + Instance{Status: Terminated}, + ) - cluster.UpdateStatus() - assert.Equal(t, cluster.Status, Stopped) + c.UpdateStatus() + assert.Equal(t, Stopped, c.Status) +} + +// testCluster_UpdateStatus_AllStoppedStopped validates that all Stopped instances +// result in cluster status Stopped. +func testCluster_UpdateStatus_AllStoppedStopped(t *testing.T) { + c := testClusterWithInstances( + Instance{Status: Stopped}, + Instance{Status: Stopped}, + ) + + c.UpdateStatus() + assert.Equal(t, Stopped, c.Status) +} + +// testClusterWithInstances builds a Cluster with the provided instances. +// It keeps tests concise and consistent across scenarios. +func testClusterWithInstances(instances ...Instance) *Cluster { + c := &Cluster{ + Instances: make([]Instance, 0, len(instances)), + } + c.Instances = append(c.Instances, instances...) + return c } // TestAddInstance tests the AddInstance function including repeated instances diff --git a/internal/models/dto/cluster_dto.go b/internal/models/dto/cluster_dto.go index b6e45142..c69d3bb0 100644 --- a/internal/models/dto/cluster_dto.go +++ b/internal/models/dto/cluster_dto.go @@ -40,6 +40,7 @@ func (c ClusterDTORequest) ToInventoryCluster() *inventory.Cluster { cluster.CreatedAt = c.CreatedAt cluster.Status = c.Status cluster.AccountID = c.AccountID + cluster.Age = c.Age return cluster } diff --git a/internal/stocker/aws_stocker_ec2.go b/internal/stocker/aws_stocker_ec2.go index b3e4d5c2..887ac4f9 100644 --- a/internal/stocker/aws_stocker_ec2.go +++ b/internal/stocker/aws_stocker_ec2.go @@ -44,30 +44,28 @@ func (s *AWSStocker) processInstances(instances []inventory.Instance) { continue } - clusterID := inventory.GenerateClusterID(clusterName, infraID) - if !s.Account.IsClusterInAccount(clusterID) { - cluster, err := inventory.NewCluster( - clusterName, - infraID, - inventory.AWSProvider, - s.conn.GetRegion(), - unknownConsoleLinkCode, - inventory.GetOwnerFromTags(instance.Tags), - ) - if err != nil { - s.logger.Error("error creating new cluster during instance processing", zap.Error(err)) - continue - } + cluster, err := inventory.NewCluster( + clusterName, + infraID, + inventory.AWSProvider, + s.conn.GetRegion(), + unknownConsoleLinkCode, + inventory.GetOwnerFromTags(instance.Tags), + ) - if !s.Account.IsClusterInAccount(cluster.ClusterID) { - _ = s.Account.AddCluster(cluster) - } + if err != nil { + s.logger.Error("error creating new cluster during instance processing", zap.Error(err)) + continue + } + + if !s.Account.IsClusterInAccount(cluster.ClusterID) { + _ = s.Account.AddCluster(cluster) } - if err := s.Account.Clusters[clusterID].AddInstance(&instance); err != nil { + if err := s.Account.Clusters[cluster.ClusterID].AddInstance(&instance); err != nil { s.logger.Error("error adding instance to cluster during instance processing", zap.String("account_id", s.Account.AccountID), - zap.String("cluster_id", clusterID), + zap.String("cluster_id", cluster.ClusterID), zap.String("instance_id", instance.InstanceID), zap.Error(err)) } From bb9c6d0d7bdaeb16e2fc2d7e7aa4b0ea4bc32213 Mon Sep 17 00:00:00 2001 From: Alejandro Villegas Date: Thu, 22 Jan 2026 09:27:23 +0100 Subject: [PATCH 06/53] fix: Replaced actions API endpoint and fixed Cron+Scheduled Actions management before sending them to the channel (#251) Signed-off-by: r2dedios --- cmd/agent/executor_agent_service.go | 17 +++++++++-------- cmd/agent/instant_agent_service.go | 4 ++-- cmd/agent/schedule_agent_service.go | 6 +++--- 3 files changed, 14 insertions(+), 13 deletions(-) diff --git a/cmd/agent/executor_agent_service.go b/cmd/agent/executor_agent_service.go index c174e32d..e99f2958 100644 --- a/cmd/agent/executor_agent_service.go +++ b/cmd/agent/executor_agent_service.go @@ -127,36 +127,36 @@ func (e *ExecutorAgentService) createExecutors() error { for _, account := range accounts { switch account.Provider { case inventory.AWSProvider: // AWS - e.logger.Info("Creating Executor for AWS account", zap.String("account_name", account.Name)) - account, err := inventory.NewAccount(account.ID, account.Name, account.Provider, account.User, account.Key) + e.logger.Info("Creating Executor for AWS account", zap.String("account_id", account.ID)) + newAccount, err := inventory.NewAccount(account.ID, account.Name, account.Provider, account.User, account.Key) if err != nil { return err } exec := cexec.NewAWSExecutor( - account, + newAccount, e.actionsChannel, logger, ) err = e.AddExecutor(exec) if err != nil { - e.logger.Error("Cannot create an AWSEexecutor for account", zap.String("account_name", account.AccountName), zap.Error(err)) + e.logger.Error("Cannot create an AWSEexecutor for account", zap.String("account_id", newAccount.AccountID), zap.Error(err)) return err } case inventory.GCPProvider: // GCP e.logger.Warn("Failed to create Executor for GCP account", - zap.String("account", account.Name), + zap.String("account_id", account.ID), zap.String("reason", "not implemented"), ) case inventory.AzureProvider: // Azure e.logger.Warn("Failed to create Executor for Azure account", - zap.String("account", account.Name), + zap.String("account_id", account.ID), zap.String("reason", "not implemented"), ) case inventory.UnknownProvider: e.logger.Warn("Failed to create Executor for Unknown Provider account", - zap.String("account", account.Name), + zap.String("account_id", account.ID), zap.Any("provider", account.Provider), zap.String("reason", "Unknown provider"), ) @@ -202,6 +202,7 @@ func (e *ExecutorAgentService) Start() error { Severity: eventservice.SeverityInfo, TriggeredBy: newAction.GetRequester(), }) + // TODO: Keep this or transform the cannel into: 'chan *action.Action' // Mark the incoming action as 'Running' since it arrives to the ExecutorService newAction.(actions.MutableAction).SetStatus(actions.StatusRunning) @@ -213,7 +214,7 @@ func (e *ExecutorAgentService) Start() error { exec := e.GetExecutor(newAction.GetTarget().AccountID) if exec == nil { - e.logger.Error("there's no Executor available for the requested account", zap.String("account", newAction.GetTarget().AccountID)) + e.logger.Error("there's no Executor available for the requested account", zap.String("account_id", newAction.GetTarget().AccountID)) // Updating Action status m := newAction.(actions.MutableAction) diff --git a/cmd/agent/instant_agent_service.go b/cmd/agent/instant_agent_service.go index 6ad45237..e2bbf44a 100644 --- a/cmd/agent/instant_agent_service.go +++ b/cmd/agent/instant_agent_service.go @@ -104,7 +104,7 @@ func (i *InstantAgentService) Start() error { // - error: An error if the operation fails. func (i *InstantAgentService) PowerOnCluster(ctx context.Context, req *pb.PowerOnClusterRequest) (*pb.PowerOnClusterResponse, error) { i.logger.Warn("Powering On Cluster", - zap.String("account_name", req.AccountId), + zap.String("account_id", req.AccountId), zap.String("region", req.Region), zap.String("cluster_id", req.ClusterId), zap.Strings("instances", req.InstancesIdList), @@ -151,7 +151,7 @@ func (i *InstantAgentService) PowerOnCluster(ctx context.Context, req *pb.PowerO // - error: An error if the operation fails. func (i *InstantAgentService) PowerOffCluster(ctx context.Context, req *pb.PowerOffClusterRequest) (*pb.PowerOffClusterResponse, error) { i.logger.Warn("Powering Off Cluster", - zap.String("account_name", req.AccountId), + zap.String("account_id", req.AccountId), zap.String("region", req.Region), zap.String("cluster_id", req.ClusterId), zap.Strings("instances", req.InstancesIdList), diff --git a/cmd/agent/schedule_agent_service.go b/cmd/agent/schedule_agent_service.go index 12d8f460..79ccd85a 100644 --- a/cmd/agent/schedule_agent_service.go +++ b/cmd/agent/schedule_agent_service.go @@ -22,7 +22,7 @@ import ( const ( // APIScheduleActionsPath endpoint for retrieving the list of actions that needs to be rescheduled - APIScheduleActionsPath = "/schedule" + APIScheduleActionsPath = "/actions" ) // scheduleItem represents the pair of action and CancelFunc for tracking the already running actions @@ -103,7 +103,7 @@ func (a *ScheduleAgentService) scheduleNewScheduledAction(newAction actions.Sche select { case <-time.After(duration): // When the timestamp is "now" a.logger.Debug("Sending to execution channel", zap.String("action_id", actionID), zap.Int("channel", len(a.actionsChannel))) - a.actionsChannel <- newAction + a.actionsChannel <- &newAction a.logger.Debug("Action sent to execution channel", zap.String("action_id", actionID), zap.Int("channel", len(a.actionsChannel))) case <-ctx.Done(): // Context cancelling a.logger.Warn("Task cancelled before execution", zap.String("action_id", actionID)) @@ -163,7 +163,7 @@ func (a *ScheduleAgentService) scheduleNewCronAction(newAction actions.CronActio a.logger.Warn("Task cancelled before execution", zap.String("action_id", actionID), zap.String("action_cron_exp", newAction.GetCronExpression())) default: a.logger.Debug("Sending to execution channel", zap.String("action_id", actionID), zap.Int("channel", len(a.actionsChannel))) - a.actionsChannel <- newAction + a.actionsChannel <- &newAction a.logger.Debug("Action sent to execution channel", zap.String("action_id", actionID), zap.Int("channel", len(a.actionsChannel))) } }) From 9422c0c1b580b8e052f1eebc9dd1e5966576a1a8 Mon Sep 17 00:00:00 2001 From: Alejandro Villegas Date: Thu, 22 Jan 2026 12:39:15 +0100 Subject: [PATCH 07/53] fix: zero value expense for AWS stocker (#248) * fix: Fixed billing stocker by replacing InstanceName by InstanceID when asking to AWS for expenses Signed-off-by: r2dedios * doc: Updated handler comments for Swagger doc Signed-off-by: r2dedios --------- Signed-off-by: r2dedios --- cmd/scanner/scanner.go | 5 ++++- internal/api/handlers/account_handler.go | 2 +- internal/stocker/aws_billing_stocker.go | 8 ++++---- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/cmd/scanner/scanner.go b/cmd/scanner/scanner.go index 345ea32a..a3bf1e82 100644 --- a/cmd/scanner/scanner.go +++ b/cmd/scanner/scanner.go @@ -383,11 +383,14 @@ func (s *Scanner) postScannerInventory() error { return fmt.Errorf("error when posting Scanner inventory") } + s.logger.Info("Inventory posted correctly") + + // HTTP post to /inventory to refresh views if err := postData(apiInventoryEndpoint, []byte{}); err != nil { return err } - s.logger.Info("Inventory posted correctly") + s.logger.Info("Inventory refreshed correctly") return nil } diff --git a/internal/api/handlers/account_handler.go b/internal/api/handlers/account_handler.go index 49c5aecc..23be32c5 100644 --- a/internal/api/handlers/account_handler.go +++ b/internal/api/handlers/account_handler.go @@ -179,7 +179,7 @@ func (h *AccountHandler) GetAccountClustersByID(c *gin.Context) { // @Success 200 {object} dto.InstanceListResponse // @Failure 404 {object} responsetypes.GenericErrorResponse // @Failure 500 {object} responsetypes.GenericErrorResponse -// @Router /accounts/{id}/expense_update_instances [get] +// @Router /accounts/{id}/expense_update [get] // // NOTE: Align the documented route with the actual router configuration. func (h *AccountHandler) GetExpensesUpdateInstances(c *gin.Context) { diff --git a/internal/stocker/aws_billing_stocker.go b/internal/stocker/aws_billing_stocker.go index 2cfe66b9..f8ae4a4f 100644 --- a/internal/stocker/aws_billing_stocker.go +++ b/internal/stocker/aws_billing_stocker.go @@ -66,7 +66,7 @@ func (s *AWSBillingStocker) MakeStock() error { if err != nil { s.logger.Error("Error querying billing info for an instance", zap.String("account", s.Account.AccountID), - zap.String("instance_name", instance.InstanceName), + zap.String("instance_id", instance.InstanceID), zap.String("error", err.Error()), ) // Continue to the next region even if an error occurs @@ -91,7 +91,7 @@ func (s *AWSBillingStocker) getInstanceExpenses(instance *inventory.Instance) er s.logger.Debug("Getting expenses for instance", zap.String("account", s.Account.AccountName), - zap.String("instance_name", instance.InstanceName), + zap.String("instance_id", instance.InstanceID), zap.String("start_date", startDate), zap.String("end_date", endDate), ) @@ -106,7 +106,7 @@ func (s *AWSBillingStocker) getInstanceExpenses(instance *inventory.Instance) er Filter: &costexplorer.Expression{ Dimensions: &costexplorer.DimensionValues{ Key: aws.String("RESOURCE_ID"), - Values: []*string{aws.String(instance.InstanceName)}, + Values: []*string{aws.String(instance.InstanceID)}, }, }, Metrics: []*string{aws.String("UnblendedCost")}, @@ -117,7 +117,7 @@ func (s *AWSBillingStocker) getInstanceExpenses(instance *inventory.Instance) er if err != nil { s.logger.Error("Error getting cost and usage with resources", zap.String("account", s.Account.AccountName), - zap.String("instance_name", instance.InstanceName), + zap.String("instance_id", instance.InstanceID), zap.Error(err)) return err } From bb58405964a380451a20a8a635381372aab24e3a Mon Sep 17 00:00:00 2001 From: Alejandro Villegas Date: Mon, 26 Jan 2026 20:09:47 +0100 Subject: [PATCH 08/53] fix(db): correct schema inconsistencies and logical errors (#246) * fix: Applied fixes to DB schema Signed-off-by: r2dedios * doc: Updated migration guides Signed-off-by: r2dedios * fix: Updated VIEWs creation to overwrite previous definition Signed-off-by: r2dedios * fix: Fixed ResourceType management in DB tests and API repository Signed-off-by: r2dedios * fix: Replaced hardcoded 'cluster'/'instance' strings by ResourceType constant Signed-off-by: r2dedios * refact: Removed dangling file Signed-off-by: r2dedios * fix: Removed 'Name' and 'MachineName' from clusters_tags Signed-off-by: r2dedios --------- Signed-off-by: r2dedios --- db/conf/create_expenses_partition.sh | 7 - db/conf/create_expenses_partition.sql | 14 - db/sql/clean.sql | 22 -- db/sql/cron.sql | 3 +- db/sql/init.sql | 48 +-- db/sql/old_test_data.sql | 330 ---------------- db/sql/sql-backup.sql | 247 ------------ db/test_files/integration_tests_data.sql | 4 +- db/test_files/new-expense.json | 372 ------------------ db/test_files/new-instance.json | 22 -- .../v0.5.1/data-migration-0.5-0.5.1.md | 142 +++++++ .../v0.5}/data-migration-0.4.X-0.5.md | 27 +- .../event_service/event_service_test.go | 9 +- internal/inventory/cost_stats.go | 15 - internal/inventory/types.go | 6 +- internal/models/db/events_test.go | 17 +- internal/models/dto/event_dto_test.go | 13 +- internal/repositories/event_repository.go | 4 +- .../api_events_integration_test.go | 5 +- 19 files changed, 223 insertions(+), 1084 deletions(-) delete mode 100644 db/conf/create_expenses_partition.sh delete mode 100644 db/conf/create_expenses_partition.sql delete mode 100644 db/sql/clean.sql delete mode 100644 db/sql/old_test_data.sql delete mode 100644 db/sql/sql-backup.sql delete mode 100644 db/test_files/new-expense.json delete mode 100644 db/test_files/new-instance.json create mode 100644 doc/releases/v0.5.1/data-migration-0.5-0.5.1.md rename doc/{developers => releases/v0.5}/data-migration-0.4.X-0.5.md (95%) delete mode 100644 internal/inventory/cost_stats.go diff --git a/db/conf/create_expenses_partition.sh b/db/conf/create_expenses_partition.sh deleted file mode 100644 index 2144e3da..00000000 --- a/db/conf/create_expenses_partition.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -PGUSER=postgres -PGDATABASE=clusteriq -PGHOST=localhost - -psql -U "$PGUSER" -d "$PGDATABASE" -h "$PGHOST" -f /opt/scripts/create_expenses_partition.sql - diff --git a/db/conf/create_expenses_partition.sql b/db/conf/create_expenses_partition.sql deleted file mode 100644 index 70fa7a4f..00000000 --- a/db/conf/create_expenses_partition.sql +++ /dev/null @@ -1,14 +0,0 @@ -DO $$ -DECLARE - next_month DATE := date_trunc('month', current_date) + INTERVAL '1 month'; - partition_name TEXT := format('expenses_%s', to_char(next_month, 'YYYY_MM')); - start_date DATE := next_month; - end_date DATE := next_month + INTERVAL '1 month'; -BEGIN - EXECUTE format( - 'CREATE TABLE IF NOT EXISTS %I PARTITION OF expenses - FOR VALUES FROM (%L) TO (%L);', - partition_name, start_date, end_date - ); -END$$; - diff --git a/db/sql/clean.sql b/db/sql/clean.sql deleted file mode 100644 index 37b3095d..00000000 --- a/db/sql/clean.sql +++ /dev/null @@ -1,22 +0,0 @@ --- Drop Triggers -DROP TRIGGER update_instance_total_cost_after_insert ON expenses; -DROP TRIGGER update_instance_total_cost_after_delete ON expenses; -DROP TRIGGER update_instance_daily_cost_after_insert ON expenses; -DROP TRIGGER update_instance_daily_cost_after_delete ON expenses; -DROP TRIGGER update_cluster_total_cost ON instances; - --- Drop Functins -DROP FUNCTION update_instance_total_costs_after_insert; -DROP FUNCTION update_instance_total_costs_after_delete; -DROP FUNCTION update_instance_daily_costs_after_insert; -DROP FUNCTION update_instance_daily_costs_after_delete; -DROP FUNCTION update_cluster_total_costs; - --- Drop tables -DROP TABLE tags; -DROP TABLE expenses; -DROP TABLE instances; -DROP TABLE clusters; -DROP TABLE accounts; -DROP TABLE providers; -DROP TABLE status; diff --git a/db/sql/cron.sql b/db/sql/cron.sql index 1ea002f3..fe841b3a 100644 --- a/db/sql/cron.sql +++ b/db/sql/cron.sql @@ -1,7 +1,8 @@ \c postgres --- pg_cron task for updating the 'Terminated' elements in the inventory every 6 hours CREATE EXTENSION IF NOT EXISTS pg_cron; + +-- pg_cron task for updating the 'Terminated' elements in the inventory every 6 hours SELECT cron.schedule_in_database( 'check_terminated_inventory', '0 */6 * * *', diff --git a/db/sql/init.sql b/db/sql/init.sql index efa4f39b..4fd4293e 100644 --- a/db/sql/init.sql +++ b/db/sql/init.sql @@ -127,7 +127,7 @@ CREATE TABLE IF NOT EXISTS instances ( provider CLOUD_PROVIDER NOT NULL, availability_zone TEXT, status STATUS DEFAULT 'Unknown' NOT NULL, - cluster_id INTEGER REFERENCES clusters(id) ON DELETE CASCADE NOT NULL, + cluster_id BIGINT REFERENCES clusters(id) ON DELETE CASCADE NOT NULL, last_scan_ts TIMESTAMP WITH TIME ZONE, created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(), age INTEGER DEFAULT 0, @@ -194,12 +194,11 @@ CREATE TABLE IF NOT EXISTS events ( event_timestamp TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, triggered_by TEXT NOT NULL, action TEXT NOT NULL, - resource_id INTEGER, - resource_type TEXT NOT NULL, + resource_id BIGINT, + resource_type RESOURCE_TYPE NOT NULL, result ACTION_STATUS NOT NULL, description TEXT NULL, severity TEXT DEFAULT 'info'::TEXT NOT NULL, - CONSTRAINT events_resource_type_check CHECK ((resource_type = ANY (ARRAY['cluster'::TEXT, 'instance'::TEXT]))), PRIMARY KEY (id, event_timestamp) ) PARTITION BY RANGE (event_timestamp); @@ -244,13 +243,13 @@ CREATE INDEX IF NOT EXISTS ix_schedule_status ON schedule (status); -- ############################################################ -- Accounts Cluster Count view -CREATE VIEW accounts_with_cluster_count AS +CREATE OR REPLACE VIEW accounts_with_cluster_count AS SELECT c.account_id, COUNT(*)::bigint AS cluster_count FROM clusters c GROUP BY c.account_id; -- Accounts Costs view -CREATE VIEW accounts_with_costs AS +CREATE OR REPLACE VIEW accounts_with_costs AS WITH base AS ( SELECT a.id, e.date, e.amount FROM accounts a @@ -274,7 +273,7 @@ LEFT JOIN base b ON b.id = a.id GROUP BY a.id; -- Accounts Full view -CREATE VIEW accounts_full_view AS +CREATE OR REPLACE VIEW accounts_full_view AS SELECT a.account_id, a.account_name, @@ -299,13 +298,13 @@ CREATE MATERIALIZED VIEW m_accounts_full_view AS SELECT * FROM accounts_full_vie -- ############################################################ -- Cluster Instances Count view -CREATE VIEW clusters_with_instance_count AS +CREATE OR REPLACE VIEW clusters_with_instance_count AS SELECT i.cluster_id, COUNT(*)::bigint AS instance_count FROM instances i GROUP BY i.cluster_id; -- clusters Costs view -CREATE VIEW clusters_with_costs AS +CREATE OR REPLACE VIEW clusters_with_costs AS WITH base AS ( SELECT c.id, e.date, e.amount FROM clusters c @@ -327,7 +326,7 @@ LEFT JOIN base b ON b.id = c.id GROUP BY c.id; -- clusters Full view -CREATE VIEW clusters_full_view AS +CREATE OR REPLACE VIEW clusters_full_view AS SELECT c.cluster_id, c.cluster_name, @@ -355,7 +354,7 @@ LEFT JOIN clusters_with_costs ac ON ac.id = c.id; CREATE MATERIALIZED VIEW m_clusters_full_view AS SELECT * FROM clusters_full_view; -- cluster tags view. Returns the cluster_id + every tag omitting repeated tags keys -CREATE view clusters_tags AS +CREATE OR REPLACE VIEW clusters_tags AS SELECT c.cluster_id, t.key, @@ -363,6 +362,7 @@ SELECT FROM clusters c JOIN instances i ON i.cluster_id = c.id JOIN tags t ON t.instance_id = i.id +WHERE t.key != 'Name' AND t.key != 'MachineName' GROUP BY c.cluster_id, t.key HAVING COUNT(*) > 1; @@ -373,7 +373,7 @@ HAVING COUNT(*) > 1; -- ############################################################################# -- Instances Costs view -CREATE VIEW instances_with_costs AS +CREATE OR REPLACE VIEW instances_with_costs AS WITH base AS ( SELECT i.id, e.date, e.amount FROM instances i @@ -394,7 +394,7 @@ LEFT JOIN base b ON b.id = i.id GROUP BY i.id; -- Instances Full view -CREATE VIEW instances_full_view AS +CREATE OR REPLACE VIEW instances_full_view AS SELECT i.instance_id, i.instance_name, @@ -418,7 +418,7 @@ LEFT JOIN instances_with_costs ic ON ic.id = i.id; CREATE MATERIALIZED VIEW m_instances_full_view AS SELECT * FROM instances_full_view; -- Instances Full view -CREATE VIEW instances_full_view_with_tags AS +CREATE OR REPLACE VIEW instances_full_view_with_tags AS SELECT i.instance_id, i.instance_name, @@ -449,7 +449,7 @@ LEFT JOIN LATERAL ( CREATE MATERIALIZED VIEW m_instances_full_view_with_tags AS SELECT * FROM instances_full_view_with_tags; -- Instances pending for expense update -CREATE VIEW instances_pending_expense_update AS +CREATE OR REPLACE VIEW instances_pending_expense_update AS SELECT a.account_id, i.instance_id @@ -490,7 +490,7 @@ WHERE -- ############################################################################# -- Schedule with cluster and instances list view -CREATE VIEW schedule_full_view AS +CREATE OR REPLACE VIEW schedule_full_view AS SELECT s.id, s.type, @@ -547,7 +547,7 @@ $$; -- ############################################################################# -- View for Cluster Events -CREATE VIEW cluster_events AS +CREATE OR REPLACE VIEW cluster_events AS SELECT ev.id, ev.event_timestamp, @@ -559,12 +559,12 @@ SELECT ev.description, ev.severity FROM events ev -LEFT JOIN clusters c ON ev.resource_type = 'cluster' AND c.id = ev.resource_id -LEFT JOIN instances i ON ev.resource_type = 'instance' AND i.id = ev.resource_id +LEFT JOIN clusters c ON ev.resource_type = 'Cluster'::RESOURCE_TYPE AND c.id = ev.resource_id +LEFT JOIN instances i ON ev.resource_type = 'Instance'::RESOURCE_TYPE AND i.id = ev.resource_id ORDER BY event_timestamp DESC; -- View for System Events -CREATE VIEW system_events AS +CREATE OR REPLACE VIEW system_events AS SELECT ev.id, ev.event_timestamp, @@ -578,13 +578,13 @@ SELECT acc.account_id, acc.provider FROM events ev -LEFT JOIN clusters c ON ev.resource_type = 'cluster' AND c.id = ev.resource_id -LEFT JOIN instances i ON ev.resource_type = 'instance' AND i.id = ev.resource_id +LEFT JOIN clusters c ON ev.resource_type = 'Cluster'::RESOURCE_TYPE AND c.id = ev.resource_id +LEFT JOIN instances i ON ev.resource_type = 'Instance'::RESOURCE_TYPE AND i.id = ev.resource_id LEFT JOIN accounts acc ON acc.id = ( CASE - WHEN ev.resource_type = 'cluster' + WHEN ev.resource_type = 'Cluster'::RESOURCE_TYPE THEN (SELECT c.account_id FROM clusters c WHERE c.id = ev.resource_id) - WHEN ev.resource_type = 'instance' + WHEN ev.resource_type = 'Instance'::RESOURCE_TYPE THEN (SELECT c.account_id FROM clusters c WHERE c.id = (SELECT i.cluster_id FROM instances i WHERE i.id = ev.resource_id)) END ) diff --git a/db/sql/old_test_data.sql b/db/sql/old_test_data.sql deleted file mode 100644 index 27c5a162..00000000 --- a/db/sql/old_test_data.sql +++ /dev/null @@ -1,330 +0,0 @@ --- Cleaning -DELETE FROM tags; -DELETE FROM expenses; -DELETE FROM instances; -DELETE FROM clusters; -DELETE FROM accounts; -DELETE FROM providers; -DELETE FROM status; - --- Providers -INSERT INTO - providers(name) -VALUES - ('AWS'), - ('GCP'), - ('Azure'), - ('UNKNOWN') -; - - --- Status -INSERT INTO - status(value) -VALUES - ('Running'), - ('Stopped'), - ('Terminated'), - ('Unknown') -; - - --- Accounts -INSERT INTO - accounts (id, name, provider, cluster_count, last_scan_timestamp) -VALUES - ('ABC123', 'engineering', 'AWS', 2, TO_DATE('20/10/2021', 'DD/MM/YYYY')), - ('XYZ098', 'partners', 'GCP', 2, TO_DATE('20/10/2021', 'DD/MM/YYYY')), - ('FGH456', 'business', 'Azure', 2, TO_DATE('20/10/2021', 'DD/MM/YYYY')) -; - - --- Clusters -INSERT INTO - clusters (id, name, infra_id, provider, status, region, account_name, console_link, instance_count, last_scan_timestamp, creation_timestamp, age, owner, total_cost) -VALUES - ('cluster-A-A01-engineering', 'cluster-A', 'A01', 'AWS', 'Running', 'eu-west-1', 'engineering', 'http://console.cluster-A', 6, TO_DATE('24/06/2024', 'DD/MM/YYYY'), TO_DATE('20/06/2024', 'DD/MM/YYYY'), 4, 'John Doe 1', 0.0), - ('cluster-B-B02-engineering', 'cluster-B', 'B02', 'AWS', 'Stopped', 'eu-east-1', 'engineering', 'http://console.cluster-B', 6, TO_DATE('24/06/2024', 'DD/MM/YYYY'), TO_DATE('20/06/2024', 'DD/MM/YYYY'), 4, 'John Doe 2', 0.0), - ('cluster-C-C03-partners', 'cluster-C', 'C03', 'GCP', 'Running', 'eu-north-1', 'partners', 'http://console.cluster-C', 6, TO_DATE('24/06/2024', 'DD/MM/YYYY'), TO_DATE('20/06/2024', 'DD/MM/YYYY'), 4, 'John Doe 3', 0.0), - ('cluster-D-D04-partners', 'cluster-D', 'D04', 'GCP', 'Unknown', 'eu-south-1', 'partners', 'http://console.cluster-D', 6, TO_DATE('24/06/2024', 'DD/MM/YYYY'), TO_DATE('20/06/2024', 'DD/MM/YYYY'), 4, 'John Doe 4', 0.0), - ('cluster-E-E05-business', 'cluster-E', 'E05', 'Azure', 'Terminated', 'us-west-1', 'business', 'http://console.cluster-E', 6, TO_DATE('24/06/2024', 'DD/MM/YYYY'), TO_DATE('20/06/2024', 'DD/MM/YYYY'), 4, 'John Doe 5', 0.0), - ('cluster-F-F06-business', 'cluster-F', 'F06', 'Azure', 'Stopped', 'us-east-1', 'business', 'http://console.cluster-F', 6, TO_DATE('24/06/2024', 'DD/MM/YYYY'), TO_DATE('20/06/2024', 'DD/MM/YYYY'), 4, 'John Doe 6', 0.0) -; - - --- Instances (36) -INSERT INTO - instances (id, name, provider, instance_type, availability_zone, status, cluster_id, last_scan_timestamp, creation_timestamp, age, daily_cost, total_cost) -VALUES - ('id-001', 'instance-01-master', 'AWS', 't2.medium', 'eu-west-1', 'Running', 'cluster-A-A01-engineering', TO_DATE('24/06/2024', 'DD/MM/YYYY'), TO_DATE('20/06/2024', 'DD/MM/YYYY'), 4, 0.0, 0.0), - ('id-002', 'instance-02-master', 'AWS', 't2.medium', 'eu-west-1', 'Running', 'cluster-A-A01-engineering', TO_DATE('24/06/2024', 'DD/MM/YYYY'), TO_DATE('20/06/2024', 'DD/MM/YYYY'), 4, 0.0, 0.0), - ('id-003', 'instance-03-master', 'AWS', 't2.medium', 'eu-west-1', 'Running', 'cluster-A-A01-engineering', TO_DATE('24/06/2024', 'DD/MM/YYYY'), TO_DATE('20/06/2024', 'DD/MM/YYYY'), 4, 0.0, 0.0), - ('id-004', 'instance-04-worker', 'AWS', 't2.medium', 'eu-west-1', 'Running', 'cluster-A-A01-engineering', TO_DATE('24/06/2024', 'DD/MM/YYYY'), TO_DATE('20/06/2024', 'DD/MM/YYYY'), 4, 0.0, 0.0), - ('id-005', 'instance-05-worker', 'AWS', 't2.medium', 'eu-west-1', 'Running', 'cluster-A-A01-engineering', TO_DATE('24/06/2024', 'DD/MM/YYYY'), TO_DATE('20/06/2024', 'DD/MM/YYYY'), 4, 0.0, 0.0), - ('id-006', 'instance-06-worker', 'AWS', 't2.medium', 'eu-west-1', 'Running', 'cluster-A-A01-engineering', TO_DATE('24/06/2024', 'DD/MM/YYYY'), TO_DATE('20/06/2024', 'DD/MM/YYYY'), 4, 0.0, 0.0), - ('id-007', 'instance-01-master', 'AWS', 't2.medium', 'eu-west-1', 'Running', 'cluster-B-B02-engineering', TO_DATE('24/06/2024', 'DD/MM/YYYY'), TO_DATE('20/06/2024', 'DD/MM/YYYY'), 4, 0.0, 0.0), - ('id-008', 'instance-02-master', 'AWS', 't2.medium', 'eu-west-1', 'Running', 'cluster-B-B02-engineering', TO_DATE('24/06/2024', 'DD/MM/YYYY'), TO_DATE('20/06/2024', 'DD/MM/YYYY'), 4, 0.0, 0.0), - ('id-009', 'instance-03-master', 'AWS', 't2.medium', 'eu-west-1', 'Running', 'cluster-B-B02-engineering', TO_DATE('24/06/2024', 'DD/MM/YYYY'), TO_DATE('20/06/2024', 'DD/MM/YYYY'), 4, 0.0, 0.0), - ('id-010', 'instance-04-worker', 'AWS', 't2.medium', 'eu-west-1', 'Running', 'cluster-B-B02-engineering', TO_DATE('24/06/2024', 'DD/MM/YYYY'), TO_DATE('20/06/2024', 'DD/MM/YYYY'), 4, 0.0, 0.0), - ('id-011', 'instance-05-worker', 'AWS', 't2.medium', 'eu-west-1', 'Running', 'cluster-B-B02-engineering', TO_DATE('24/06/2024', 'DD/MM/YYYY'), TO_DATE('20/06/2024', 'DD/MM/YYYY'), 4, 0.0, 0.0), - ('id-012', 'instance-06-worker', 'AWS', 't2.medium', 'eu-west-1', 'Running', 'cluster-B-B02-engineering', TO_DATE('24/06/2024', 'DD/MM/YYYY'), TO_DATE('20/06/2024', 'DD/MM/YYYY'), 4, 0.0, 0.0), - ('id-013', 'instance-01-master', 'GCP', 't2.medium', 'eu-west-1', 'Running', 'cluster-C-C03-partners', TO_DATE('24/06/2024', 'DD/MM/YYYY'), TO_DATE('20/06/2024', 'DD/MM/YYYY'), 4, 0.0, 0.0), - ('id-014', 'instance-02-master', 'GCP', 't2.medium', 'eu-west-1', 'Running', 'cluster-C-C03-partners', TO_DATE('24/06/2024', 'DD/MM/YYYY'), TO_DATE('20/06/2024', 'DD/MM/YYYY'), 4, 0.0, 0.0), - ('id-015', 'instance-03-master', 'GCP', 't2.medium', 'eu-west-1', 'Running', 'cluster-C-C03-partners', TO_DATE('24/06/2024', 'DD/MM/YYYY'), TO_DATE('20/06/2024', 'DD/MM/YYYY'), 4, 0.0, 0.0), - ('id-016', 'instance-04-worker', 'GCP', 't2.medium', 'eu-west-1', 'Running', 'cluster-C-C03-partners', TO_DATE('24/06/2024', 'DD/MM/YYYY'), TO_DATE('20/06/2024', 'DD/MM/YYYY'), 4, 0.0, 0.0), - ('id-017', 'instance-05-worker', 'GCP', 't2.medium', 'eu-west-1', 'Running', 'cluster-C-C03-partners', TO_DATE('24/06/2024', 'DD/MM/YYYY'), TO_DATE('20/06/2024', 'DD/MM/YYYY'), 4, 0.0, 0.0), - ('id-018', 'instance-06-worker', 'GCP', 't2.medium', 'eu-west-1', 'Running', 'cluster-C-C03-partners', TO_DATE('24/06/2024', 'DD/MM/YYYY'), TO_DATE('20/06/2024', 'DD/MM/YYYY'), 4, 0.0, 0.0), - ('id-019', 'instance-01-master', 'GCP', 't2.medium', 'eu-west-1', 'Running', 'cluster-D-D04-partners', TO_DATE('24/06/2024', 'DD/MM/YYYY'), TO_DATE('20/06/2024', 'DD/MM/YYYY'), 4, 0.0, 0.0), - ('id-020', 'instance-02-master', 'GCP', 't2.medium', 'eu-west-1', 'Running', 'cluster-D-D04-partners', TO_DATE('24/06/2024', 'DD/MM/YYYY'), TO_DATE('20/06/2024', 'DD/MM/YYYY'), 4, 0.0, 0.0), - ('id-021', 'instance-03-master', 'GCP', 't2.medium', 'eu-west-1', 'Running', 'cluster-D-D04-partners', TO_DATE('24/06/2024', 'DD/MM/YYYY'), TO_DATE('20/06/2024', 'DD/MM/YYYY'), 4, 0.0, 0.0), - ('id-022', 'instance-04-worker', 'GCP', 't2.medium', 'eu-west-1', 'Running', 'cluster-D-D04-partners', TO_DATE('24/06/2024', 'DD/MM/YYYY'), TO_DATE('20/06/2024', 'DD/MM/YYYY'), 4, 0.0, 0.0), - ('id-023', 'instance-05-worker', 'GCP', 't2.medium', 'eu-west-1', 'Running', 'cluster-D-D04-partners', TO_DATE('24/06/2024', 'DD/MM/YYYY'), TO_DATE('20/06/2024', 'DD/MM/YYYY'), 4, 0.0, 0.0), - ('id-024', 'instance-06-worker', 'GCP', 't2.medium', 'eu-west-1', 'Running', 'cluster-D-D04-partners', TO_DATE('24/06/2024', 'DD/MM/YYYY'), TO_DATE('20/06/2024', 'DD/MM/YYYY'), 4, 0.0, 0.0), - ('id-025', 'instance-01-master', 'Azure', 't2.medium', 'eu-west-1', 'Running', 'cluster-E-E05-business', TO_DATE('24/06/2024', 'DD/MM/YYYY'), TO_DATE('20/06/2024', 'DD/MM/YYYY'), 4, 0.0, 0.0), - ('id-026', 'instance-02-master', 'Azure', 't2.medium', 'eu-west-1', 'Running', 'cluster-E-E05-business', TO_DATE('24/06/2024', 'DD/MM/YYYY'), TO_DATE('20/06/2024', 'DD/MM/YYYY'), 4, 0.0, 0.0), - ('id-027', 'instance-03-master', 'Azure', 't2.medium', 'eu-west-1', 'Running', 'cluster-E-E05-business', TO_DATE('24/06/2024', 'DD/MM/YYYY'), TO_DATE('20/06/2024', 'DD/MM/YYYY'), 4, 0.0, 0.0), - ('id-028', 'instance-04-worker', 'Azure', 't2.medium', 'eu-west-1', 'Running', 'cluster-E-E05-business', TO_DATE('24/06/2024', 'DD/MM/YYYY'), TO_DATE('20/06/2024', 'DD/MM/YYYY'), 4, 0.0, 0.0), - ('id-029', 'instance-05-worker', 'Azure', 't2.medium', 'eu-west-1', 'Running', 'cluster-E-E05-business', TO_DATE('24/06/2024', 'DD/MM/YYYY'), TO_DATE('20/06/2024', 'DD/MM/YYYY'), 4, 0.0, 0.0), - ('id-030', 'instance-06-worker', 'Azure', 't2.medium', 'eu-west-1', 'Running', 'cluster-E-E05-business', TO_DATE('24/06/2024', 'DD/MM/YYYY'), TO_DATE('20/06/2024', 'DD/MM/YYYY'), 4, 0.0, 0.0), - ('id-031', 'instance-01-master', 'Azure', 't2.medium', 'eu-west-1', 'Running', 'cluster-F-F06-business', TO_DATE('24/06/2024', 'DD/MM/YYYY'), TO_DATE('20/06/2024', 'DD/MM/YYYY'), 4, 0.0, 0.0), - ('id-032', 'instance-02-master', 'Azure', 't2.medium', 'eu-west-1', 'Running', 'cluster-F-F06-business', TO_DATE('24/06/2024', 'DD/MM/YYYY'), TO_DATE('20/06/2024', 'DD/MM/YYYY'), 4, 0.0, 0.0), - ('id-033', 'instance-03-master', 'Azure', 't2.medium', 'eu-west-1', 'Running', 'cluster-F-F06-business', TO_DATE('24/06/2024', 'DD/MM/YYYY'), TO_DATE('20/06/2024', 'DD/MM/YYYY'), 4, 0.0, 0.0), - ('id-034', 'instance-04-worker', 'Azure', 't2.medium', 'eu-west-1', 'Running', 'cluster-F-F06-business', TO_DATE('24/06/2024', 'DD/MM/YYYY'), TO_DATE('20/06/2024', 'DD/MM/YYYY'), 4, 0.0, 0.0), - ('id-035', 'instance-05-worker', 'Azure', 't2.medium', 'eu-west-1', 'Running', 'cluster-F-F06-business', TO_DATE('24/06/2024', 'DD/MM/YYYY'), TO_DATE('20/06/2024', 'DD/MM/YYYY'), 4, 0.0, 0.0), - ('id-036', 'instance-06-worker', 'Azure', 't2.medium', 'eu-west-1', 'Running', 'cluster-F-F06-business', TO_DATE('24/06/2024', 'DD/MM/YYYY'), TO_DATE('20/06/2024', 'DD/MM/YYYY'), 4, 0.0, 0.0) -; - - --- Expenses (72) -INSERT INTO - expenses (instance_id, date, amount) -VALUES - ('id-001', TO_DATE('20/06/2024', 'DD/MM/YYYY'), 8.25), - ('id-001', TO_DATE('21/06/2024', 'DD/MM/YYYY'), 12.35), - ('id-001', TO_DATE('22/06/2024', 'DD/MM/YYYY'), 12.35), - ('id-001', TO_DATE('24/06/2024', 'DD/MM/YYYY'), 10.12), - ('id-002', TO_DATE('20/06/2024', 'DD/MM/YYYY'), 8.25), - ('id-002', TO_DATE('21/06/2024', 'DD/MM/YYYY'), 12.35), - ('id-002', TO_DATE('22/06/2024', 'DD/MM/YYYY'), 12.35), - ('id-002', TO_DATE('24/06/2024', 'DD/MM/YYYY'), 10.12), - ('id-003', TO_DATE('20/06/2024', 'DD/MM/YYYY'), 8.25), - ('id-003', TO_DATE('21/06/2024', 'DD/MM/YYYY'), 12.35), - ('id-003', TO_DATE('22/06/2024', 'DD/MM/YYYY'), 12.35), - ('id-003', TO_DATE('24/06/2024', 'DD/MM/YYYY'), 10.12), - ('id-004', TO_DATE('20/06/2024', 'DD/MM/YYYY'), 8.25), - ('id-004', TO_DATE('21/06/2024', 'DD/MM/YYYY'), 12.35), - ('id-004', TO_DATE('22/06/2024', 'DD/MM/YYYY'), 12.35), - ('id-004', TO_DATE('24/06/2024', 'DD/MM/YYYY'), 10.12), - ('id-005', TO_DATE('20/06/2024', 'DD/MM/YYYY'), 8.25), - ('id-005', TO_DATE('21/06/2024', 'DD/MM/YYYY'), 12.35), - ('id-005', TO_DATE('22/06/2024', 'DD/MM/YYYY'), 12.35), - ('id-005', TO_DATE('24/06/2024', 'DD/MM/YYYY'), 10.12), - ('id-006', TO_DATE('20/06/2024', 'DD/MM/YYYY'), 8.25), - ('id-006', TO_DATE('21/06/2024', 'DD/MM/YYYY'), 12.35), - ('id-006', TO_DATE('22/06/2024', 'DD/MM/YYYY'), 12.35), - ('id-006', TO_DATE('24/06/2024', 'DD/MM/YYYY'), 10.12), - ('id-007', TO_DATE('20/06/2024', 'DD/MM/YYYY'), 8.25), - ('id-007', TO_DATE('21/06/2024', 'DD/MM/YYYY'), 12.35), - ('id-007', TO_DATE('22/06/2024', 'DD/MM/YYYY'), 12.35), - ('id-007', TO_DATE('24/06/2024', 'DD/MM/YYYY'), 10.12), - ('id-008', TO_DATE('20/06/2024', 'DD/MM/YYYY'), 8.25), - ('id-008', TO_DATE('21/06/2024', 'DD/MM/YYYY'), 12.35), - ('id-008', TO_DATE('22/06/2024', 'DD/MM/YYYY'), 12.35), - ('id-008', TO_DATE('24/06/2024', 'DD/MM/YYYY'), 10.12), - ('id-009', TO_DATE('20/06/2024', 'DD/MM/YYYY'), 8.25), - ('id-009', TO_DATE('21/06/2024', 'DD/MM/YYYY'), 12.35), - ('id-009', TO_DATE('22/06/2024', 'DD/MM/YYYY'), 12.35), - ('id-009', TO_DATE('24/06/2024', 'DD/MM/YYYY'), 10.12), - ('id-010', TO_DATE('20/06/2024', 'DD/MM/YYYY'), 8.25), - ('id-010', TO_DATE('21/06/2024', 'DD/MM/YYYY'), 12.35), - ('id-010', TO_DATE('22/06/2024', 'DD/MM/YYYY'), 12.35), - ('id-010', TO_DATE('24/06/2024', 'DD/MM/YYYY'), 10.12), - ('id-011', TO_DATE('20/06/2024', 'DD/MM/YYYY'), 8.25), - ('id-011', TO_DATE('21/06/2024', 'DD/MM/YYYY'), 12.35), - ('id-011', TO_DATE('22/06/2024', 'DD/MM/YYYY'), 12.35), - ('id-011', TO_DATE('24/06/2024', 'DD/MM/YYYY'), 10.12), - ('id-012', TO_DATE('20/06/2024', 'DD/MM/YYYY'), 8.25), - ('id-012', TO_DATE('21/06/2024', 'DD/MM/YYYY'), 12.35), - ('id-012', TO_DATE('22/06/2024', 'DD/MM/YYYY'), 12.35), - ('id-012', TO_DATE('24/06/2024', 'DD/MM/YYYY'), 10.12), - ('id-013', TO_DATE('20/06/2024', 'DD/MM/YYYY'), 8.25), - ('id-013', TO_DATE('21/06/2024', 'DD/MM/YYYY'), 12.35), - ('id-013', TO_DATE('22/06/2024', 'DD/MM/YYYY'), 12.35), - ('id-013', TO_DATE('24/06/2024', 'DD/MM/YYYY'), 10.12), - ('id-014', TO_DATE('20/06/2024', 'DD/MM/YYYY'), 8.25), - ('id-014', TO_DATE('21/06/2024', 'DD/MM/YYYY'), 12.35), - ('id-014', TO_DATE('22/06/2024', 'DD/MM/YYYY'), 12.35), - ('id-014', TO_DATE('24/06/2024', 'DD/MM/YYYY'), 10.12), - ('id-015', TO_DATE('20/06/2024', 'DD/MM/YYYY'), 8.25), - ('id-015', TO_DATE('21/06/2024', 'DD/MM/YYYY'), 12.35), - ('id-015', TO_DATE('22/06/2024', 'DD/MM/YYYY'), 12.35), - ('id-015', TO_DATE('24/06/2024', 'DD/MM/YYYY'), 10.12), - ('id-016', TO_DATE('20/06/2024', 'DD/MM/YYYY'), 8.25), - ('id-016', TO_DATE('21/06/2024', 'DD/MM/YYYY'), 12.35), - ('id-016', TO_DATE('22/06/2024', 'DD/MM/YYYY'), 12.35), - ('id-016', TO_DATE('24/06/2024', 'DD/MM/YYYY'), 10.12), - ('id-017', TO_DATE('20/06/2024', 'DD/MM/YYYY'), 8.25), - ('id-017', TO_DATE('21/06/2024', 'DD/MM/YYYY'), 12.35), - ('id-017', TO_DATE('22/06/2024', 'DD/MM/YYYY'), 12.35), - ('id-017', TO_DATE('24/06/2024', 'DD/MM/YYYY'), 10.12), - ('id-018', TO_DATE('20/06/2024', 'DD/MM/YYYY'), 8.25), - ('id-018', TO_DATE('21/06/2024', 'DD/MM/YYYY'), 12.35), - ('id-018', TO_DATE('22/06/2024', 'DD/MM/YYYY'), 12.35), - ('id-018', TO_DATE('24/06/2024', 'DD/MM/YYYY'), 10.12), - ('id-019', TO_DATE('20/06/2024', 'DD/MM/YYYY'), 8.25), - ('id-019', TO_DATE('21/06/2024', 'DD/MM/YYYY'), 12.35), - ('id-019', TO_DATE('22/06/2024', 'DD/MM/YYYY'), 12.35), - ('id-019', TO_DATE('24/06/2024', 'DD/MM/YYYY'), 10.12), - ('id-020', TO_DATE('20/06/2024', 'DD/MM/YYYY'), 8.25), - ('id-020', TO_DATE('21/06/2024', 'DD/MM/YYYY'), 12.35), - ('id-020', TO_DATE('22/06/2024', 'DD/MM/YYYY'), 12.35), - ('id-020', TO_DATE('24/06/2024', 'DD/MM/YYYY'), 10.12), - ('id-021', TO_DATE('20/06/2024', 'DD/MM/YYYY'), 8.25), - ('id-021', TO_DATE('21/06/2024', 'DD/MM/YYYY'), 12.35), - ('id-021', TO_DATE('22/06/2024', 'DD/MM/YYYY'), 12.35), - ('id-021', TO_DATE('24/06/2024', 'DD/MM/YYYY'), 10.12), - ('id-022', TO_DATE('20/06/2024', 'DD/MM/YYYY'), 8.25), - ('id-022', TO_DATE('21/06/2024', 'DD/MM/YYYY'), 12.35), - ('id-022', TO_DATE('22/06/2024', 'DD/MM/YYYY'), 12.35), - ('id-022', TO_DATE('24/06/2024', 'DD/MM/YYYY'), 10.12), - ('id-023', TO_DATE('20/06/2024', 'DD/MM/YYYY'), 8.25), - ('id-023', TO_DATE('21/06/2024', 'DD/MM/YYYY'), 12.35), - ('id-023', TO_DATE('22/06/2024', 'DD/MM/YYYY'), 12.35), - ('id-023', TO_DATE('24/06/2024', 'DD/MM/YYYY'), 10.12), - ('id-024', TO_DATE('20/06/2024', 'DD/MM/YYYY'), 8.25), - ('id-024', TO_DATE('21/06/2024', 'DD/MM/YYYY'), 12.35), - ('id-024', TO_DATE('22/06/2024', 'DD/MM/YYYY'), 12.35), - ('id-024', TO_DATE('24/06/2024', 'DD/MM/YYYY'), 10.12), - ('id-025', TO_DATE('20/06/2024', 'DD/MM/YYYY'), 8.25), - ('id-025', TO_DATE('21/06/2024', 'DD/MM/YYYY'), 12.35), - ('id-025', TO_DATE('22/06/2024', 'DD/MM/YYYY'), 12.35), - ('id-025', TO_DATE('24/06/2024', 'DD/MM/YYYY'), 10.12), - ('id-026', TO_DATE('20/06/2024', 'DD/MM/YYYY'), 8.25), - ('id-026', TO_DATE('21/06/2024', 'DD/MM/YYYY'), 12.35), - ('id-026', TO_DATE('22/06/2024', 'DD/MM/YYYY'), 12.35), - ('id-026', TO_DATE('24/06/2024', 'DD/MM/YYYY'), 10.12), - ('id-027', TO_DATE('20/06/2024', 'DD/MM/YYYY'), 8.25), - ('id-027', TO_DATE('21/06/2024', 'DD/MM/YYYY'), 12.35), - ('id-027', TO_DATE('22/06/2024', 'DD/MM/YYYY'), 12.35), - ('id-027', TO_DATE('24/06/2024', 'DD/MM/YYYY'), 10.12), - ('id-028', TO_DATE('20/06/2024', 'DD/MM/YYYY'), 8.25), - ('id-028', TO_DATE('21/06/2024', 'DD/MM/YYYY'), 12.35), - ('id-028', TO_DATE('22/06/2024', 'DD/MM/YYYY'), 12.35), - ('id-028', TO_DATE('24/06/2024', 'DD/MM/YYYY'), 10.12), - ('id-029', TO_DATE('20/06/2024', 'DD/MM/YYYY'), 8.25), - ('id-029', TO_DATE('21/06/2024', 'DD/MM/YYYY'), 12.35), - ('id-029', TO_DATE('22/06/2024', 'DD/MM/YYYY'), 12.35), - ('id-029', TO_DATE('24/06/2024', 'DD/MM/YYYY'), 10.12), - ('id-030', TO_DATE('20/06/2024', 'DD/MM/YYYY'), 8.25), - ('id-030', TO_DATE('21/06/2024', 'DD/MM/YYYY'), 12.35), - ('id-030', TO_DATE('22/06/2024', 'DD/MM/YYYY'), 12.35), - ('id-030', TO_DATE('24/06/2024', 'DD/MM/YYYY'), 10.12), - ('id-031', TO_DATE('20/06/2024', 'DD/MM/YYYY'), 8.25), - ('id-031', TO_DATE('21/06/2024', 'DD/MM/YYYY'), 12.35), - ('id-031', TO_DATE('22/06/2024', 'DD/MM/YYYY'), 12.35), - ('id-031', TO_DATE('24/06/2024', 'DD/MM/YYYY'), 10.12), - ('id-032', TO_DATE('20/06/2024', 'DD/MM/YYYY'), 8.25), - ('id-032', TO_DATE('21/06/2024', 'DD/MM/YYYY'), 12.35), - ('id-032', TO_DATE('22/06/2024', 'DD/MM/YYYY'), 12.35), - ('id-032', TO_DATE('24/06/2024', 'DD/MM/YYYY'), 10.12), - ('id-033', TO_DATE('20/06/2024', 'DD/MM/YYYY'), 8.25), - ('id-033', TO_DATE('21/06/2024', 'DD/MM/YYYY'), 12.35), - ('id-033', TO_DATE('22/06/2024', 'DD/MM/YYYY'), 12.35), - ('id-033', TO_DATE('24/06/2024', 'DD/MM/YYYY'), 10.12), - ('id-034', TO_DATE('20/06/2024', 'DD/MM/YYYY'), 8.25), - ('id-034', TO_DATE('21/06/2024', 'DD/MM/YYYY'), 12.35), - ('id-034', TO_DATE('22/06/2024', 'DD/MM/YYYY'), 12.35), - ('id-034', TO_DATE('24/06/2024', 'DD/MM/YYYY'), 10.12), - ('id-035', TO_DATE('20/06/2024', 'DD/MM/YYYY'), 8.25), - ('id-035', TO_DATE('21/06/2024', 'DD/MM/YYYY'), 12.35), - ('id-035', TO_DATE('22/06/2024', 'DD/MM/YYYY'), 12.35), - ('id-035', TO_DATE('24/06/2024', 'DD/MM/YYYY'), 10.12), - ('id-036', TO_DATE('20/06/2024', 'DD/MM/YYYY'), 8.25), - ('id-036', TO_DATE('21/06/2024', 'DD/MM/YYYY'), 12.35), - ('id-036', TO_DATE('22/06/2024', 'DD/MM/YYYY'), 12.35), - ('id-036', TO_DATE('24/06/2024', 'DD/MM/YYYY'), 10.12) -; - - -INSERT INTO - tags(key, value, instance_id) -values - ('key01', 'value', 'id-001'), - ('key02', 'value', 'id-001'), - ('key01', 'value', 'id-002'), - ('key02', 'value', 'id-002'), - ('key01', 'value', 'id-003'), - ('key02', 'value', 'id-003'), - ('key01', 'value', 'id-004'), - ('key02', 'value', 'id-004'), - ('key01', 'value', 'id-005'), - ('key02', 'value', 'id-005'), - ('key01', 'value', 'id-006'), - ('key02', 'value', 'id-006'), - ('key01', 'value', 'id-007'), - ('key02', 'value', 'id-007'), - ('key01', 'value', 'id-008'), - ('key02', 'value', 'id-008'), - ('key01', 'value', 'id-009'), - ('key02', 'value', 'id-009'), - ('key01', 'value', 'id-010'), - ('key02', 'value', 'id-010'), - ('key01', 'value', 'id-011'), - ('key02', 'value', 'id-011'), - ('key01', 'value', 'id-012'), - ('key02', 'value', 'id-012'), - ('key01', 'value', 'id-013'), - ('key02', 'value', 'id-013'), - ('key01', 'value', 'id-014'), - ('key02', 'value', 'id-014'), - ('key01', 'value', 'id-015'), - ('key02', 'value', 'id-015'), - ('key01', 'value', 'id-016'), - ('key02', 'value', 'id-016'), - ('key01', 'value', 'id-017'), - ('key02', 'value', 'id-017'), - ('key01', 'value', 'id-018'), - ('key02', 'value', 'id-018'), - ('key01', 'value', 'id-019'), - ('key02', 'value', 'id-019'), - ('key01', 'value', 'id-020'), - ('key02', 'value', 'id-020'), - ('key01', 'value', 'id-021'), - ('key02', 'value', 'id-021'), - ('key01', 'value', 'id-022'), - ('key02', 'value', 'id-022'), - ('key01', 'value', 'id-023'), - ('key02', 'value', 'id-023'), - ('key01', 'value', 'id-024'), - ('key02', 'value', 'id-024'), - ('key01', 'value', 'id-025'), - ('key02', 'value', 'id-025'), - ('key01', 'value', 'id-026'), - ('key02', 'value', 'id-026'), - ('key01', 'value', 'id-027'), - ('key02', 'value', 'id-027'), - ('key01', 'value', 'id-028'), - ('key02', 'value', 'id-028'), - ('key01', 'value', 'id-029'), - ('key02', 'value', 'id-029'), - ('key01', 'value', 'id-030'), - ('key02', 'value', 'id-030'), - ('key01', 'value', 'id-031'), - ('key02', 'value', 'id-031'), - ('key01', 'value', 'id-032'), - ('key02', 'value', 'id-032'), - ('key01', 'value', 'id-033'), - ('key02', 'value', 'id-033'), - ('key01', 'value', 'id-034'), - ('key02', 'value', 'id-034'), - ('key01', 'value', 'id-035'), - ('key02', 'value', 'id-035'), - ('key01', 'value', 'id-036'), - ('key02', 'value', 'id-036') -; - - -SELECT count(*) FROM accounts; -SELECT count(*) FROM clusters; -SELECT count(*) FROM instances; -SELECT count(*) FROM expenses; diff --git a/db/sql/sql-backup.sql b/db/sql/sql-backup.sql deleted file mode 100644 index 97208fd5..00000000 --- a/db/sql/sql-backup.sql +++ /dev/null @@ -1,247 +0,0 @@ - - --- ## Functions ## --- Updates the total cost of an instance after a new expense record is inserted -CREATE OR REPLACE FUNCTION update_instance_total_costs_after_insert() - RETURNS TRIGGER - LANGUAGE PLPGSQL - AS -$$ -BEGIN - UPDATE instances - SET - total_cost = ( - SELECT SUM(amount) - FROM expenses - WHERE instance_id = NEW.instance_id - ) - WHERE id = NEW.instance_id; - RETURN NEW; -END; -$$; - - --- Updates the total cost of an instance after an expense record is deleted -CREATE OR REPLACE FUNCTION update_instance_total_costs_after_delete() - RETURNS TRIGGER - LANGUAGE PLPGSQL - AS -$$ -BEGIN - UPDATE instances - SET - total_cost = ( - SELECT SUM(amount) - FROM expenses - WHERE instance_id = OLD.instance_id - ) - WHERE id = OLD.instance_id; - RETURN OLD; -END; -$$; - - --- Updates the daily cost of an instance after a new expense record is inserted -CREATE OR REPLACE FUNCTION update_instance_daily_costs_after_insert() - RETURNS TRIGGER - LANGUAGE PLPGSQL - AS -$$ -BEGIN - UPDATE instances - SET - daily_cost = ( - SELECT COALESCE(SUM(amount)/NULLIF(COUNT(*), 0), 0) - FROM expenses - WHERE instance_id = NEW.instance_id - ) - WHERE id = NEW.instance_id; - RETURN NEW; -END; -$$; - - --- Updates the daily cost of an instance after an expense record is deleted -CREATE OR REPLACE FUNCTION update_instance_daily_costs_after_delete() - RETURNS TRIGGER - LANGUAGE PLPGSQL - AS -$$ -BEGIN - UPDATE instances - SET - daily_cost = ( - SELECT COALESCE(SUM(amount)/NULLIF(COUNT(*), 0), 0) - FROM expenses - WHERE instance_id = NEW.instance_id - ) - WHERE id = OLD.instance_id; - RETURN OLD; -END; -$$; - - --- Updates the total cost of a cluster based on its associated instances -CREATE OR REPLACE FUNCTION update_cluster_cost_info() - RETURNS TRIGGER - LANGUAGE PLPGSQL - AS -$$ -BEGIN - UPDATE clusters - SET - total_cost = ( - SELECT COALESCE(SUM(total_cost), 0) as sum - FROM instances - WHERE cluster_id = NEW.cluster_id - ), - last_15_days_cost = ( - SELECT COALESCE(SUM(expenses.amount), 0) - FROM instances - JOIN expenses ON instances.id = expenses.instance_id - WHERE instances.cluster_id = NEW.cluster_id - AND expenses.date >= NOW()::date - interval '15 day' - ), - last_month_cost = ( - SELECT COALESCE(SUM(expenses.amount), 0) - FROM instances - JOIN expenses ON instances.id = expenses.instance_id - WHERE instances.cluster_id = NEW.cluster_id - AND EXTRACT(YEAR FROM NOW()::date - interval '1 month') = EXTRACT(YEAR FROM expenses.date) - AND EXTRACT(MONTH FROM NOW()::date - interval '1 month') = EXTRACT(MONTH FROM expenses.date) - ), - current_month_so_far_cost = ( - SELECT COALESCE(SUM(expenses.amount), 0) - FROM instances - JOIN expenses ON instances.id = expenses.instance_id - WHERE instances.cluster_id = NEW.cluster_id - AND (EXTRACT(MONTH FROM NOW()::date) = EXTRACT(MONTH FROM expenses.date) - ) - ) - WHERE id = NEW.cluster_id; - RETURN NEW; -END; -$$; - - --- Updates the total cost of an account based on its associated clusters -CREATE OR REPLACE FUNCTION update_account_cost_info() - RETURNS TRIGGER - LANGUAGE PLPGSQL - AS -$$ -BEGIN - UPDATE accounts - SET - total_cost = ( - SELECT COALESCE(SUM(clusters.total_cost), 0) - FROM clusters - WHERE account_name = NEW.account_name - ), - last_15_days_cost = ( - SELECT COALESCE(SUM(clusters.last_15_days_cost), 0) - FROM clusters - WHERE account_name = NEW.account_name - ), - last_month_cost = ( - SELECT COALESCE(SUM(clusters.last_month_cost), 0) - FROM clusters - WHERE account_name = NEW.account_name - ), - current_month_so_far_cost = ( - SELECT COALESCE(SUM(clusters.current_month_so_far_cost), 0) - FROM clusters - WHERE account_name = NEW.account_name - ) - WHERE name = NEW.account_name; - RETURN NEW; -END; -$$; - - --- ## Maintenance Functions ## --- Marks instances as 'Terminated' if they haven't been scanned in the last 24 hours -CREATE OR REPLACE FUNCTION check_terminated_instances() -RETURNS void AS $$ -BEGIN - UPDATE instances - SET status = 'Terminated' - WHERE last_scan_timestamp < NOW() - INTERVAL '1 day'; -END; -$$ LANGUAGE plpgsql; - - --- Marks clusters as 'Terminated' if they haven't been scanned in the last 24 hours -CREATE OR REPLACE FUNCTION check_terminated_clusters() -RETURNS void AS $$ -BEGIN - UPDATE clusters - SET status = 'Terminated' - WHERE last_scan_timestamp < NOW() - INTERVAL '1 day'; -END; -$$ LANGUAGE plpgsql; - - --- ## Triggers ## --- Trigger to update instance total cost after an expense is inserted -CREATE TRIGGER update_instance_total_cost_after_insert -AFTER INSERT -ON expenses -FOR EACH ROW - EXECUTE PROCEDURE update_instance_total_costs_after_insert(); - - --- Trigger to update instance total cost after an expense is updated -CREATE TRIGGER update_instance_total_cost_after_update -AFTER UPDATE -ON expenses -FOR EACH ROW - EXECUTE PROCEDURE update_instance_total_costs_after_insert(); - - --- Trigger to update instance total cost after an expense is deleted -CREATE TRIGGER update_instance_total_cost_after_delete -AFTER DELETE -ON expenses -FOR EACH ROW - EXECUTE PROCEDURE update_instance_total_costs_after_delete(); - - --- Trigger to update instance daily cost after an expense is inserted -CREATE TRIGGER update_instance_daily_cost_after_insert -AFTER INSERT -ON expenses -FOR EACH ROW - EXECUTE PROCEDURE update_instance_daily_costs_after_insert(); - - --- Trigger to update instance daily cost after an expense is updated -CREATE TRIGGER update_instance_daily_cost_after_update -AFTER UPDATE -ON expenses -FOR EACH ROW - EXECUTE PROCEDURE update_instance_daily_costs_after_insert(); - - --- Trigger to update instance daily cost after an expense is deleted -CREATE TRIGGER update_instance_daily_cost_after_delete -AFTER DELETE -ON expenses -FOR EACH ROW - EXECUTE PROCEDURE update_instance_daily_costs_after_delete(); - - --- Trigger to update cluster costs info -CREATE TRIGGER update_cluster_cost_info -AFTER UPDATE -ON instances -FOR EACH ROW - EXECUTE PROCEDURE update_cluster_cost_info(); - - --- Trigger to update account total cost after a cluster is updated -CREATE TRIGGER update_account_cost_info -AFTER UPDATE -ON clusters -FOR EACH ROW - EXECUTE PROCEDURE update_account_cost_info(); diff --git a/db/test_files/integration_tests_data.sql b/db/test_files/integration_tests_data.sql index a8e5d062..cbf25640 100644 --- a/db/test_files/integration_tests_data.sql +++ b/db/test_files/integration_tests_data.sql @@ -71,8 +71,8 @@ INSERT INTO expenses (instance_id, date, amount) VALUES INSERT INTO events (event_timestamp, triggered_by, action, resource_id, resource_type, result, description, severity) VALUES - ('2025-08-02 12:00:00+00', 'cluster-iq-tester', 'test', '1', 'cluster', 'Success', 'integration test event', 'info'), - ('2025-08-02 12:00:00+00', 'cluster-iq-tester', 'test', '10', 'instance', 'Pending', 'integration test event', 'critical'); + ('2025-08-02 12:00:00+00', 'cluster-iq-tester', 'test', '1', 'Cluster', 'Success', 'integration test event', 'info'), + ('2025-08-02 12:00:00+00', 'cluster-iq-tester', 'test', '10', 'Instance', 'Pending', 'integration test event', 'critical'); INSERT INTO schedule (type, time, operation, target, status, enabled) VALUES diff --git a/db/test_files/new-expense.json b/db/test_files/new-expense.json deleted file mode 100644 index 91490cfa..00000000 --- a/db/test_files/new-expense.json +++ /dev/null @@ -1,372 +0,0 @@ -[ - { - "instanceID": "id-001", - "amount": 10.1, - "date": "2022-06-10T00:00:00Z" - }, - { - "instanceID": "id-001", - "amount": 12.4, - "date": "2022-06-11T00:00:00Z" - }, - { - "instanceID": "id-002", - "amount": 10.1, - "date": "2022-06-10T00:00:00Z" - }, - { - "instanceID": "id-002", - "amount": 12.4, - "date": "2022-06-11T00:00:00Z" - }, - { - "instanceID": "id-002", - "amount": 10.1, - "date": "2022-06-10T00:00:00Z" - }, - { - "instanceID": "id-002", - "amount": 12.4, - "date": "2022-06-11T00:00:00Z" - }, - { - "instanceID": "id-003", - "amount": 10.1, - "date": "2022-06-10T00:00:00Z" - }, - { - "instanceID": "id-003", - "amount": 12.4, - "date": "2022-06-11T00:00:00Z" - }, - { - "instanceID": "id-004", - "amount": 10.1, - "date": "2022-06-10T00:00:00Z" - }, - { - "instanceID": "id-004", - "amount": 12.4, - "date": "2022-06-11T00:00:00Z" - }, - { - "instanceID": "id-005", - "amount": 10.1, - "date": "2022-06-10T00:00:00Z" - }, - { - "instanceID": "id-005", - "amount": 12.4, - "date": "2022-06-11T00:00:00Z" - }, - { - "instanceID": "id-006", - "amount": 10.1, - "date": "2022-06-10T00:00:00Z" - }, - { - "instanceID": "id-006", - "amount": 12.4, - "date": "2022-06-11T00:00:00Z" - }, - { - "instanceID": "id-007", - "amount": 10.1, - "date": "2022-06-10T00:00:00Z" - }, - { - "instanceID": "id-007", - "amount": 12.4, - "date": "2022-06-11T00:00:00Z" - }, - { - "instanceID": "id-008", - "amount": 10.1, - "date": "2022-06-10T00:00:00Z" - }, - { - "instanceID": "id-008", - "amount": 12.4, - "date": "2022-06-11T00:00:00Z" - }, - { - "instanceID": "id-009", - "amount": 10.1, - "date": "2022-06-10T00:00:00Z" - }, - { - "instanceID": "id-009", - "amount": 12.4, - "date": "2022-06-11T00:00:00Z" - }, - { - "instanceID": "id-010", - "amount": 10.1, - "date": "2022-06-10T00:00:00Z" - }, - { - "instanceID": "id-010", - "amount": 12.4, - "date": "2022-06-11T00:00:00Z" - }, - { - "instanceID": "id-011", - "amount": 10.1, - "date": "2022-06-10T00:00:00Z" - }, - { - "instanceID": "id-011", - "amount": 12.4, - "date": "2022-06-11T00:00:00Z" - }, - { - "instanceID": "id-012", - "amount": 10.1, - "date": "2022-06-10T00:00:00Z" - }, - { - "instanceID": "id-012", - "amount": 12.4, - "date": "2022-06-11T00:00:00Z" - }, - { - "instanceID": "id-013", - "amount": 10.1, - "date": "2022-06-10T00:00:00Z" - }, - { - "instanceID": "id-013", - "amount": 12.4, - "date": "2022-06-11T00:00:00Z" - }, - { - "instanceID": "id-014", - "amount": 10.1, - "date": "2022-06-10T00:00:00Z" - }, - { - "instanceID": "id-014", - "amount": 12.4, - "date": "2022-06-11T00:00:00Z" - }, - { - "instanceID": "id-015", - "amount": 10.1, - "date": "2022-06-10T00:00:00Z" - }, - { - "instanceID": "id-015", - "amount": 12.4, - "date": "2022-06-11T00:00:00Z" - }, - { - "instanceID": "id-016", - "amount": 10.1, - "date": "2022-06-10T00:00:00Z" - }, - { - "instanceID": "id-016", - "amount": 12.4, - "date": "2022-06-11T00:00:00Z" - }, - { - "instanceID": "id-017", - "amount": 10.1, - "date": "2022-06-10T00:00:00Z" - }, - { - "instanceID": "id-017", - "amount": 12.4, - "date": "2022-06-11T00:00:00Z" - }, - { - "instanceID": "id-018", - "amount": 10.1, - "date": "2022-06-10T00:00:00Z" - }, - { - "instanceID": "id-018", - "amount": 12.4, - "date": "2022-06-11T00:00:00Z" - }, - { - "instanceID": "id-019", - "amount": 10.1, - "date": "2022-06-10T00:00:00Z" - }, - { - "instanceID": "id-019", - "amount": 12.4, - "date": "2022-06-11T00:00:00Z" - }, - { - "instanceID": "id-020", - "amount": 10.1, - "date": "2022-06-10T00:00:00Z" - }, - { - "instanceID": "id-020", - "amount": 12.4, - "date": "2022-06-11T00:00:00Z" - }, - { - "instanceID": "id-021", - "amount": 10.1, - "date": "2022-06-10T00:00:00Z" - }, - { - "instanceID": "id-021", - "amount": 12.4, - "date": "2022-06-11T00:00:00Z" - }, - { - "instanceID": "id-022", - "amount": 10.1, - "date": "2022-06-10T00:00:00Z" - }, - { - "instanceID": "id-022", - "amount": 12.4, - "date": "2022-06-11T00:00:00Z" - }, - { - "instanceID": "id-023", - "amount": 10.1, - "date": "2022-06-10T00:00:00Z" - }, - { - "instanceID": "id-023", - "amount": 12.4, - "date": "2022-06-11T00:00:00Z" - }, - { - "instanceID": "id-024", - "amount": 10.1, - "date": "2022-06-10T00:00:00Z" - }, - { - "instanceID": "id-024", - "amount": 12.4, - "date": "2022-06-11T00:00:00Z" - }, - { - "instanceID": "id-025", - "amount": 10.1, - "date": "2022-06-10T00:00:00Z" - }, - { - "instanceID": "id-025", - "amount": 12.4, - "date": "2022-06-11T00:00:00Z" - }, - { - "instanceID": "id-026", - "amount": 10.1, - "date": "2022-06-10T00:00:00Z" - }, - { - "instanceID": "id-026", - "amount": 12.4, - "date": "2022-06-11T00:00:00Z" - }, - { - "instanceID": "id-027", - "amount": 10.1, - "date": "2022-06-10T00:00:00Z" - }, - { - "instanceID": "id-027", - "amount": 12.4, - "date": "2022-06-11T00:00:00Z" - }, - { - "instanceID": "id-028", - "amount": 10.1, - "date": "2022-06-10T00:00:00Z" - }, - { - "instanceID": "id-028", - "amount": 12.4, - "date": "2022-06-11T00:00:00Z" - }, - { - "instanceID": "id-029", - "amount": 10.1, - "date": "2022-06-10T00:00:00Z" - }, - { - "instanceID": "id-029", - "amount": 12.4, - "date": "2022-06-11T00:00:00Z" - }, - { - "instanceID": "id-030", - "amount": 10.1, - "date": "2022-06-10T00:00:00Z" - }, - { - "instanceID": "id-030", - "amount": 12.4, - "date": "2022-06-11T00:00:00Z" - }, - { - "instanceID": "id-031", - "amount": 10.1, - "date": "2022-06-10T00:00:00Z" - }, - { - "instanceID": "id-031", - "amount": 12.4, - "date": "2022-06-11T00:00:00Z" - }, - { - "instanceID": "id-032", - "amount": 10.1, - "date": "2022-06-10T00:00:00Z" - }, - { - "instanceID": "id-032", - "amount": 12.4, - "date": "2022-06-11T00:00:00Z" - }, - { - "instanceID": "id-033", - "amount": 10.1, - "date": "2022-06-10T00:00:00Z" - }, - { - "instanceID": "id-033", - "amount": 12.4, - "date": "2022-06-11T00:00:00Z" - }, - { - "instanceID": "id-034", - "amount": 10.1, - "date": "2022-06-10T00:00:00Z" - }, - { - "instanceID": "id-034", - "amount": 12.4, - "date": "2022-06-11T00:00:00Z" - }, - { - "instanceID": "id-035", - "amount": 10.1, - "date": "2022-06-10T00:00:00Z" - }, - { - "instanceID": "id-035", - "amount": 12.4, - "date": "2022-06-11T00:00:00Z" - }, - { - "instanceID": "id-036", - "amount": 10.1, - "date": "2022-06-10T00:00:00Z" - }, - { - "instanceID": "id-036", - "amount": 12.4, - "date": "2022-06-11T00:00:00Z" - } -] diff --git a/db/test_files/new-instance.json b/db/test_files/new-instance.json deleted file mode 100644 index ae28757a..00000000 --- a/db/test_files/new-instance.json +++ /dev/null @@ -1,22 +0,0 @@ -[ - { - "id": "id-A01", - "name": "instance-01-master", - "provider": "AWS", - "instanceType": "t2.large", - "region": "eu-west-1", - "status": "Running", - "clusterName": "cluster-01", - "tags": null - }, - { - "id": "id-A02", - "name": "instance-02-master", - "provider": "AWS", - "instanceType": "t2.large", - "region": "eu-west-1", - "status": "Running", - "clusterName": "cluster-01", - "tags": null - } -] diff --git a/doc/releases/v0.5.1/data-migration-0.5-0.5.1.md b/doc/releases/v0.5.1/data-migration-0.5-0.5.1.md new file mode 100644 index 00000000..a0b02a4f --- /dev/null +++ b/doc/releases/v0.5.1/data-migration-0.5-0.5.1.md @@ -0,0 +1,142 @@ +# ClusterIQ Data Migration Guide (v0.4.X → v0.5) +This document defines a deterministic, manual procedure to migrate ClusterIQ +data from version `0.5` to `0.5.1`. + +The migration is performed directly in the target database: +- Apply DB schema transformations + +## Scope +This procedure covers the data migration from ClusterIQ `0.5` to `0.5.1`. + +This procedure does NOT: +- Deploy ClusterIQ +- Perform application upgrades +- Modify ClusterIQ configuration outside the database +- Execute automated CI/CD tasks +- Origin Database is running in Openshift +- Destination Database is running in the Development setup (podman-compose) + +## Preconditions +- Source ClusterIQ version is `0.5` +- pgsql deployment using `v0.5.1` image tag +- Operator has access to OpenShift and PostgreSQL +- `oc`, `psql`, and `podman` CLIs are installed and configured +- No write operations are running against the source database + +## Procedure Checklist + +### Pre-checks + +* [ ] **M0** — Prepare CLI and Namespace + ```sh + # Login into Openshift + oc login … + + # Define Namespace + export CIQ_NAMESPACE="" + oc project $CIQ_NAMESPACE + + # Stop Scanner to prevent updates during the data migration + oc patch cronjob scanner -p '{"spec" : {"suspend" : true }}' --type=merge -n $CIQ_NAMESPACE + ``` + +* [ ] **M1** — Connect to the target Database (assuming it runs in OCP). + ```sh + oc rsh pgsql-0 -n $CIQ_NAMESPACE + + psql -d clusteriq + ``` + +* [ ] **M2** — Check `RESOURCE_TYPE` enum exists. + ```sql + clusteriq=# \dT+ RESOURCE_TYPE + ``` + **Expected result:** + ```sql + List of data types + Schema | Name | Internal name | Size | Elements | Owner | Access privileges | Description + --------+---------------+---------------+------+----------+-------+-------------------+------------- + public | resource_type | resource_type | 4 | Account +| user | | + | | | | Cluster +| | | + | | | | Instance | | | + (1 row) + + ``` + +* [ ] **M3** — Modify instances table. + ```sql + -- Count before modifying + SELECT COUNT(*) FROM instances; + + -- Tx + BEGIN; + + -- 1. Drop FK + ALTER TABLE instances DROP CONSTRAINT instances_cluster_id_fkey; + + -- 2. Change column type + ALTER TABLE instances + ALTER COLUMN cluster_id TYPE BIGINT + USING cluster_id::bigint; + + -- 3. Recreate FK + ALTER TABLE instances + ADD CONSTRAINT instances_cluster_id_fkey + FOREIGN KEY (cluster_id) REFERENCES clusters(id) + ON DELETE CASCADE; + + COMMIT; + + -- Count after modifying must be same result as before + SELECT COUNT(*) FROM instances; + ``` + + **Expected result:** Alter table tx correct + +* [ ] **M4** — Modify events table. + ```sql + -- Count before modifying + SELECT COUNT(*) FROM instances; + + -- Tx + BEGIN; + + -- 1) Drop legacy CHECK constraint (if present) + ALTER TABLE public.events + DROP CONSTRAINT IF EXISTS events_resource_type_check; + + -- 2) resource_id: INTEGER -> BIGINT + ALTER TABLE public.events + ALTER COLUMN resource_id TYPE BIGINT + USING resource_id::bigint; + + -- 3) resource_type: TEXT -> RESOURCE_TYPE (align with enum: Account/Cluster/Instance) + ALTER TABLE public.events + ALTER COLUMN resource_type TYPE public.resource_type + USING ( + CASE resource_type + WHEN 'cluster' THEN 'Cluster'::public.resource_type + WHEN 'Cluster' THEN 'Cluster'::public.resource_type + WHEN 'instance' THEN 'Instance'::public.resource_type + WHEN 'Instance' THEN 'Instance'::public.resource_type + WHEN 'account' THEN 'Account'::public.resource_type + WHEN 'Account' THEN 'Account'::public.resource_type + ELSE resource_type::public.resource_type + END + ); + + -- 4) Keep NOT NULL (it was NOT NULL already, but explicit is fine) + ALTER TABLE public.events + ALTER COLUMN resource_type SET NOT NULL; + + COMMIT; + + -- Count after modifying must be same result as before + SELECT COUNT(*) FROM instances; + ``` + + **Expected result:** Alter table tx correct + +## Postconditions +- Tables modified correctly. +- Views re-created. diff --git a/doc/developers/data-migration-0.4.X-0.5.md b/doc/releases/v0.5/data-migration-0.4.X-0.5.md similarity index 95% rename from doc/developers/data-migration-0.4.X-0.5.md rename to doc/releases/v0.5/data-migration-0.4.X-0.5.md index 50038a52..ba8e4eaf 100644 --- a/doc/developers/data-migration-0.4.X-0.5.md +++ b/doc/releases/v0.5/data-migration-0.4.X-0.5.md @@ -209,6 +209,7 @@ This procedure does NOT: DROP TABLE stage_clusters; + -- Remove "UNKNOWN" clusters DELETE FROM clusters WHERE cluster_name = 'NO_CLUSTER'; DELETE FROM clusters WHERE cluster_name = 'UNKNOWN-CLUSTER' AND infra_id = ''; ``` @@ -323,6 +324,9 @@ This procedure does NOT: amount NUMERIC(12,2) ); + CREATE TABLE IF NOT EXISTS expenses_default PARTITION OF expenses DEFAULT; + + -- Loading tags backup \COPY stage_expenses (instance_id, date, amount) FROM '/tmp/backups/expenses.csv' CSV HEADER; @@ -465,6 +469,27 @@ This procedure does NOT: * [ ] **M12** — Clean and normalize clusters data. ```sql + -- Verifying "-" suffix + SELECT + c.id, + c.cluster_id, + a.account_name + FROM clusters c + JOIN accounts a ON a.id = c.account_id + WHERE c.cluster_id ~* ('-' || regexp_replace(a.account_name, '([\\W])', '\\\1', 'g') || '$'); + + -- Remove "-" suffix from cluster_id + UPDATE clusters c + SET cluster_id = regexp_replace( + c.cluster_id, + '-' || regexp_replace(a.account_name, '([\\W])', '\\\1', 'g') || '$', + '', + 'i' + ) + FROM accounts a + WHERE a.id = c.account_id + AND c.cluster_id ~* ('-' || regexp_replace(a.account_name, '([\\W])', '\\\1', 'g') || '$'); + -- Processing no-clustered clusters UPDATE clusters SET infra_id = '', cluster_name = 'NO_CLUSTER', cluster_id = 'NO_CLUSTER' WHERE infra_id = '' OR infra_id = 'UNKNOWN-CLUSTER'; -- Processing cluster_id column for removing embeeded account_id @@ -495,5 +520,3 @@ This procedure does NOT: - Row counts match or are explainable against pre-check values - No temporary tables remain - Database is ready for ClusterIQ `0.5` - - diff --git a/internal/events/event_service/event_service_test.go b/internal/events/event_service/event_service_test.go index 89fba541..558f9b58 100644 --- a/internal/events/event_service/event_service_test.go +++ b/internal/events/event_service/event_service_test.go @@ -8,6 +8,7 @@ import ( "github.com/RHEcosystemAppEng/cluster-iq/internal/actions" "github.com/RHEcosystemAppEng/cluster-iq/internal/events" + "github.com/RHEcosystemAppEng/cluster-iq/internal/inventory" "github.com/RHEcosystemAppEng/cluster-iq/internal/models" "github.com/RHEcosystemAppEng/cluster-iq/internal/models/db" "github.com/stretchr/testify/assert" @@ -95,7 +96,7 @@ func testLogEvent_Success(t *testing.T) { TriggeredBy: "scanner", Action: actions.ActionOperation("START"), ResourceID: "cluster-1", - ResourceType: "cluster", + ResourceType: inventory.ClusterResourceType, Result: ResultPending, Description: &desc, Severity: SeverityInfo, @@ -138,7 +139,7 @@ func testLogEvent_RepoError(t *testing.T) { TriggeredBy: "api", Action: actions.ActionOperation("STOP"), ResourceID: "cluster-1", - ResourceType: "cluster", + ResourceType: inventory.ClusterResourceType, Result: ResultPending, Description: nil, Severity: SeverityError, @@ -210,7 +211,7 @@ func testStartTracking_Success(t *testing.T) { TriggeredBy: "agent", Action: actions.ActionOperation("START"), ResourceID: "cluster-1", - ResourceType: "cluster", + ResourceType: inventory.ClusterResourceType, Result: ResultPending, Description: nil, Severity: SeverityInfo, @@ -236,7 +237,7 @@ func testStartTracking_LogEventError(t *testing.T) { TriggeredBy: "agent", Action: actions.ActionOperation("STOP"), ResourceID: "cluster-1", - ResourceType: "cluster", + ResourceType: inventory.ClusterResourceType, Result: ResultPending, Description: nil, Severity: SeverityError, diff --git a/internal/inventory/cost_stats.go b/internal/inventory/cost_stats.go deleted file mode 100644 index 16f3929d..00000000 --- a/internal/inventory/cost_stats.go +++ /dev/null @@ -1,15 +0,0 @@ -package inventory - -type CostStats struct { - // Total cost (US Dollars) - TotalCost float64 - - // Cost Last 15d - Last15DaysCost float64 - - // Last month cost - LastMonthCost float64 - - // Current month so far cost - CurrentMonthSoFarCost float64 -} diff --git a/internal/inventory/types.go b/internal/inventory/types.go index fb1cf2eb..2945ecbc 100644 --- a/internal/inventory/types.go +++ b/internal/inventory/types.go @@ -3,14 +3,12 @@ package inventory const ( // ClusterTagKey string to identify to which cluster is the instance associated ClusterTagKey string = "kubernetes.io/cluster/" -) -const ( // Cluster actions ClusterPowerOnAction = "PowerOn" ClusterPowerOffAction = "PowerOff" // Resource types - ClusterResourceType = "cluster" - InstanceResourceType = "instance" + ClusterResourceType = "Cluster" + InstanceResourceType = "Instance" ) diff --git a/internal/models/db/events_test.go b/internal/models/db/events_test.go index dc52c46d..597a649d 100644 --- a/internal/models/db/events_test.go +++ b/internal/models/db/events_test.go @@ -5,6 +5,7 @@ import ( "time" "github.com/RHEcosystemAppEng/cluster-iq/internal/actions" + "github.com/RHEcosystemAppEng/cluster-iq/internal/inventory" "github.com/RHEcosystemAppEng/cluster-iq/internal/models/convert" "github.com/RHEcosystemAppEng/cluster-iq/internal/models/db" "github.com/stretchr/testify/assert" @@ -31,7 +32,7 @@ func testClusterEventDBResponse_ToClusterEventDTOResponse_Correct(t *testing.T) TriggeredBy: "api", Action: "START", ResourceID: "cluster-1", - ResourceType: "cluster", + ResourceType: inventory.ClusterResourceType, Result: "Success", Description: &desc, Severity: "Info", @@ -60,7 +61,7 @@ func testClusterEventDBResponse_ToClusterEventDTOResponse_NilDescription(t *test TriggeredBy: "scanner", Action: "STOP", ResourceID: "cluster-2", - ResourceType: "cluster", + ResourceType: inventory.ClusterResourceType, Result: "Failed", Description: nil, Severity: "Error", @@ -84,8 +85,8 @@ func testToClusterEventDTOResponseList_Correct(t *testing.T) { conv := &convert.ConverterImpl{} models := []db.ClusterEventDBResponse{ - {ID: 1, EventTimestamp: now, TriggeredBy: "api", Action: "START", ResourceID: "c1", ResourceType: "cluster", Result: "Success", Severity: "Info"}, - {ID: 2, EventTimestamp: now.Add(-time.Minute), TriggeredBy: "agent", Action: "STOP", ResourceID: "c2", ResourceType: "cluster", Result: "Failed", Severity: "Error"}, + {ID: 1, EventTimestamp: now, TriggeredBy: "api", Action: "START", ResourceID: "c1", ResourceType: inventory.ClusterResourceType, Result: "Success", Severity: "Info"}, + {ID: 2, EventTimestamp: now.Add(-time.Minute), TriggeredBy: "agent", Action: "STOP", ResourceID: "c2", ResourceType: inventory.ClusterResourceType, Result: "Failed", Severity: "Error"}, } dtos := conv.ToClusterEventDTOs(models) @@ -114,7 +115,7 @@ func testSystemEventDBResponse_ToSystemEventDTOResponse_Correct(t *testing.T) { TriggeredBy: "scheduler", Action: "START", ResourceID: "cluster-10", - ResourceType: "cluster", + ResourceType: inventory.ClusterResourceType, Result: "Pending", Description: &desc, Severity: "Warning", @@ -130,7 +131,7 @@ func testSystemEventDBResponse_ToSystemEventDTOResponse_Correct(t *testing.T) { assert.Equal(t, "scheduler", dto.TriggeredBy) assert.Equal(t, "START", dto.Action) assert.Equal(t, "cluster-10", dto.ResourceID) - assert.Equal(t, "cluster", dto.ResourceType) + assert.Equal(t, inventory.ClusterResourceType, dto.ResourceType) assert.Equal(t, actions.StatusPending, dto.Result) assert.Equal(t, &desc, dto.Description) assert.Equal(t, "Warning", dto.Severity) @@ -157,7 +158,7 @@ func testToSystemEventDTOResponseList_Correct(t *testing.T) { TriggeredBy: "api", Action: "START", ResourceID: "c1", - ResourceType: "cluster", + ResourceType: inventory.ClusterResourceType, Result: "Success", Severity: "Info", }, @@ -171,7 +172,7 @@ func testToSystemEventDTOResponseList_Correct(t *testing.T) { TriggeredBy: "agent", Action: "STOP", ResourceID: "c2", - ResourceType: "cluster", + ResourceType: inventory.ClusterResourceType, Result: "Failed", Severity: "Error", }, diff --git a/internal/models/dto/event_dto_test.go b/internal/models/dto/event_dto_test.go index 5471d0fe..f9a733a2 100644 --- a/internal/models/dto/event_dto_test.go +++ b/internal/models/dto/event_dto_test.go @@ -6,6 +6,7 @@ import ( "github.com/RHEcosystemAppEng/cluster-iq/internal/actions" "github.com/RHEcosystemAppEng/cluster-iq/internal/events" + "github.com/RHEcosystemAppEng/cluster-iq/internal/inventory" "github.com/stretchr/testify/assert" ) @@ -22,7 +23,7 @@ func testEventDTORequest_ToModelEvent_Correct(t *testing.T) { ID: 42, Action: "START", ResourceID: "cluster-1", - ResourceType: "cluster", + ResourceType: inventory.ClusterResourceType, EventTimestamp: now, Result: "Success", Severity: "Info", @@ -56,7 +57,7 @@ func testEventDTORequest_ToModelEvent_NilDescription(t *testing.T) { ID: 7, Action: "STOP", ResourceID: "instance-1", - ResourceType: "instance", + ResourceType: inventory.InstanceResourceType, EventTimestamp: now, Result: "Failed", Severity: "Error", @@ -84,7 +85,7 @@ func testClusterEventDTOResponse_Struct(t *testing.T) { ID: 1, Action: "START", ResourceID: "cluster-1", - ResourceType: "cluster", + ResourceType: inventory.ClusterResourceType, EventTimestamp: now, Result: "Success", Severity: "Info", @@ -95,7 +96,7 @@ func testClusterEventDTOResponse_Struct(t *testing.T) { assert.Equal(t, int64(1), dto.ID) assert.Equal(t, "START", dto.Action) assert.Equal(t, "cluster-1", dto.ResourceID) - assert.Equal(t, "cluster", dto.ResourceType) + assert.Equal(t, inventory.ClusterResourceType, dto.ResourceType) assert.Equal(t, now, dto.EventTimestamp) assert.Equal(t, actions.StatusSuccess, dto.Result) assert.Equal(t, "Info", dto.Severity) @@ -112,7 +113,7 @@ func testSystemEventDTOResponse_Struct(t *testing.T) { ID: 2, Action: "STOP", ResourceID: "cluster-2", - ResourceType: "cluster", + ResourceType: inventory.ClusterResourceType, EventTimestamp: now, Result: "Failed", Severity: "Error", @@ -126,7 +127,7 @@ func testSystemEventDTOResponse_Struct(t *testing.T) { assert.Equal(t, int64(2), dto.ID) assert.Equal(t, "STOP", dto.Action) assert.Equal(t, "cluster-2", dto.ResourceID) - assert.Equal(t, "cluster", dto.ResourceType) + assert.Equal(t, inventory.ClusterResourceType, dto.ResourceType) assert.Equal(t, now, dto.EventTimestamp) assert.Equal(t, actions.StatusFailed, dto.Result) assert.Equal(t, "Error", dto.Severity) diff --git a/internal/repositories/event_repository.go b/internal/repositories/event_repository.go index 325835e2..96373fb1 100644 --- a/internal/repositories/event_repository.go +++ b/internal/repositories/event_repository.go @@ -34,9 +34,9 @@ const ( :action, ( CASE - WHEN :resource_type = 'cluster' + WHEN :resource_type = 'Cluster' THEN (SELECT id FROM clusters c WHERE c.cluster_id = :resource_id) - WHEN :resource_type = 'instance' + WHEN :resource_type = 'Instance' THEN (SELECT id FROM instances i WHERE i.instance_id = :resource_id) END ), diff --git a/test/integration/api_events_integration_test.go b/test/integration/api_events_integration_test.go index e0e12594..34435187 100644 --- a/test/integration/api_events_integration_test.go +++ b/test/integration/api_events_integration_test.go @@ -8,6 +8,7 @@ import ( "time" responsetypes "github.com/RHEcosystemAppEng/cluster-iq/internal/api/response_types" + "github.com/RHEcosystemAppEng/cluster-iq/internal/inventory" "github.com/RHEcosystemAppEng/cluster-iq/internal/models/dto" ) @@ -97,7 +98,7 @@ func testPostEvents(t *testing.T) { event := dto.EventDTORequest{ Action: "TestAction", ResourceID: "aws-cluster-2-aws-infra-2", - ResourceType: "cluster", + ResourceType: inventory.ClusterResourceType, EventTimestamp: time.Now(), Result: "Pending", Severity: "info", @@ -143,7 +144,7 @@ func testUpdateEvent(t *testing.T) { ID: 2, Action: "TestAction", ResourceID: "aws-cluster-2-aws-infra-2", - ResourceType: "cluster", + ResourceType: inventory.ClusterResourceType, EventTimestamp: time.Now(), Result: "Success", Severity: "info", From 337dce5c54d9f94b4ef5c176903e8c1f0186f04b Mon Sep 17 00:00:00 2001 From: Vladimir Belousov <89217163+vbelouso@users.noreply.github.com> Date: Mon, 2 Feb 2026 15:15:52 +0200 Subject: [PATCH 09/53] chore(helm): add Hashicorp Vault integration (#255) Signed-off-by: Vladimir Belousov --- .../templates/agent/deployment.yaml | 25 ++++++++++-- .../agent/secret-provider-class.yaml | 18 +++++++++ .../cluster-iq/templates/scanner/cronjob.yaml | 12 +++++- .../scanner/secret-provider-class.yaml | 18 +++++++++ .../templates/scanner/serviceaccount.yaml | 13 +++++++ deployments/helm/cluster-iq/values.yaml | 38 ++++++++++++++----- 6 files changed, 110 insertions(+), 14 deletions(-) create mode 100644 deployments/helm/cluster-iq/templates/agent/secret-provider-class.yaml create mode 100644 deployments/helm/cluster-iq/templates/scanner/secret-provider-class.yaml create mode 100644 deployments/helm/cluster-iq/templates/scanner/serviceaccount.yaml diff --git a/deployments/helm/cluster-iq/templates/agent/deployment.yaml b/deployments/helm/cluster-iq/templates/agent/deployment.yaml index 29d297df..c8bec19d 100644 --- a/deployments/helm/cluster-iq/templates/agent/deployment.yaml +++ b/deployments/helm/cluster-iq/templates/agent/deployment.yaml @@ -21,6 +21,7 @@ spec: {{- end }} {{- include "cluster-iq.componentLabels" "agent" | nindent 8 }} spec: + serviceAccountName: {{ include "cluster-iq.agentServiceAccountName" . }} {{- with .Values.agent.imagePullSecrets }} imagePullSecrets: {{- toYaml . | nindent 8 }} @@ -60,14 +61,30 @@ spec: tcpSocket: port: {{ .Values.agent.service.port }} {{- toYaml .Values.agent.livenessProbe | nindent 12 }} - {{- with .Values.agent.volumeMounts }} volumeMounts: + - name: credentials + readOnly: true + mountPath: /credentials + {{- with .Values.agent.volumeMounts }} {{- toYaml . | nindent 12 }} - {{- end }} - {{- with .Values.agent.volumes }} + {{- end }} volumes: + {{- if .Values.vault.enabled }} + - name: credentials + csi: + driver: secrets-store.csi.k8s.io + readOnly: true + volumeAttributes: + secretProviderClass: "cluster-iq-credentials-agent" + {{- else }} + - name: credentials + secret: + secretName: credentials + optional: false + {{- end }} + {{- with .Values.agent.volumes }} {{- toYaml . | nindent 8 }} - {{- end }} + {{- end }} {{- with .Values.agent.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/deployments/helm/cluster-iq/templates/agent/secret-provider-class.yaml b/deployments/helm/cluster-iq/templates/agent/secret-provider-class.yaml new file mode 100644 index 00000000..9639d7ac --- /dev/null +++ b/deployments/helm/cluster-iq/templates/agent/secret-provider-class.yaml @@ -0,0 +1,18 @@ +{{- if .Values.vault.enabled -}} +apiVersion: secrets-store.csi.x-k8s.io/v1 +kind: SecretProviderClass +metadata: + name: cluster-iq-credentials-agent + namespace: {{ .Release.Namespace }} + labels: + {{- include "cluster-iq.labels" . | nindent 4 }} +spec: + provider: vault + parameters: + vaultAddress: "{{ .Values.vault.address }}" + roleName: "agent" + objects: | + - objectName: "credentials" + secretPath: "{{ .Values.vault.secretPath }}" + secretKey: "{{ .Values.vault.secretKey }}" +{{- end -}} diff --git a/deployments/helm/cluster-iq/templates/scanner/cronjob.yaml b/deployments/helm/cluster-iq/templates/scanner/cronjob.yaml index dad80804..e1d6a635 100644 --- a/deployments/helm/cluster-iq/templates/scanner/cronjob.yaml +++ b/deployments/helm/cluster-iq/templates/scanner/cronjob.yaml @@ -4,7 +4,7 @@ metadata: name: scanner labels: {{- include "cluster-iq.labels" . | nindent 4 }} - {{- include "cluster-iq.componentLabels" "api" | nindent 4 }} + {{- include "cluster-iq.componentLabels" "scanner" | nindent 4 }} spec: schedule: 0 0 * * * concurrencyPolicy: Allow @@ -15,10 +15,20 @@ spec: spec: template: spec: + serviceAccountName: {{ include "cluster-iq.scannerServiceAccountName" . }} volumes: + {{- if .Values.vault.enabled }} + - name: credentials + csi: + driver: secrets-store.csi.k8s.io + readOnly: true + volumeAttributes: + secretProviderClass: "cluster-iq-credentials-scanner" + {{- else }} - name: credentials secret: secretName: credentials + {{- end }} containers: - name: scanner image: "{{ .Values.scanner.image.repository }}:{{ .Values.scanner.image.tag | default .Chart.AppVersion }}" diff --git a/deployments/helm/cluster-iq/templates/scanner/secret-provider-class.yaml b/deployments/helm/cluster-iq/templates/scanner/secret-provider-class.yaml new file mode 100644 index 00000000..8019c5a1 --- /dev/null +++ b/deployments/helm/cluster-iq/templates/scanner/secret-provider-class.yaml @@ -0,0 +1,18 @@ +{{- if .Values.vault.enabled -}} +apiVersion: secrets-store.csi.x-k8s.io/v1 +kind: SecretProviderClass +metadata: + name: cluster-iq-credentials-scanner + namespace: {{ .Release.Namespace }} + labels: + {{- include "cluster-iq.labels" . | nindent 4 }} +spec: + provider: vault + parameters: + vaultAddress: "{{ .Values.vault.address }}" + roleName: "scanner" + objects: | + - objectName: "credentials" + secretPath: "{{ .Values.vault.secretPath }}" + secretKey: "{{ .Values.vault.secretKey }}" +{{- end -}} diff --git a/deployments/helm/cluster-iq/templates/scanner/serviceaccount.yaml b/deployments/helm/cluster-iq/templates/scanner/serviceaccount.yaml new file mode 100644 index 00000000..3e925f30 --- /dev/null +++ b/deployments/helm/cluster-iq/templates/scanner/serviceaccount.yaml @@ -0,0 +1,13 @@ +{{- if .Values.scanner.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "cluster-iq.scannerServiceAccountName" . }} + labels: + {{- include "cluster-iq.labels" . | nindent 4 }} + {{- include "cluster-iq.componentLabels" "scanner" | nindent 4 }} + {{- with .Values.scanner.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/deployments/helm/cluster-iq/values.yaml b/deployments/helm/cluster-iq/values.yaml index c933318c..f25825e9 100644 --- a/deployments/helm/cluster-iq/values.yaml +++ b/deployments/helm/cluster-iq/values.yaml @@ -2,6 +2,14 @@ # This is a YAML-formatted file. # Declare variables to be passed into your templates. +# This section configures HashiCorp Vault integration +vault: + enabled: false + address: "http://vault.hashicorp-vault.svc.cluster.local:8200" + secretPath: "secret/data/cluster-iq/credentials" + secretKey: "credentials" + # ServiceAccount names are derived from component names or values + api: # This will set the replicaset count more information can be found here: https://kubernetes.io/docs/concepts/workloads/controllers/replicaset/ replicaCount: 1 @@ -254,6 +262,18 @@ scanner: skipNoOpenshiftInstances: true + #This section builds out the service account + serviceAccount: + # Specifies whether a service account should be created + create: true + # Automatically mount a ServiceAccount's API credentials? + automount: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + agent: # This will set the replicaset count more information can be found here: https://kubernetes.io/docs/concepts/workloads/controllers/replicaset/ replicaCount: 1 @@ -345,17 +365,17 @@ agent: autoscaling: {} # Additional volumes on the output Deployment definition. - volumes: - - name: credentials - secret: - secretName: credentials - optional: false + volumes: [] + # - name: credentials + # secret: + # secretName: credentials + # optional: false # Additional volumeMounts on the output Deployment definition. - volumeMounts: - - name: credentials - readOnly: true - mountPath: /credentials + volumeMounts: [] + # - name: credentials + # readOnly: true + # mountPath: /credentials nodeSelector: {} From f894c1728bbf225e6db2d144ab6f79636597bbcb Mon Sep 17 00:00:00 2001 From: Vladimir Belousov <89217163+vbelouso@users.noreply.github.com> Date: Mon, 2 Feb 2026 15:16:47 +0200 Subject: [PATCH 10/53] docs: add Hashicorp Vault integration guide (#254) Signed-off-by: Vladimir Belousov --- README.md | 5 + doc/vault/README.md | 425 ++++++++++++++++++ .../00-csi-operator-subscription.yaml | 12 + doc/vault/manifests/01-csi-driver.yaml | 6 + .../manifests/10-vault-values-autounseal.yaml | 74 +++ .../manifests/10-vault-values-standard.yaml | 58 +++ .../20-vault-credentials-request.yaml | 19 + doc/vault/manifests/20-vault-policy.hcl | 3 + 8 files changed, 602 insertions(+) create mode 100644 doc/vault/README.md create mode 100644 doc/vault/manifests/00-csi-operator-subscription.yaml create mode 100644 doc/vault/manifests/01-csi-driver.yaml create mode 100644 doc/vault/manifests/10-vault-values-autounseal.yaml create mode 100644 doc/vault/manifests/10-vault-values-standard.yaml create mode 100644 doc/vault/manifests/20-vault-credentials-request.yaml create mode 100644 doc/vault/manifests/20-vault-policy.hcl diff --git a/README.md b/README.md index ec2f27c1..27c0f4a0 100644 --- a/README.md +++ b/README.md @@ -45,6 +45,11 @@ available for every cloud provider: The following graph shows the architecture of this project: ![ClusterIQ architecture diagram](./doc/architecture.png) +### Secrets Management (Optional) + +By default, Cluster IQ uses standard Kubernetes Secrets. For production environments, you can integrate with HashiCorp Vault. +See [Vault Integration Guide](doc/vault/readme.md) for detailed setup instructions. + ## Installation ClusterIQ was designed to run in K8s/Openshift platforms, but it can also run in local using `podman-compose`. diff --git a/doc/vault/README.md b/doc/vault/README.md new file mode 100644 index 00000000..21863f5e --- /dev/null +++ b/doc/vault/README.md @@ -0,0 +1,425 @@ +# Vault Integration Guide for Cluster IQ + +Cluster IQ supports HashiCorp Vault for secure storage and retrieval of cloud provider credentials. This guide explains how to configure your existing Vault instance and deploy Cluster IQ with Vault integration enabled. + +> **Note:** If you do not have a Vault instance and want to install one for testing or development, please refer to the [Reference Installation Guide](#appendix-reference-installation-guide) in the Appendix. + +## Prerequisites + +1. **Secrets Store CSI Driver Operator** installed on the cluster. + * If not installed, follow the [CSI Driver Installation](#1-install-secrets-store-csi-driver-operator) steps in the Appendix. +2. **HashiCorp Vault** installed and accessible. + +## Part I: Vault Configuration Requirements + +Ensure your Vault instance is configured with the following resources. + +### 1. Kubernetes Authentication + +Vault must have the Kubernetes authentication method enabled and configured to communicate with your OpenShift cluster. + +### 2. Policy + +Create a policy named `cluster-iq-credentials` that allows reading the secret. + +* **Example Policy File:** [`doc/vault/manifests/20-vault-policy.hcl`](manifests/20-vault-policy.hcl) + +### 3. Kubernetes Roles + +Create roles to bind the application's ServiceAccounts to the policy. + +| Role Name | ServiceAccount Name | Namespace | Policy | +|-----------|---------------------|-----------|--------| +| `scanner` | `scanner` * | `cluster-iq` | `cluster-iq-credentials` | +| `agent` | `agent` * | `cluster-iq` | `cluster-iq-credentials` | + +*\* Note: ServiceAccount names must match those defined in your Helm values.* + +### 4. Credentials Secret + +Store your cloud provider credentials in the KV v2 secrets engine. + +* **Secret Path:** `secret/cluster-iq/credentials` +* **Key:** `credentials` +* **Value:** Content of your credentials file. + +--- + +## Part II: Application Deployment + +Once Vault is configured, you can deploy (or upgrade) Cluster IQ with Vault integration enabled. + +**1. Create a Helm values file (`vault-config.yaml`)** + +Enable Vault and provide the connection details. + +```yaml +vault: + enabled: true + # Address of your Vault instance + address: "http://vault.hashicorp-vault.svc.cluster.local:8200" + # Path to the secret + secretPath: "secret/data/cluster-iq/credentials" + # Key within the secret + secretKey: "credentials" +``` + +**2. Deploy/Upgrade Cluster IQ** + +```bash +export APP_NS="cluster-iq" +helm upgrade --install cluster-iq deployments/helm/cluster-iq -n $APP_NS -f vault-config.yaml +``` + +**3. Verification** + +```bash +# Trigger a test job +oc create job --from=cronjob/scanner scanner-test -n $APP_NS + +# Check logs for success +oc logs job/scanner-test -n $APP_NS + +# Verify credentials mount in agent pod +oc exec -n $APP_NS deployment/agent -- ls -la /credentials +``` + +--- + +# Appendix: Reference Installation Guide + +This section provides a complete, step-by-step guide to installing and configuring a reference Vault instance on OpenShift. Use this for development, testing, or if you are setting up Vault from scratch. + +## 1. Install Secrets Store CSI Driver Operator + +### 1.1 Create Subscription + +```bash +export CSI_NS="openshift-cluster-csi-drivers" +oc create -f doc/vault/manifests/00-csi-operator-subscription.yaml +``` + +### 1.2 Approve InstallPlan (if Manual approval mode) + +```bash +oc patch installplan $(oc get subscription secrets-store-csi-driver-operator -n $CSI_NS -o jsonpath='{.status.installPlanRef.name}') -n $CSI_NS --type merge -p '{"spec":{"approved":true}}' +``` + +### 1.3 Verify Installation + +```bash +oc get csv -n $CSI_NS | grep secrets-store +``` + +### 1.4 Create ClusterCSIDriver + +```bash +oc create -f doc/vault/manifests/01-csi-driver.yaml +``` + +### 1.5 Verify CSI Driver pods + +```bash +oc get pods -n $CSI_NS | grep secrets-store +``` + +## 2. Install Reference Vault Instance + +Use this to set up a production-ready Vault instance on OpenShift for testing/demo purposes. + +**Choose one of the following installation options:** + +* **[Option A: Standard Installation (Manual Unseal)](#option-a-standard-installation-manual-unseal)** + * Simpler setup, no external dependencies. + * Requires manual unseal with keys after every restart. +* **[Option B: Production Installation (AWS KMS Auto-Unseal)](#option-b-production-installation-aws-kms-auto-unseal)** + * Production-grade setup. + * Requires AWS access (KMS, IAM). + * Vault unseals automatically. + +### Common Preparation (Required for BOTH options) + +```bash +# Add Helm repo +helm repo add hashicorp https://helm.releases.hashicorp.com +helm repo update + +# Create Namespace +export VAULT_NS="hashicorp-vault" +oc new-project $VAULT_NS + +# Configure SCC +oc adm policy add-scc-to-user privileged -z vault-csi-provider -n $VAULT_NS +``` + +### Option A: Standard Installation (Manual Unseal) + +**1. Install Vault** + +```bash +helm upgrade --install vault hashicorp/vault -n $VAULT_NS \ + -f doc/vault/manifests/10-vault-values-standard.yaml +``` + +**2. Initialize & Unseal** + +```bash +# Initialize vault-0 +oc exec -n $VAULT_NS vault-0 -- vault operator init -key-shares=5 -key-threshold=3 -format=json > /tmp/vault-init.json + +# Extract keys +UNSEAL_KEY_1=$(jq -r '.unseal_keys_b64[0]' /tmp/vault-init.json) +UNSEAL_KEY_2=$(jq -r '.unseal_keys_b64[1]' /tmp/vault-init.json) +UNSEAL_KEY_3=$(jq -r '.unseal_keys_b64[2]' /tmp/vault-init.json) + +# Unseal vault-0 +oc exec -n $VAULT_NS vault-0 -- vault operator unseal $UNSEAL_KEY_1 +oc exec -n $VAULT_NS vault-0 -- vault operator unseal $UNSEAL_KEY_2 +oc exec -n $VAULT_NS vault-0 -- vault operator unseal $UNSEAL_KEY_3 + +# Join and unseal followers +for pod in vault-1 vault-2; do + echo "Joining and unsealing $pod..." + oc exec -n $VAULT_NS $pod -- vault operator raft join http://vault-0.vault-internal:8200 + oc exec -n $VAULT_NS $pod -- vault operator unseal $UNSEAL_KEY_1 + oc exec -n $VAULT_NS $pod -- vault operator unseal $UNSEAL_KEY_2 + oc exec -n $VAULT_NS $pod -- vault operator unseal $UNSEAL_KEY_3 +done +``` + +### Option B: Production Installation (AWS KMS Auto-Unseal) + +**1. Set AWS Variables** + +```bash +export AWS_REGION="us-east-1" +export IAM_USER_NAME="vault-unseal-user" +export KMS_KEY_TAG_NAME="vault-autounseal-key" +``` + +**2. Create AWS KMS key** + +```bash +aws kms create-key \ + --description "Vault auto-unseal key" \ + --tags TagKey=Name,TagValue=$KMS_KEY_TAG_NAME \ + --region $AWS_REGION \ + --output json | tee /tmp/kms-key.json + +# Extract Key ARN +export KMS_KEY_ARN=$(jq -r '.KeyMetadata.Arn' /tmp/kms-key.json) +echo "KMS Key ARN: $KMS_KEY_ARN" +``` + +**3. Configure IAM Access** + +Choose one of the following sub-options to configure IAM access. + +* **Sub-Option B1 (Automatic):** Uses OpenShift `CredentialsRequest` (Mint Mode). **Requirement:** You must have permissions to create `CredentialsRequest` objects in the cluster. +* **Sub-Option B2 (Manual):** Manually creates IAM users/policies and Kubernetes Secrets. + +**Sub-Option B1: Automatic Setup (OpenShift Mint Mode)** + +```bash +# Prepare Manifest +cp doc/vault/manifests/20-vault-credentials-request.yaml /tmp/vault-credentials-request.yaml +KMS_KEY_ARN_ESCAPED=$(echo $KMS_KEY_ARN | sed 's/\//\\\//g') +sed -i "s/REPLACE_WITH_YOUR_KMS_KEY_ARN/$KMS_KEY_ARN_ESCAPED/g" /tmp/vault-credentials-request.yaml +sed -i "s/namespace: hashicorp-vault/namespace: $VAULT_NS/g" /tmp/vault-credentials-request.yaml + +# Apply +oc apply -f /tmp/vault-credentials-request.yaml + +# Wait for Secret +while ! oc get secret vault-kms-creds -n $VAULT_NS > /dev/null 2>&1; do sleep 5; echo "Waiting..."; done +``` + +**Sub-Option B2: Manual Setup (Universal)** + +```bash +# Create IAM policy +cat > /tmp/vault-kms-policy.json < /tmp/vault-init.json + +# Join followers +oc exec -n $VAULT_NS vault-1 -- vault operator raft join http://vault-0.vault-internal:8200 +oc exec -n $VAULT_NS vault-2 -- vault operator raft join http://vault-0.vault-internal:8200 + +# Verify status +oc exec -n $VAULT_NS vault-0 -- vault status +``` + +## 3. Configure Vault (Reference) + +This section details how to configure the reference Vault instance for Cluster IQ. + +**3.1 Enable Kubernetes Authentication** + +```bash +# Get Root Token +ROOT_TOKEN=$(jq -r '.root_token' /tmp/vault-init.json) +oc exec -n $VAULT_NS vault-0 -- vault login $ROOT_TOKEN + +# Enable auth method +oc exec -n $VAULT_NS vault-0 -- vault auth enable kubernetes + +# Configure auth +oc exec -n $VAULT_NS vault-0 -- sh -c 'vault write auth/kubernetes/config \ + kubernetes_host="https://$KUBERNETES_PORT_443_TCP_ADDR:443"' +``` + +**3.2 Create Policy and Roles** + +```bash +export APP_NS="cluster-iq" + +# Create Policy +cat doc/vault/manifests/20-vault-policy.hcl | \ + oc exec -i -n $VAULT_NS vault-0 -- vault policy write cluster-iq-credentials - + +# Create Role for Scanner +oc exec -n $VAULT_NS vault-0 -- vault write auth/kubernetes/role/scanner \ + bound_service_account_names=scanner \ + bound_service_account_namespaces=$APP_NS \ + policies=cluster-iq-credentials \ + ttl=24h \ + audience=vault + +# Create Role for Agent +oc exec -n $VAULT_NS vault-0 -- vault write auth/kubernetes/role/agent \ + bound_service_account_names=agent \ + bound_service_account_namespaces=$APP_NS \ + policies=cluster-iq-credentials \ + ttl=24h \ + audience=vault +``` + +**3.3 Load Credentials** + +```bash +# Enable KV v2 engine +oc exec -n $VAULT_NS vault-0 -- vault secrets enable -path=secret kv-v2 + +# Replace with the real path to credentials file +cat /path/to/credentials/file | oc exec -i -n $VAULT_NS vault-0 -- \ + sh -c "VAULT_TOKEN=$ROOT_TOKEN vault kv put secret/cluster-iq/credentials credentials=-" +``` + +## 4. Troubleshooting & Verification + +**Check CSI Driver Logs** +If pods are failing to start, check the CSI driver logs on the node where the pod is scheduled. + +```bash +oc logs -n openshift-cluster-csi-drivers -l app=secrets-store-csi-driver-node +``` + +**Verify Secret Content in Vault** + +```bash +oc exec -n $VAULT_NS vault-0 -- vault kv get secret/cluster-iq/credentials +``` + +**Check Vault Cluster Status** + +```bash +# Check overall status (Sealed/Unsealed, HA Mode) +oc exec -n $VAULT_NS vault-0 -- vault status + +# List Raft peers (Leader/Follower status) +oc exec -n $VAULT_NS vault-0 -- vault operator raft list-peers + +# Check all pods status +oc get pods -n $VAULT_NS +``` + +**Login to Vault CLI** + +```bash +# Login with Root Token (from initialization step) +oc exec -ti -n $VAULT_NS vault-0 -- vault login + +# Or extract from init file +ROOT_TOKEN=$(jq -r '.root_token' /tmp/vault-init.json) +oc exec -ti -n $VAULT_NS vault-0 -- vault login $ROOT_TOKEN +``` + +**Common Vault Operations** + +```bash +# List all secrets engines +oc exec -n $VAULT_NS vault-0 -- vault secrets list + +# List all auth methods +oc exec -n $VAULT_NS vault-0 -- vault auth list + +# List all policies +oc exec -n $VAULT_NS vault-0 -- vault policy list + +# Read a specific policy +oc exec -n $VAULT_NS vault-0 -- vault policy read cluster-iq-credentials + +# List Kubernetes auth roles +oc exec -n $VAULT_NS vault-0 -- vault list auth/kubernetes/role +``` + +**Restart Application Pods** +If you updated the secret in Vault, restart the application pods to sync changes immediately (otherwise, wait for the rotation interval). + +```bash +oc delete pod -l app.kubernetes.io/component=agent -n cluster-iq +``` diff --git a/doc/vault/manifests/00-csi-operator-subscription.yaml b/doc/vault/manifests/00-csi-operator-subscription.yaml new file mode 100644 index 00000000..91c0a1f8 --- /dev/null +++ b/doc/vault/manifests/00-csi-operator-subscription.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: operators.coreos.com/v1alpha1 +kind: Subscription +metadata: + name: secrets-store-csi-driver-operator + namespace: openshift-cluster-csi-drivers +spec: + channel: stable + name: secrets-store-csi-driver-operator + source: redhat-operators + sourceNamespace: openshift-marketplace + installPlanApproval: Manual \ No newline at end of file diff --git a/doc/vault/manifests/01-csi-driver.yaml b/doc/vault/manifests/01-csi-driver.yaml new file mode 100644 index 00000000..7e9be143 --- /dev/null +++ b/doc/vault/manifests/01-csi-driver.yaml @@ -0,0 +1,6 @@ +apiVersion: operator.openshift.io/v1 +kind: ClusterCSIDriver +metadata: + name: secrets-store.csi.k8s.io +spec: + managementState: Managed \ No newline at end of file diff --git a/doc/vault/manifests/10-vault-values-autounseal.yaml b/doc/vault/manifests/10-vault-values-autounseal.yaml new file mode 100644 index 00000000..ce283c94 --- /dev/null +++ b/doc/vault/manifests/10-vault-values-autounseal.yaml @@ -0,0 +1,74 @@ +# Based on official values.openshift.yaml from Vault Helm chart +# with HA Raft configuration and AWS KMS Auto-Unseal + +global: + openshift: true + +injector: + enabled: false + + image: + repository: "registry.connect.redhat.com/hashicorp/vault-k8s" + tag: "1.7.0-ubi" + + agentImage: + repository: "registry.connect.redhat.com/hashicorp/vault" + tag: "1.20.4-ubi" + +server: + image: + repository: "registry.connect.redhat.com/hashicorp/vault" + tag: "1.20.4-ubi" + + ha: + enabled: true + replicas: 3 + + raft: + enabled: true + setNodeId: true + config: | + ui = true + listener "tcp" { + address = "[::]:8200" + cluster_address = "[::]:8201" + tls_disable = 1 + } + storage "raft" { + path = "/vault/data" + } + seal "awskms" { + region = "REPLACE_WITH_YOUR_REGION" + kms_key_id = "REPLACE_WITH_YOUR_KMS_KEY_ARN" + } + service_registration "kubernetes" {} + + # AWS credentials from Kubernetes Secret + extraSecretEnvironmentVars: + - envName: AWS_ACCESS_KEY_ID + secretName: vault-kms-creds + secretKey: AWS_ACCESS_KEY_ID + - envName: AWS_SECRET_ACCESS_KEY + secretName: vault-kms-creds + secretKey: AWS_SECRET_ACCESS_KEY + - envName: AWS_REGION + secretName: vault-kms-creds + secretKey: AWS_REGION + + readinessProbe: + path: "/v1/sys/health?standbyok=true&uninitcode=204" + + route: + enabled: false + +csi: + enabled: true + + image: + repository: "registry.connect.redhat.com/hashicorp/vault-csi-provider" + tag: "1.6.0-ubi" + + agent: + image: + repository: "registry.connect.redhat.com/hashicorp/vault" + tag: "1.20.4-ubi" diff --git a/doc/vault/manifests/10-vault-values-standard.yaml b/doc/vault/manifests/10-vault-values-standard.yaml new file mode 100644 index 00000000..48d7c0a6 --- /dev/null +++ b/doc/vault/manifests/10-vault-values-standard.yaml @@ -0,0 +1,58 @@ +# Based on official values.openshift.yaml from Vault Helm chart +# with HA Raft configuration added + +global: + openshift: true + +injector: + enabled: false + + image: + repository: "registry.connect.redhat.com/hashicorp/vault-k8s" + tag: "1.7.0-ubi" + + agentImage: + repository: "registry.connect.redhat.com/hashicorp/vault" + tag: "1.20.4-ubi" + +server: + image: + repository: "registry.connect.redhat.com/hashicorp/vault" + tag: "1.20.4-ubi" + + ha: + enabled: true + replicas: 3 + + raft: + enabled: true + setNodeId: true + config: | + ui = true + listener "tcp" { + address = "[::]:8200" + cluster_address = "[::]:8201" + tls_disable = 1 + } + storage "raft" { + path = "/vault/data" + } + service_registration "kubernetes" {} + + readinessProbe: + path: "/v1/sys/health?standbyok=true&uninitcode=204" + + route: + enabled: false + +csi: + enabled: true + + image: + repository: "registry.connect.redhat.com/hashicorp/vault-csi-provider" + tag: "1.6.0-ubi" + + agent: + image: + repository: "registry.connect.redhat.com/hashicorp/vault" + tag: "1.20.4-ubi" diff --git a/doc/vault/manifests/20-vault-credentials-request.yaml b/doc/vault/manifests/20-vault-credentials-request.yaml new file mode 100644 index 00000000..7a710265 --- /dev/null +++ b/doc/vault/manifests/20-vault-credentials-request.yaml @@ -0,0 +1,19 @@ +apiVersion: cloudcredential.openshift.io/v1 +kind: CredentialsRequest +metadata: + name: vault-kms-creds + namespace: hashicorp-vault +spec: + secretRef: + name: vault-kms-creds + namespace: hashicorp-vault + providerSpec: + apiVersion: cloudcredential.openshift.io/v1 + kind: AWSProviderSpec + statementEntries: + - action: + - kms:Encrypt + - kms:Decrypt + - kms:DescribeKey + effect: Allow + resource: "REPLACE_WITH_YOUR_KMS_KEY_ARN" diff --git a/doc/vault/manifests/20-vault-policy.hcl b/doc/vault/manifests/20-vault-policy.hcl new file mode 100644 index 00000000..4a6d01aa --- /dev/null +++ b/doc/vault/manifests/20-vault-policy.hcl @@ -0,0 +1,3 @@ +path "secret/data/cluster-iq/credentials" { + capabilities = ["read"] +} From f6bcc081c927d4d1cc78a4c1c01c0454a6abb270 Mon Sep 17 00:00:00 2001 From: Alejandro Villegas Date: Tue, 3 Feb 2026 12:08:08 +0100 Subject: [PATCH 11/53] fix(agent): Fixed CronActions re-scheduling (#256) Signed-off-by: r2dedios --- cmd/agent/executor_agent_service.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/cmd/agent/executor_agent_service.go b/cmd/agent/executor_agent_service.go index e99f2958..0d18e03f 100644 --- a/cmd/agent/executor_agent_service.go +++ b/cmd/agent/executor_agent_service.go @@ -240,10 +240,23 @@ func (e *ExecutorAgentService) Start() error { tracker.Success() } + // Update action status to Success/Failed if err := e.updateActionStatus(newAction); err != nil { e.logger.Error("Error updating action status", zap.String("action_id", newAction.GetID()), zap.Error(err)) continue } + + // For CronActions, reset status back to Pending so they can be rescheduled + if newAction.GetType() == actions.CronActionType { + e.logger.Debug("Resetting CronAction status to Pending for next execution", + zap.String("action_id", newAction.GetID()), + ) + newAction.(actions.MutableAction).SetStatus(actions.StatusPending) + if err := e.updateActionStatus(newAction); err != nil { + e.logger.Error("Error resetting CronAction status to Pending", zap.String("action_id", newAction.GetID()), zap.Error(err)) + continue + } + } } return nil From 715e2adb22c09bd607c9aca11ffc6da9b9bdaf37 Mon Sep 17 00:00:00 2001 From: Alejandro Villegas Date: Tue, 3 Feb 2026 12:43:23 +0100 Subject: [PATCH 12/53] fix(scanner): Fixed creation timestamp capture for OSD clusters (#258) Signed-off-by: r2dedios --- internal/cloud_providers/aws/aws-ec2.go | 31 ++++++++++++++++--------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/internal/cloud_providers/aws/aws-ec2.go b/internal/cloud_providers/aws/aws-ec2.go index 7715d27b..977c18f9 100644 --- a/internal/cloud_providers/aws/aws-ec2.go +++ b/internal/cloud_providers/aws/aws-ec2.go @@ -10,12 +10,6 @@ import ( "github.com/aws/aws-sdk-go/service/ec2" ) -const ( - // Reference https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/device_naming.html#available-ec2-device-names - rootDeviceXvda = "/dev/xvda" - rootDeviceSda = "/dev/sda1" -) - // AWSEC2Connection represents the EC2 client for AWS type AWSEC2Connection struct { client *ec2.EC2 @@ -252,13 +246,28 @@ func EC2InstanceToInventoryInstance(ec2instance *ec2.Instance) (*inventory.Insta } // getInstanceCreationTimestamp retrieves the creation timestamp of an EC2 instance. -// It determines the instance creation time based on the attach time of the root block device. -// If the root device is not found among the block device mappings, it returns an empty time.Time. +// It uses a hybrid strategy to get the most accurate creation time: +// 1. Primary: Root EBS volume attach time (persists through stop/start cycles) +// 2. Fallback: Instance LaunchTime (always available, but changes on restart) +// 3. Last resort: Zero time (should never happen with valid AWS instances) func getInstanceCreationTimestamp(instance ec2.Instance) time.Time { - for _, mapping := range instance.BlockDeviceMappings { - if *mapping.DeviceName == rootDeviceXvda || *mapping.DeviceName == rootDeviceSda { - return *mapping.Ebs.AttachTime + // Strategy 1: Use root device attach time (most accurate for original creation) + // This approach works with all device naming conventions (Xen, NVMe, etc.) + if instance.RootDeviceName != nil { + for _, mapping := range instance.BlockDeviceMappings { + if mapping.DeviceName != nil && *mapping.DeviceName == *instance.RootDeviceName { + if mapping.Ebs != nil && mapping.Ebs.AttachTime != nil { + return *mapping.Ebs.AttachTime + } + } } } + + // Strategy 2: Fallback to LaunchTime (always available) + if instance.LaunchTime != nil { + return *instance.LaunchTime + } + + // Strategy 3: Last resort (should never happen with valid AWS instances) return time.Time{} } From 3d39c364cae5c222de02c6e02444c6abb2028489 Mon Sep 17 00:00:00 2001 From: Alejandro Villegas Date: Mon, 2 Mar 2026 15:54:47 +0100 Subject: [PATCH 13/53] chore: add requester and description parameters (#252) * chore: Added requester and decription parameters when running instant actions on cluster details page Signed-off-by: r2dedios * fix(handlers): Removed duplicated struct and added swagger doc to power on/off methods Signed-off-by: r2dedios * refactor: moved PowerActionRequest to DTO package Signed-off-by: r2dedios * chore: updated go version to 1.25.7 Signed-off-by: r2dedios * fix: fixed annotation order error on Converter Signed-off-by: r2dedios * fix: using golangci latest version Signed-off-by: r2dedios --------- Signed-off-by: r2dedios --- .github/workflows/validate-pr.yaml | 1 - .../containerfiles/Containerfile-agent | 2 +- deployments/containerfiles/Containerfile-api | 2 +- .../containerfiles/Containerfile-scanner | 2 +- go.mod | 15 +++++- go.sum | 49 +++++++++++++------ internal/api/handlers/cluster_handler.go | 34 ++++++++++++- internal/events/events.go | 2 +- internal/models/convert/mapper.go | 2 +- internal/models/dto/action_dto.go | 6 +++ internal/services/cluster_service.go | 18 +++---- 11 files changed, 97 insertions(+), 36 deletions(-) diff --git a/.github/workflows/validate-pr.yaml b/.github/workflows/validate-pr.yaml index f41cc420..50880b43 100644 --- a/.github/workflows/validate-pr.yaml +++ b/.github/workflows/validate-pr.yaml @@ -27,7 +27,6 @@ jobs: with: only-new-issues: true args: --whole-files - version: v2.1 call-unit-tests: name: Go Unit tests diff --git a/deployments/containerfiles/Containerfile-agent b/deployments/containerfiles/Containerfile-agent index 23d95149..cbbdae7a 100644 --- a/deployments/containerfiles/Containerfile-agent +++ b/deployments/containerfiles/Containerfile-agent @@ -1,7 +1,7 @@ ## Build # vim: set ft=dockerfile : #################### -FROM golang:1.24.11 AS builder +FROM golang:1.25.7 AS builder # Build arguments ARG VERSION diff --git a/deployments/containerfiles/Containerfile-api b/deployments/containerfiles/Containerfile-api index 641fe9f9..3e94f30c 100644 --- a/deployments/containerfiles/Containerfile-api +++ b/deployments/containerfiles/Containerfile-api @@ -1,7 +1,7 @@ ## Build # vim: set ft=dockerfile : #################### -FROM golang:1.24.11 AS builder +FROM golang:1.25.7 AS builder # Build arguments ARG VERSION diff --git a/deployments/containerfiles/Containerfile-scanner b/deployments/containerfiles/Containerfile-scanner index 75007d84..2511b17c 100644 --- a/deployments/containerfiles/Containerfile-scanner +++ b/deployments/containerfiles/Containerfile-scanner @@ -1,7 +1,7 @@ ## Build # vim: set ft=dockerfile : #################### -FROM golang:1.24.11 AS builder +FROM golang:1.25.7 AS builder # Build arguments ARG VERSION diff --git a/go.mod b/go.mod index eb4602b8..cbdfb09a 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/RHEcosystemAppEng/cluster-iq -go 1.24.0 +go 1.25.7 require ( github.com/aws/aws-sdk-go v1.55.5 @@ -11,6 +11,7 @@ require ( github.com/lib/pq v1.10.9 github.com/robfig/cron/v3 v3.0.1 github.com/stretchr/testify v1.9.0 + github.com/swaggo/swag v1.16.6 go.uber.org/zap v1.25.0 google.golang.org/grpc v1.69.4 google.golang.org/protobuf v1.35.1 @@ -18,22 +19,31 @@ require ( ) require ( + github.com/KyleBanks/depth v1.2.1 // indirect + github.com/PuerkitoBio/purell v1.1.1 // indirect + github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect github.com/bytedance/sonic v1.9.1 // indirect github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect github.com/dave/jennifer v1.6.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/gabriel-vasile/mimetype v1.4.2 // indirect github.com/gin-contrib/sse v0.1.0 // indirect + github.com/go-openapi/jsonpointer v0.19.5 // indirect + github.com/go-openapi/jsonreference v0.19.6 // indirect + github.com/go-openapi/spec v0.20.4 // indirect + github.com/go-openapi/swag v0.19.15 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect github.com/go-playground/validator/v10 v10.14.0 // indirect github.com/goccy/go-json v0.10.2 // indirect - github.com/jmattheis/goverter v1.9.2 // indirect + github.com/jmattheis/goverter v1.9.3 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect + github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/cpuid/v2 v2.2.4 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/leodido/go-urn v1.2.4 // indirect + github.com/mailru/easyjson v0.7.6 // indirect github.com/mattn/go-isatty v0.0.19 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect @@ -53,5 +63,6 @@ require ( golang.org/x/text v0.31.0 // indirect golang.org/x/tools v0.38.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20241015192408-796eee8c2d53 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index d8e6ba19..d5bb586e 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,9 @@ +github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc= +github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE= +github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= +github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= +github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/aws/aws-sdk-go v1.55.5 h1:KKUZBfBoyqy5d3swXyiC7Q76ic40rYcbqH7qjh59kzU= github.com/aws/aws-sdk-go v1.55.5/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= @@ -32,6 +38,16 @@ github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= +github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonreference v0.19.6 h1:UBIxjkht+AWIgYzCDSv2GN+E/togfwXUJFRTWhl2Jjs= +github.com/go-openapi/jsonreference v0.19.6/go.mod h1:diGHMEHg2IqXZGKxqyvWdfWU/aim5Dprw5bqpKkTvns= +github.com/go-openapi/spec v0.20.4 h1:O8hJrt0UMnhHcluhIdUgCLRWyM2x7QkBXRvOs7m+O1M= +github.com/go-openapi/spec v0.20.4/go.mod h1:faYFR1CvsJZ0mNsmsphTMSoRrNV3TEDoAM7FOEWeq8I= +github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-openapi/swag v0.19.15 h1:D2NRCBzS9/pEY3gP9Nl8aDqGUcPFrwG2p+CNFrLyrCM= +github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= @@ -59,14 +75,16 @@ github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeN github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/jmattheis/goverter v1.9.2 h1:pBjvkhJ0F3PKMqGyHPL0yqnbTe08jjZqt/Z9ZmNKtTQ= -github.com/jmattheis/goverter v1.9.2/go.mod h1:1n3q6zf7j58tXcRWHbLFxK2Jk8WQVzr0d3nuaCcRqeg= +github.com/jmattheis/goverter v1.9.3 h1:dTGAI9sqNgVsd9vkHDKpVJCE/Aw6WlIokqrAdSWP2PI= +github.com/jmattheis/goverter v1.9.3/go.mod h1:1n3q6zf7j58tXcRWHbLFxK2Jk8WQVzr0d3nuaCcRqeg= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g= github.com/jmoiron/sqlx v1.3.5/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= @@ -87,6 +105,10 @@ github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNa github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA= +github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= @@ -97,6 +119,7 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/pelletier/go-toml/v2 v2.0.1/go.mod h1:r9LEWfGN8R5k0VXJ+0BkIe7MYkRdwZOjgMj2KwnJFUo= github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZR9tGQ= github.com/pelletier/go-toml/v2 v2.0.8/go.mod h1:vuYfssBdrU2XDZ9bYydBu6t+6a6PYNcZljzZR9VXg+4= @@ -123,6 +146,8 @@ github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/swaggo/swag v1.16.6 h1:qBNcx53ZaX+M5dxVyTrgQ0PJ/ACK+NzhwcbieTt+9yI= +github.com/swaggo/swag v1.16.6/go.mod h1:ngP2etMK5a0P3QBizic5MEwpRmluJZPHjXcMoj4Xesg= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= github.com/ugorji/go v1.2.7/go.mod h1:nF9osbDWLy6bDVv/Rtoh6QgnvNDpmCalQV5urGCCS6M= @@ -158,14 +183,10 @@ golang.org/x/arch v0.3.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.38.0 h1:jt+WWG8IZlBnVbomuhg2Mdq0+BBQaHbtqHEFEigjUV8= -golang.org/x/crypto v0.38.0/go.mod h1:MvrbAqul58NNYPKnOra203SB9vpuZW0e+RRZV+Ggqjw= golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q= golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.24.0 h1:ZfthKaKaT4NrhGVZHO1/WDTwGES4De8KtWO0SIbNJMU= -golang.org/x/mod v0.24.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww= golang.org/x/mod v0.29.0 h1:HV8lRxZC4l2cr3Zq1LvtOsi/ThTgWnUk/y64QSs8GwA= golang.org/x/mod v0.29.0/go.mod h1:NyhrlYXJ2H4eJiRy/WDBO6HMqZQ6q9nk4JzS3NuCK+w= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -173,44 +194,37 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.40.0 h1:79Xs7wF06Gbdcg4kdCCIQArK11Z1hr5POQ6+fIYHNuY= -golang.org/x/net v0.40.0/go.mod h1:y0hY0exeL2Pku80/zKK7tpntoX23cqL3Oa6njdgRtds= +golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM= golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.14.0 h1:woo0S4Yywslg6hp4eUFjTVOyKt0RookbpAHG4c1HmhQ= -golang.org/x/sync v0.14.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I= golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= -golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.25.0 h1:qVyWApTSYLk/drJRO5mDlNYskwQznZmkpV2c8q9zls4= -golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.31.0 h1:0EedkvKDbh+qistFTd0Bcwe/YLh4vHwWEkiI0toFIBU= -golang.org/x/tools v0.31.0/go.mod h1:naFTU+Cev749tSJRXJlna0T3WxKvb1kWEx15xA4SdmQ= golang.org/x/tools v0.38.0 h1:Hx2Xv8hISq8Lm16jvBZ2VQf+RLmbd7wVUsALibYI/IQ= golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -227,15 +241,18 @@ google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFyt google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/api/handlers/cluster_handler.go b/internal/api/handlers/cluster_handler.go index 3b0b7c98..b380cb0b 100644 --- a/internal/api/handlers/cluster_handler.go +++ b/internal/api/handlers/cluster_handler.go @@ -255,14 +255,29 @@ func (h *ClusterHandler) Delete(c *gin.Context) { // @Accept json // @Produce json // @Param id path string true "Cluster ID" +// @Param request body dto.PowerActionRequest true "Power action details" // @Success 202 {object} responsetypes.GenericResponse +// @Failure 400 {object} responsetypes.GenericErrorResponse // @Failure 404 {object} responsetypes.GenericErrorResponse // @Failure 500 {object} responsetypes.GenericErrorResponse // @Router /clusters/{id}/power_on [post] func (h *ClusterHandler) PowerOn(c *gin.Context) { clusterID := c.Param("id") + var request dto.PowerActionRequest - if err := h.service.PowerOn(c.Request.Context(), clusterID); err != nil { + // Bind JSON body to request struct + if err := c.ShouldBindJSON(&request); err != nil { + h.logger.Error("Bad request for PowerOn cluster operation", zap.String("cluster_id", clusterID), zap.String("requester", request.Requester), zap.String("description", request.Description)) + c.JSON(http.StatusBadRequest, responsetypes.GenericErrorResponse{ + Message: "Invalid request body: " + err.Error(), + }) + return + } + + // Log requester and description + h.logger.Info("Power on request received", zap.String("requester", request.Requester), zap.String("description", request.Description)) + + if err := h.service.PowerOn(c.Request.Context(), clusterID, request.Requester, &request.Description); err != nil { h.logger.Error("error powering on a cluster", zap.String("cluster_id", clusterID), zap.Error(err)) if errors.Is(err, repositories.ErrNotFound) { c.JSON(http.StatusNotFound, responsetypes.GenericErrorResponse{ @@ -290,14 +305,29 @@ func (h *ClusterHandler) PowerOn(c *gin.Context) { // @Accept json // @Produce json // @Param id path string true "Cluster ID" +// @Param request body dto.PowerActionRequest true "Power action details" // @Success 202 {object} responsetypes.GenericResponse +// @Failure 400 {object} responsetypes.GenericErrorResponse // @Failure 404 {object} responsetypes.GenericErrorResponse // @Failure 500 {object} responsetypes.GenericErrorResponse // @Router /clusters/{id}/power_off [post] func (h *ClusterHandler) PowerOff(c *gin.Context) { clusterID := c.Param("id") + var request dto.PowerActionRequest + + // Bind JSON body to request struct + if err := c.ShouldBindJSON(&request); err != nil { + h.logger.Error("Bad request for PowerOff cluster operation", zap.String("cluster_id", clusterID), zap.String("requester", request.Requester), zap.String("description", request.Description)) + c.JSON(http.StatusBadRequest, responsetypes.GenericErrorResponse{ + Message: "Invalid request body: " + err.Error(), + }) + return + } + + // Log requester and description + h.logger.Info("Power off request received", zap.String("requester", request.Requester), zap.String("description", request.Description)) - if err := h.service.PowerOff(c.Request.Context(), clusterID); err != nil { + if err := h.service.PowerOff(c.Request.Context(), clusterID, request.Requester, &request.Description); err != nil { h.logger.Error("error powering off a cluster", zap.String("cluster_id", clusterID), zap.Error(err)) if errors.Is(err, repositories.ErrNotFound) { c.JSON(http.StatusNotFound, responsetypes.GenericErrorResponse{ diff --git a/internal/events/events.go b/internal/events/events.go index 544f1396..f793121c 100644 --- a/internal/events/events.go +++ b/internal/events/events.go @@ -18,7 +18,7 @@ type Event struct { Description *string `db:"description"` // ID of the affected resource (e.g., cluster_id, instance_id). ResourceID string `db:"resource_id"` - // Type of resource affected (e.g., "cluster", "instance"). + // Type of resource affected (e.g., "Cluster", "Instance"). ResourceType string `db:"resource_type"` // Outcome of the action (e.g., "success", "error"). Result string `db:"result"` diff --git a/internal/models/convert/mapper.go b/internal/models/convert/mapper.go index ade53382..b6d6e29d 100644 --- a/internal/models/convert/mapper.go +++ b/internal/models/convert/mapper.go @@ -12,12 +12,12 @@ import ( ) // goverter:converter +// goverter:output:file ./generated.go // goverter:extend Time // goverter:extend NullTime // goverter:extend NullString // goverter:extend StringArray // goverter:extend TagDBResponsesToDTO -// goverter:output:file ./generated.go type Converter interface { // Account ToAccountDTO(src db.AccountDBResponse) dto.AccountDTOResponse diff --git a/internal/models/dto/action_dto.go b/internal/models/dto/action_dto.go index 26549cbd..2dd36a2f 100644 --- a/internal/models/dto/action_dto.go +++ b/internal/models/dto/action_dto.go @@ -162,3 +162,9 @@ func ToModelActionListFromResponse(dtos []ActionDTOResponse) ([]actions.Action, } return resultActions, nil } + +// PowerActionRequest defines the information required by the API to complement the actions and events for a Power On/Off request0 +type PowerActionRequest struct { + Requester string `json:"requester" binding:"required"` + Description string `json:"description"` +} diff --git a/internal/services/cluster_service.go b/internal/services/cluster_service.go index ec9eb69e..04609e4e 100644 --- a/internal/services/cluster_service.go +++ b/internal/services/cluster_service.go @@ -19,8 +19,8 @@ type ClusterService interface { Get(ctx context.Context, clusterID string) (*db.ClusterDBResponse, error) GetInstances(ctx context.Context, clusterID string) ([]db.InstanceDBResponse, error) GetSummary(ctx context.Context) (inventory.ClustersSummary, error) - PowerOn(ctx context.Context, clusterID string) error - PowerOff(ctx context.Context, clusterID string) error + PowerOn(ctx context.Context, clusterID string, requester string, description *string) error + PowerOff(ctx context.Context, clusterID string, requester string, description *string) error Create(ctx context.Context, clusters []inventory.Cluster) error Delete(ctx context.Context, clusterID string) error GetTags(ctx context.Context, clusterID string) ([]db.TagDBResponse, error) @@ -70,7 +70,7 @@ func (s *clusterServiceImpl) GetSummary(ctx context.Context) (inventory.Clusters } // PowerOn sends a request to power on a cluster. -func (s *clusterServiceImpl) PowerOn(ctx context.Context, clusterID string) error { +func (s *clusterServiceImpl) PowerOn(ctx context.Context, clusterID string, requester string, description *string) error { cluster, err := s.repo.GetClusterByID(ctx, clusterID) if err != nil { return err @@ -84,7 +84,6 @@ func (s *clusterServiceImpl) PowerOn(ctx context.Context, clusterID string) erro instanceIDs[i] = inst.InstanceID } - description := "triggered by user request" action := actions.NewPowerOnClusterAction( *actions.NewActionTarget( cluster.AccountID, @@ -92,15 +91,15 @@ func (s *clusterServiceImpl) PowerOn(ctx context.Context, clusterID string) erro cluster.ClusterID, instanceIDs, ), - "cluster-iq-API", // TODO: include username who created this request - &description, + requester, + description, ) return s.agentClient.PowerOnCluster(ctx, action) } // PowerOff sends a request to power off a cluster. -func (s *clusterServiceImpl) PowerOff(ctx context.Context, clusterID string) error { +func (s *clusterServiceImpl) PowerOff(ctx context.Context, clusterID string, requester string, description *string) error { cluster, err := s.repo.GetClusterByID(ctx, clusterID) if err != nil { return err @@ -114,7 +113,6 @@ func (s *clusterServiceImpl) PowerOff(ctx context.Context, clusterID string) err instanceIDs[i] = inst.InstanceID } - description := "triggered by user request" action := actions.NewPowerOffClusterAction( *actions.NewActionTarget( cluster.AccountID, @@ -122,8 +120,8 @@ func (s *clusterServiceImpl) PowerOff(ctx context.Context, clusterID string) err cluster.ClusterID, instanceIDs, ), - "cluster-iq-API", // TODO: include username who created this request - &description, + requester, + description, ) return s.agentClient.PowerOffCluster(ctx, action) From 0764dc405679c2b7679cc87e831c51ce96d747cc Mon Sep 17 00:00:00 2001 From: Alejandro Villegas Date: Wed, 11 Mar 2026 17:15:59 +0100 Subject: [PATCH 14/53] fix(agent): resolve memory leak in CronActions and unsafe type assertions (#260) * fix(agent): Fixed memory leak, logger call and minor errors Signed-off-by: r2dedios * fix(agent): Reduce cyclomatic complexity and adding handler functions for the different action statuses Signed-off-by: r2dedios * fix(agent): Fixed net.Listen linter error Signed-off-by: r2dedios --------- Signed-off-by: r2dedios --- cmd/agent/executor_agent_service.go | 181 +++++++++++++++++----------- cmd/agent/instant_agent_service.go | 5 +- cmd/agent/schedule_agent_service.go | 51 +++++--- 3 files changed, 151 insertions(+), 86 deletions(-) diff --git a/cmd/agent/executor_agent_service.go b/cmd/agent/executor_agent_service.go index 0d18e03f..82eee92e 100644 --- a/cmd/agent/executor_agent_service.go +++ b/cmd/agent/executor_agent_service.go @@ -172,94 +172,139 @@ func (e *ExecutorAgentService) createExecutors() error { // - accountID: The name of the account for which the executor is requested. // // Returns: -// - cexec.CloudExecutor: The executor for the specified account. -// - error: An error if no executor is found for the given account. -func (e *ExecutorAgentService) GetExecutor(accountID string) *cexec.CloudExecutor { +// - cexec.CloudExecutor: The executor for the specified account, or nil if not found. +func (e *ExecutorAgentService) GetExecutor(accountID string) cexec.CloudExecutor { exec, ok := e.executors[accountID] if !ok { return nil } - return &exec + return exec } func (e *ExecutorAgentService) Start() error { e.logger.Debug("Starting ExecutorAgentService") - // Reading actions from channel to prepare its execution for newAction := range e.actionsChannel { - e.logger.Debug("New action received by ExecutorAgentService", - zap.Any("action_id", newAction.GetID()), - zap.Any("requester", newAction.GetRequester()), - ) + e.processAction(newAction) + } - // Initialize event tracker - tracker := e.eventService.StartTracking(&eventservice.EventOptions{ - Action: newAction.GetActionOperation(), - Description: newAction.GetDescription(), - ResourceID: newAction.GetTarget().ClusterID, - ResourceType: inventory.ClusterResourceType, - Result: eventservice.ResultPending, - Severity: eventservice.SeverityInfo, - TriggeredBy: newAction.GetRequester(), - }) - // TODO: Keep this or transform the cannel into: 'chan *action.Action' - - // Mark the incoming action as 'Running' since it arrives to the ExecutorService - newAction.(actions.MutableAction).SetStatus(actions.StatusRunning) - if err := e.updateActionStatus(newAction); err != nil { - e.logger.Error("Error updating action status", zap.String("action_id", newAction.GetID()), zap.Error(err)) - tracker.Failed() - continue - } + return nil +} - exec := e.GetExecutor(newAction.GetTarget().AccountID) - if exec == nil { - e.logger.Error("there's no Executor available for the requested account", zap.String("account_id", newAction.GetTarget().AccountID)) +// processAction handles the complete lifecycle of a single action from channel to execution +func (e *ExecutorAgentService) processAction(action actions.Action) { + e.logger.Debug("New action received by ExecutorAgentService", + zap.Any("action_id", action.GetID()), + zap.Any("requester", action.GetRequester()), + ) + + // Initialize event tracker + tracker := e.eventService.StartTracking(&eventservice.EventOptions{ + Action: action.GetActionOperation(), + Description: action.GetDescription(), + ResourceID: action.GetTarget().ClusterID, + ResourceType: inventory.ClusterResourceType, + Result: eventservice.ResultPending, + Severity: eventservice.SeverityInfo, + TriggeredBy: action.GetRequester(), + }) + + // Mark as running + if !e.setActionStatus(action, actions.StatusRunning) { + tracker.Failed() + return + } - // Updating Action status - m := newAction.(actions.MutableAction) - m.SetStatus(actions.StatusFailed) - if err := e.updateActionStatus(newAction); err != nil { - e.logger.Error("Error updating action status", zap.String("action_id", newAction.GetID()), zap.Error(err)) - continue - } - tracker.Failed() + // Get executor + executor := e.GetExecutor(action.GetTarget().AccountID) + if executor == nil { + e.handleMissingExecutor(action, tracker) + return + } - continue - } + // Execute action + if err := executor.ProcessAction(action); err != nil { + e.handleExecutionFailure(action, tracker, err) + return + } - executor := *exec + // Mark as success + e.handleExecutionSuccess(action, tracker) - if err := executor.ProcessAction(newAction); err != nil { - e.logger.Error("Error while processing action", zap.String("action_id", newAction.GetID())) - newAction.(actions.MutableAction).SetStatus(actions.StatusFailed) - tracker.Failed() - } else { - e.logger.Info("Action execution correct", zap.String("action_id", newAction.GetID())) - newAction.(actions.MutableAction).SetStatus(actions.StatusSuccess) - tracker.Success() - } + // For CronActions, reset status back to Pending so they can be rescheduled + e.resetCronActionStatus(action) +} - // Update action status to Success/Failed - if err := e.updateActionStatus(newAction); err != nil { - e.logger.Error("Error updating action status", zap.String("action_id", newAction.GetID()), zap.Error(err)) - continue - } +// setActionStatus safely updates action status with type assertion. +// Returns false if update failed (caller should abort). +func (e *ExecutorAgentService) setActionStatus(action actions.Action, status actions.ActionStatus) bool { + mutable, ok := action.(actions.MutableAction) + if !ok { + e.logger.Warn("Action does not implement MutableAction, skipping status update", + zap.String("action_id", action.GetID())) + return true // Not an error, just skip + } - // For CronActions, reset status back to Pending so they can be rescheduled - if newAction.GetType() == actions.CronActionType { - e.logger.Debug("Resetting CronAction status to Pending for next execution", - zap.String("action_id", newAction.GetID()), - ) - newAction.(actions.MutableAction).SetStatus(actions.StatusPending) - if err := e.updateActionStatus(newAction); err != nil { - e.logger.Error("Error resetting CronAction status to Pending", zap.String("action_id", newAction.GetID()), zap.Error(err)) - continue - } - } + mutable.SetStatus(status) + if err := e.updateActionStatus(action); err != nil { + e.logger.Error("Error updating action status", + zap.String("action_id", action.GetID()), + zap.Error(err)) + return false } + return true +} - return nil +// handleMissingExecutor handles the case when no executor is available for the account +func (e *ExecutorAgentService) handleMissingExecutor(action actions.Action, tracker *eventservice.EventTracker) { + e.logger.Error("there's no Executor available for the requested account", + zap.String("account_id", action.GetTarget().AccountID)) + + e.setActionStatus(action, actions.StatusFailed) + tracker.Failed() +} + +// handleExecutionFailure handles action execution failures +func (e *ExecutorAgentService) handleExecutionFailure(action actions.Action, tracker *eventservice.EventTracker, err error) { + e.logger.Error("Error while processing action", + zap.String("action_id", action.GetID()), + zap.Error(err)) + + e.setActionStatus(action, actions.StatusFailed) + tracker.Failed() +} + +// handleExecutionSuccess handles successful action execution +func (e *ExecutorAgentService) handleExecutionSuccess(action actions.Action, tracker *eventservice.EventTracker) { + e.logger.Info("Action execution correct", + zap.String("action_id", action.GetID())) + + e.setActionStatus(action, actions.StatusSuccess) + tracker.Success() +} + +// resetCronActionStatus resets CronAction status to Pending after execution so they can be rescheduled +func (e *ExecutorAgentService) resetCronActionStatus(action actions.Action) { + if action.GetType() != actions.CronActionType { + return + } + + e.logger.Debug("Resetting CronAction status to Pending for next execution", + zap.String("action_id", action.GetID())) + + mutable, ok := action.(actions.MutableAction) + if !ok { + e.logger.Warn("CronAction does not implement MutableAction, cannot reset status", + zap.String("action_id", action.GetID())) + return + } + + mutable.SetStatus(actions.StatusPending) + if err := e.updateActionStatus(action); err != nil { + e.logger.Error("Error resetting CronAction status to Pending", + zap.String("action_id", action.GetID()), + zap.Error(err)) + } } func (e *ExecutorAgentService) updateActionStatus(action actions.Action) error { diff --git a/cmd/agent/instant_agent_service.go b/cmd/agent/instant_agent_service.go index e2bbf44a..d28efb4e 100644 --- a/cmd/agent/instant_agent_service.go +++ b/cmd/agent/instant_agent_service.go @@ -40,7 +40,8 @@ type InstantAgentService struct { // - *InstantAgentService: A pointer to the newly created AgentService instance. func NewInstantAgentService(cfg *config.InstantAgentServiceConfig, actionsChannel chan<- actions.Action, wg *sync.WaitGroup, logger *zap.Logger) *InstantAgentService { // Listener config - lis, err := net.Listen("tcp", cfg.ListenURL) + lc := net.ListenConfig{} + lis, err := lc.Listen(context.Background(), "tcp", cfg.ListenURL) if err != nil { logger.Error("Error initializing gRPC AgentService on ClusterIQ Agent", zap.Error(err)) return nil @@ -87,7 +88,7 @@ func (i *InstantAgentService) Start() error { // Serving gRPC if err := i.grpcServer.Serve(i.listener); err != nil { - logger.Fatal("failed to start server", zap.Error(err)) + i.logger.Fatal("failed to start server", zap.Error(err)) return err } return nil diff --git a/cmd/agent/schedule_agent_service.go b/cmd/agent/schedule_agent_service.go index 79ccd85a..78c5492d 100644 --- a/cmd/agent/schedule_agent_service.go +++ b/cmd/agent/schedule_agent_service.go @@ -27,8 +27,9 @@ const ( // scheduleItem represents the pair of action and CancelFunc for tracking the already running actions type scheduleItem struct { - cancel context.CancelFunc - action actions.Action + cancel context.CancelFunc + action actions.Action + cronInst *cron.Cron // Only used for CronActions, nil for ScheduledActions } // ScheduleAgentService represents the main structure for managing scheduled actions of ClusterIQ @@ -79,7 +80,7 @@ func NewScheduleAgentService(cfg *config.ScheduleAgentServiceConfig, actionsChan // - newAction: the new actions.ScheduledAction to be executed // // Returns: -func (a *ScheduleAgentService) scheduleNewScheduledAction(newAction actions.ScheduledAction) { +func (a *ScheduleAgentService) scheduleNewScheduledAction(newAction *actions.ScheduledAction) { actionID := newAction.GetID() // Check if the duration is negative, which means it refers to a past timestamp @@ -103,7 +104,7 @@ func (a *ScheduleAgentService) scheduleNewScheduledAction(newAction actions.Sche select { case <-time.After(duration): // When the timestamp is "now" a.logger.Debug("Sending to execution channel", zap.String("action_id", actionID), zap.Int("channel", len(a.actionsChannel))) - a.actionsChannel <- &newAction + a.actionsChannel <- newAction a.logger.Debug("Action sent to execution channel", zap.String("action_id", actionID), zap.Int("channel", len(a.actionsChannel))) case <-ctx.Done(): // Context cancelling a.logger.Warn("Task cancelled before execution", zap.String("action_id", actionID)) @@ -123,7 +124,7 @@ func (a *ScheduleAgentService) scheduleNewScheduledAction(newAction actions.Sche // - newAction: the new actions.ScheduledAction to be executed // // Returns: -func (a *ScheduleAgentService) rescheduleScheduledAction(newAction actions.ScheduledAction) { +func (a *ScheduleAgentService) rescheduleScheduledAction(newAction *actions.ScheduledAction) { actionID := newAction.GetID() if !reflect.DeepEqual(a.schedule[actionID].action, newAction) { @@ -142,36 +143,54 @@ func (a *ScheduleAgentService) rescheduleScheduledAction(newAction actions.Sched // - newAction: the new actions.ScheduledAction to be executed // // Returns: -func (a *ScheduleAgentService) scheduleNewCronAction(newAction actions.CronAction) { +func (a *ScheduleAgentService) scheduleNewCronAction(newAction *actions.CronAction) { actionID := newAction.GetID() // Creating new action context and cancel function ctx, cancel := context.WithCancel(context.Background()) + + // Create cron instance before goroutine + c := cron.New() + + // Store in schedule with cron instance a.schedule[actionID] = scheduleItem{ - cancel: cancel, - action: newAction, + cancel: cancel, + action: newAction, + cronInst: c, } a.logger.Info("New CronAction being scheduled", zap.String("action_id", actionID), zap.String("action_cron_exp", newAction.GetCronExpression())) // Scheduling at specified timestamp on parallel go func() { a.logger.Debug("Starting CronAction execution", zap.String("action_id", actionID), zap.String("action_cron_exp", newAction.GetCronExpression())) - c := cron.New() + _, err := c.AddFunc(newAction.GetCronExpression(), func() { select { case <-ctx.Done(): a.logger.Warn("Task cancelled before execution", zap.String("action_id", actionID), zap.String("action_cron_exp", newAction.GetCronExpression())) default: a.logger.Debug("Sending to execution channel", zap.String("action_id", actionID), zap.Int("channel", len(a.actionsChannel))) - a.actionsChannel <- &newAction + a.actionsChannel <- newAction a.logger.Debug("Action sent to execution channel", zap.String("action_id", actionID), zap.Int("channel", len(a.actionsChannel))) } }) if err != nil { a.logger.Error("Failed adding new CronAction execution", zap.Error(err)) + return } c.Start() // Cron Start + + // Wait for cancellation and stop cron + <-ctx.Done() + a.logger.Debug("Stopping cron scheduler for action", zap.String("action_id", actionID)) + c.Stop() + + // Remove action from schedule + a.mutex.Lock() + delete(a.schedule, actionID) + a.logger.Debug("Removing cron action from schedule since it was cancelled", zap.String("action_id", actionID)) + a.mutex.Unlock() }() } @@ -181,7 +200,7 @@ func (a *ScheduleAgentService) scheduleNewCronAction(newAction actions.CronActio // - newAction: the new actions.ScheduledAction to be executed // // Returns: -func (a *ScheduleAgentService) rescheduleCronAction(newAction actions.CronAction) { +func (a *ScheduleAgentService) rescheduleCronAction(newAction *actions.CronAction) { actionID := newAction.GetID() if !reflect.DeepEqual(a.schedule[actionID].action, newAction) { @@ -214,16 +233,16 @@ func (a *ScheduleAgentService) ScheduleNewActions(newSchedule []actions.Action) // Checking which actions must be cancelled if are missing on 'newSchedule' for id, item := range a.schedule { if _, exists := actionMap[id]; !exists { + // Cancel the action - the goroutine will handle cleanup (delete + cron.Stop) item.cancel() - delete(a.schedule, id) a.logger.Warn("Action Cancelled", zap.String("action_id", id)) } } // Checking the entire new schedule to schedule or reschedule actions for _, action := range newSchedule { - var scheduledFunc func(actions.ScheduledAction) - var cronFunc func(actions.CronAction) + var scheduledFunc func(*actions.ScheduledAction) + var cronFunc func(*actions.CronAction) if _, exists := a.schedule[action.GetID()]; !exists { // Schedule new actions scheduledFunc = a.scheduleNewScheduledAction @@ -236,9 +255,9 @@ func (a *ScheduleAgentService) ScheduleNewActions(newSchedule []actions.Action) // managing actions based on type switch t := action.(type) { case *actions.ScheduledAction: - scheduledFunc(*t) + scheduledFunc(t) case *actions.CronAction: - cronFunc(*t) + cronFunc(t) default: a.logger.Error("Unknown action type", zap.String("action_id", action.GetID())) } From 02a70d4567dc7497e70d38d351eee91ca645a6e6 Mon Sep 17 00:00:00 2001 From: r2dedios Date: Thu, 12 Mar 2026 11:48:15 +0100 Subject: [PATCH 15/53] feat(dto): Added Description and Requester parameters to DTOs Signed-off-by: r2dedios --- internal/models/dto/action_dto.go | 78 ++++++++++++++++--------------- 1 file changed, 41 insertions(+), 37 deletions(-) diff --git a/internal/models/dto/action_dto.go b/internal/models/dto/action_dto.go index 2dd36a2f..649b3223 100644 --- a/internal/models/dto/action_dto.go +++ b/internal/models/dto/action_dto.go @@ -9,17 +9,19 @@ import ( // ActionDTORequest represents the data needed to create or update an action. type ActionDTORequest struct { - ID string `json:"id"` - Type string `json:"type"` - Time time.Time `json:"time"` - CronExp string `json:"cronExpression"` - Operation string `json:"operation"` - Status string `json:"status"` - Enabled bool `json:"enabled"` - ClusterID string `json:"clusterId"` - Region string `json:"region"` - AccountID string `json:"accountId"` - Instances []string `json:"instances"` + ID string `json:"id"` + Type string `json:"type"` + Time time.Time `json:"time"` + CronExp string `json:"cronExpression"` + Operation string `json:"operation"` + Status string `json:"status"` + Enabled bool `json:"enabled"` + ClusterID string `json:"clusterId"` + Region string `json:"region"` + AccountID string `json:"accountId"` + Instances []string `json:"instances"` + Requester string `json:"requester"` + Description *string `json:"description,omitempty"` } // @name ActionRequest func (a ActionDTORequest) ToModelAction() actions.Action { @@ -36,8 +38,8 @@ func (a ActionDTORequest) ToModelAction() actions.Action { actions.ActionOperation(a.Operation), target, actions.ActionStatus(a.Status), - "", // TODO: Requester missing??? - nil, // TODO: Description missing??? + a.Requester, + a.Description, a.Enabled, a.Time, ) @@ -48,8 +50,8 @@ func (a ActionDTORequest) ToModelAction() actions.Action { actions.ActionOperation(a.Operation), target, actions.ActionStatus(a.Status), - "", // TODO: Requester missing??? - nil, // TODO: Description missing??? + a.Requester, + a.Description, a.Enabled, a.CronExp, ) @@ -60,8 +62,8 @@ func (a ActionDTORequest) ToModelAction() actions.Action { actions.ActionOperation(a.Operation), target, actions.ActionStatus(a.Status), - "", // TODO: Requester missing??? - nil, // TODO: Description missing??? + a.Requester, + a.Description, a.Enabled, ) action.ID = a.ID @@ -87,17 +89,19 @@ func ToModelActionList(dtos []ActionDTORequest) *[]actions.Action { // ActionDTOResponse represents the data transfer object for an action response, // containing action details including schedule, cron expression, and target resources. type ActionDTOResponse struct { - ID string `json:"id"` - Type string `json:"type"` - Time time.Time `json:"time"` - CronExp string `json:"cronExpression"` - Operation string `json:"operation"` - Status string `json:"status"` - Enabled bool `json:"enabled"` - ClusterID string `json:"clusterId"` - Region string `json:"region"` - AccountID string `json:"accountId"` - Instances []string `json:"instances"` + ID string `json:"id"` + Type string `json:"type"` + Time time.Time `json:"time"` + CronExp string `json:"cronExpression"` + Operation string `json:"operation"` + Status string `json:"status"` + Enabled bool `json:"enabled"` + ClusterID string `json:"clusterId"` + Region string `json:"region"` + AccountID string `json:"accountId"` + Instances []string `json:"instances"` + Requester string `json:"requester"` + Description *string `json:"description,omitempty"` } // @name ActionResponse // ToModelAction converts ActionDTOResponse to actions.Action @@ -115,8 +119,8 @@ func (a ActionDTOResponse) ToModelAction() actions.Action { actions.ActionOperation(a.Operation), target, actions.ActionStatus(a.Status), - "", // TODO: Requester missing??? - nil, // TODO: Description missing??? + a.Requester, + a.Description, a.Enabled, a.Time, ) @@ -127,8 +131,8 @@ func (a ActionDTOResponse) ToModelAction() actions.Action { actions.ActionOperation(a.Operation), target, actions.ActionStatus(a.Status), - "", // TODO: Requester missing??? - nil, // TODO: Description missing??? + a.Requester, + a.Description, a.Enabled, a.CronExp, ) @@ -139,8 +143,8 @@ func (a ActionDTOResponse) ToModelAction() actions.Action { actions.ActionOperation(a.Operation), target, actions.ActionStatus(a.Status), - "", // TODO: Requester missing??? - nil, // TODO: Description missing??? + a.Requester, + a.Description, a.Enabled, ) action.ID = a.ID @@ -163,8 +167,8 @@ func ToModelActionListFromResponse(dtos []ActionDTOResponse) ([]actions.Action, return resultActions, nil } -// PowerActionRequest defines the information required by the API to complement the actions and events for a Power On/Off request0 +// PowerActionRequest defines the information required by the API to complement the actions and events for a Power On/Off request type PowerActionRequest struct { - Requester string `json:"requester" binding:"required"` - Description string `json:"description"` + Requester string `json:"requester" binding:"required,max=100"` + Description string `json:"description" binding:"max=500"` } From dda2ab8a14c2f77fc50faf2be7a4b0825e670bd8 Mon Sep 17 00:00:00 2001 From: r2dedios Date: Thu, 12 Mar 2026 11:49:39 +0100 Subject: [PATCH 16/53] Fix: Fixed context managemenet from API handlers to service/repository Signed-off-by: r2dedios --- internal/api/handlers/cluster_handler.go | 8 +++++-- .../events/event_service/event_service.go | 21 ++++++++++--------- .../event_service/event_service_test.go | 8 +++---- 3 files changed, 21 insertions(+), 16 deletions(-) diff --git a/internal/api/handlers/cluster_handler.go b/internal/api/handlers/cluster_handler.go index b380cb0b..59792dd8 100644 --- a/internal/api/handlers/cluster_handler.go +++ b/internal/api/handlers/cluster_handler.go @@ -267,7 +267,9 @@ func (h *ClusterHandler) PowerOn(c *gin.Context) { // Bind JSON body to request struct if err := c.ShouldBindJSON(&request); err != nil { - h.logger.Error("Bad request for PowerOn cluster operation", zap.String("cluster_id", clusterID), zap.String("requester", request.Requester), zap.String("description", request.Description)) + h.logger.Error("Bad request for power on cluster operation - invalid JSON body", + zap.String("cluster_id", clusterID), + zap.Error(err)) c.JSON(http.StatusBadRequest, responsetypes.GenericErrorResponse{ Message: "Invalid request body: " + err.Error(), }) @@ -317,7 +319,9 @@ func (h *ClusterHandler) PowerOff(c *gin.Context) { // Bind JSON body to request struct if err := c.ShouldBindJSON(&request); err != nil { - h.logger.Error("Bad request for PowerOff cluster operation", zap.String("cluster_id", clusterID), zap.String("requester", request.Requester), zap.String("description", request.Description)) + h.logger.Error("Bad request for power off cluster operation - invalid JSON body", + zap.String("cluster_id", clusterID), + zap.Error(err)) c.JSON(http.StatusBadRequest, responsetypes.GenericErrorResponse{ Message: "Invalid request body: " + err.Error(), }) diff --git a/internal/events/event_service/event_service.go b/internal/events/event_service/event_service.go index 93bf9635..c9a04227 100644 --- a/internal/events/event_service/event_service.go +++ b/internal/events/event_service/event_service.go @@ -50,7 +50,7 @@ func NewEventService(dbClient *dbclient.DBClient, logger *zap.Logger) *EventServ } // LogEvent creates a new events log entry and returns its ID. -func (e *EventService) LogEvent(opts EventOptions) (int64, error) { +func (e *EventService) LogEvent(ctx context.Context, opts EventOptions) (int64, error) { event := events.Event{ TriggeredBy: opts.TriggeredBy, Action: opts.Action, @@ -61,8 +61,7 @@ func (e *EventService) LogEvent(opts EventOptions) (int64, error) { Severity: opts.Severity, EventTimestamp: time.Now().UTC(), } - // TODO Fix replace TODO context by request's context - eventID, err := e.repo.CreateEvent(context.TODO(), event) + eventID, err := e.repo.CreateEvent(ctx, event) e.logger.Debug("Tracking new event", zap.Int64("event_id", eventID)) if err != nil { e.logger.Error("Failed to log event", zap.Error(err)) @@ -72,9 +71,8 @@ func (e *EventService) LogEvent(opts EventOptions) (int64, error) { } // UpdateEventStatus updates the result status of an existing event. -func (e *EventService) UpdateEventStatus(eventID int64, result string) error { - // TODO Fix replace TODO context by request's context - err := e.repo.UpdateEventStatus(context.TODO(), eventID, result) +func (e *EventService) UpdateEventStatus(ctx context.Context, eventID int64, result string) error { + err := e.repo.UpdateEventStatus(ctx, eventID, result) if err != nil { e.logger.Error("Failed to update event status", zap.Int64("event_id", eventID), zap.Error(err)) return err @@ -83,8 +81,9 @@ func (e *EventService) UpdateEventStatus(eventID int64, result string) error { } // StartTracking begins tracking a new event and returns an EventTracker. +// Uses context.Background() as events are tracked asynchronously from HTTP requests. func (e *EventService) StartTracking(opts *EventOptions) *EventTracker { - eventID, err := e.LogEvent(*opts) + eventID, err := e.LogEvent(context.Background(), *opts) if err != nil { e.logger.Error("Failed to log initial event", zap.Error(err)) return nil @@ -103,16 +102,18 @@ type EventTracker struct { logger *zap.Logger } -// Failed marks the tracked event status as failed. +// Success marks the tracked event status as success. +// Uses context.Background() as event updates happen asynchronously. func (t *EventTracker) Success() { - if err := t.service.UpdateEventStatus(t.eventID, ResultSuccess); err != nil { + if err := t.service.UpdateEventStatus(context.Background(), t.eventID, ResultSuccess); err != nil { t.logger.Error("Failed to update event status", zap.Error(err)) } } // Failed marks the tracked event as failed. +// Uses context.Background() as event updates happen asynchronously. func (t *EventTracker) Failed() { - if err := t.service.UpdateEventStatus(t.eventID, ResultFailed); err != nil { + if err := t.service.UpdateEventStatus(context.Background(), t.eventID, ResultFailed); err != nil { t.logger.Error("Failed to update event status", zap.Error(err)) } } diff --git a/internal/events/event_service/event_service_test.go b/internal/events/event_service/event_service_test.go index 558f9b58..5ae45fb9 100644 --- a/internal/events/event_service/event_service_test.go +++ b/internal/events/event_service/event_service_test.go @@ -111,7 +111,7 @@ func testLogEvent_Success(t *testing.T) { svc := &EventService{repo: repo, logger: zap.NewNop()} before := time.Now().UTC() - id, err := svc.LogEvent(opts) + id, err := svc.LogEvent(context.Background(), opts) after := time.Now().UTC() assert.NoError(t, err) @@ -153,7 +153,7 @@ func testLogEvent_RepoError(t *testing.T) { svc := &EventService{repo: repo, logger: zap.NewNop()} - id, err := svc.LogEvent(opts) + id, err := svc.LogEvent(context.Background(), opts) assert.Error(t, err) assert.Equal(t, int64(0), id) assert.Equal(t, 1, repo.createEventCalls) @@ -174,7 +174,7 @@ func testUpdateEventStatus_Success(t *testing.T) { svc := &EventService{repo: repo, logger: zap.NewNop()} - err := svc.UpdateEventStatus(7, ResultSuccess) + err := svc.UpdateEventStatus(context.Background(),7, ResultSuccess) assert.NoError(t, err) assert.Equal(t, 1, repo.updateEventStatusCalls) @@ -192,7 +192,7 @@ func testUpdateEventStatus_RepoError(t *testing.T) { svc := &EventService{repo: repo, logger: zap.NewNop()} - err := svc.UpdateEventStatus(7, ResultFailed) + err := svc.UpdateEventStatus(context.Background(),7, ResultFailed) assert.Error(t, err) assert.Equal(t, 1, repo.updateEventStatusCalls) From d73489fd1e4c95d27c936be669b0f4e0dc28d7a0 Mon Sep 17 00:00:00 2001 From: r2dedios Date: Thu, 12 Mar 2026 16:29:51 +0100 Subject: [PATCH 17/53] refactor(api): eliminate code duplication in power handlers and fix typos Signed-off-by: r2dedios --- README.md | 2 +- cmd/agent/executor_agent_service.go | 2 +- cmd/agent/schedule_agent_service.go | 2 +- internal/api/handlers/cluster_handler.go | 95 ++++++++++-------------- 4 files changed, 42 insertions(+), 59 deletions(-) diff --git a/README.md b/README.md index 27c0f4a0..5e5da8fc 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,7 @@ The following graph shows the architecture of this project: ### Secrets Management (Optional) By default, Cluster IQ uses standard Kubernetes Secrets. For production environments, you can integrate with HashiCorp Vault. -See [Vault Integration Guide](doc/vault/readme.md) for detailed setup instructions. +See [Vault Integration Guide](doc/vault/README.md) for detailed setup instructions. ## Installation ClusterIQ was designed to run in K8s/Openshift platforms, but it can also run in diff --git a/cmd/agent/executor_agent_service.go b/cmd/agent/executor_agent_service.go index 82eee92e..5ba8b830 100644 --- a/cmd/agent/executor_agent_service.go +++ b/cmd/agent/executor_agent_service.go @@ -139,7 +139,7 @@ func (e *ExecutorAgentService) createExecutors() error { ) err = e.AddExecutor(exec) if err != nil { - e.logger.Error("Cannot create an AWSEexecutor for account", zap.String("account_id", newAccount.AccountID), zap.Error(err)) + e.logger.Error("Cannot create an AWSExecutor for account", zap.String("account_id", newAccount.AccountID), zap.Error(err)) return err } diff --git a/cmd/agent/schedule_agent_service.go b/cmd/agent/schedule_agent_service.go index 78c5492d..1c52f3e3 100644 --- a/cmd/agent/schedule_agent_service.go +++ b/cmd/agent/schedule_agent_service.go @@ -344,7 +344,7 @@ func (a *ScheduleAgentService) ReScheduleActions() { a.logger.Info("Adding new Actions to Agent Schedule", zap.Int("added_actions", len(*fetchedActions)), zap.Int("running_actions", len(a.schedule))) } <-ticker.C - a.logger.Debug("Current actions after pooling & rescheduling", zap.Int("actions_num", len(a.schedule))) + a.logger.Debug("Current actions after polling & rescheduling", zap.Int("actions_num", len(a.schedule))) } } diff --git a/internal/api/handlers/cluster_handler.go b/internal/api/handlers/cluster_handler.go index 59792dd8..4e53b6a7 100644 --- a/internal/api/handlers/cluster_handler.go +++ b/internal/api/handlers/cluster_handler.go @@ -1,6 +1,7 @@ package handlers import ( + "context" "errors" "net/http" "strconv" @@ -247,27 +248,19 @@ func (h *ClusterHandler) Delete(c *gin.Context) { c.Status(http.StatusNoContent) } -// PowerOn triggers a power-on operation for a cluster. -// -// @Summary Power on a cluster -// @Description Request powering on a cluster. -// @Tags Clusters -// @Accept json -// @Produce json -// @Param id path string true "Cluster ID" -// @Param request body dto.PowerActionRequest true "Power action details" -// @Success 202 {object} responsetypes.GenericResponse -// @Failure 400 {object} responsetypes.GenericErrorResponse -// @Failure 404 {object} responsetypes.GenericErrorResponse -// @Failure 500 {object} responsetypes.GenericErrorResponse -// @Router /clusters/{id}/power_on [post] -func (h *ClusterHandler) PowerOn(c *gin.Context) { +// handlePowerAction is a generic handler for power operations (on/off) on clusters. +// It reduces code duplication by centralizing the common logic for both operations. +func (h *ClusterHandler) handlePowerAction( + c *gin.Context, + action string, + serviceFunc func(ctx context.Context, clusterID string, requester string, description *string) error, +) { clusterID := c.Param("id") var request dto.PowerActionRequest // Bind JSON body to request struct if err := c.ShouldBindJSON(&request); err != nil { - h.logger.Error("Bad request for power on cluster operation - invalid JSON body", + h.logger.Error("Bad request for "+action+" cluster operation - invalid JSON body", zap.String("cluster_id", clusterID), zap.Error(err)) c.JSON(http.StatusBadRequest, responsetypes.GenericErrorResponse{ @@ -277,10 +270,16 @@ func (h *ClusterHandler) PowerOn(c *gin.Context) { } // Log requester and description - h.logger.Info("Power on request received", zap.String("requester", request.Requester), zap.String("description", request.Description)) + h.logger.Info(action+" request received", + zap.String("cluster_id", clusterID), + zap.String("requester", request.Requester), + zap.String("description", request.Description)) + + if err := serviceFunc(c.Request.Context(), clusterID, request.Requester, &request.Description); err != nil { + h.logger.Error("error "+action+" a cluster", + zap.String("cluster_id", clusterID), + zap.Error(err)) - if err := h.service.PowerOn(c.Request.Context(), clusterID, request.Requester, &request.Description); err != nil { - h.logger.Error("error powering on a cluster", zap.String("cluster_id", clusterID), zap.Error(err)) if errors.Is(err, repositories.ErrNotFound) { c.JSON(http.StatusNotFound, responsetypes.GenericErrorResponse{ Message: "Cluster not found", @@ -289,16 +288,34 @@ func (h *ClusterHandler) PowerOn(c *gin.Context) { } c.JSON(http.StatusInternalServerError, responsetypes.GenericErrorResponse{ - Message: "Failed to power on cluster: " + err.Error(), + Message: "Failed to " + action + " cluster: " + err.Error(), }) return } c.JSON(http.StatusAccepted, responsetypes.GenericResponse{ - Message: "Power on request accepted", + Message: action + " request accepted", }) } +// PowerOn triggers a power-on operation for a cluster. +// +// @Summary Power on a cluster +// @Description Request powering on a cluster. +// @Tags Clusters +// @Accept json +// @Produce json +// @Param id path string true "Cluster ID" +// @Param request body dto.PowerActionRequest true "Power action details" +// @Success 202 {object} responsetypes.GenericResponse +// @Failure 400 {object} responsetypes.GenericErrorResponse +// @Failure 404 {object} responsetypes.GenericErrorResponse +// @Failure 500 {object} responsetypes.GenericErrorResponse +// @Router /clusters/{id}/power_on [post] +func (h *ClusterHandler) PowerOn(c *gin.Context) { + h.handlePowerAction(c, "power on", h.service.PowerOn) +} + // PowerOff triggers a power-off operation for a cluster. // // @Summary Power off a cluster @@ -314,41 +331,7 @@ func (h *ClusterHandler) PowerOn(c *gin.Context) { // @Failure 500 {object} responsetypes.GenericErrorResponse // @Router /clusters/{id}/power_off [post] func (h *ClusterHandler) PowerOff(c *gin.Context) { - clusterID := c.Param("id") - var request dto.PowerActionRequest - - // Bind JSON body to request struct - if err := c.ShouldBindJSON(&request); err != nil { - h.logger.Error("Bad request for power off cluster operation - invalid JSON body", - zap.String("cluster_id", clusterID), - zap.Error(err)) - c.JSON(http.StatusBadRequest, responsetypes.GenericErrorResponse{ - Message: "Invalid request body: " + err.Error(), - }) - return - } - - // Log requester and description - h.logger.Info("Power off request received", zap.String("requester", request.Requester), zap.String("description", request.Description)) - - if err := h.service.PowerOff(c.Request.Context(), clusterID, request.Requester, &request.Description); err != nil { - h.logger.Error("error powering off a cluster", zap.String("cluster_id", clusterID), zap.Error(err)) - if errors.Is(err, repositories.ErrNotFound) { - c.JSON(http.StatusNotFound, responsetypes.GenericErrorResponse{ - Message: "Cluster not found", - }) - return - } - - c.JSON(http.StatusInternalServerError, responsetypes.GenericErrorResponse{ - Message: "Failed to power off cluster: " + err.Error(), - }) - return - } - - c.JSON(http.StatusAccepted, responsetypes.GenericResponse{ - Message: "Power off request accepted", - }) + h.handlePowerAction(c, "power off", h.service.PowerOff) } // GetTags returns tags for the specified cluster. From dfdba2133423ace7055809cb141340c4e099497b Mon Sep 17 00:00:00 2001 From: r2dedios Date: Mon, 16 Mar 2026 17:56:04 +0100 Subject: [PATCH 18/53] refactor(db): remove context.TODO() and add scanner timeout Signed-off-by: r2dedios --- cmd/scanner/scanner.go | 11 ++++- internal/models/dto/action_dto.go | 72 +++++++++++++++---------------- 2 files changed, 44 insertions(+), 39 deletions(-) diff --git a/cmd/scanner/scanner.go b/cmd/scanner/scanner.go index a3bf1e82..c28ca6b1 100644 --- a/cmd/scanner/scanner.go +++ b/cmd/scanner/scanner.go @@ -13,6 +13,7 @@ import ( "os/signal" "sync" "syscall" + "time" responsetypes "github.com/RHEcosystemAppEng/cluster-iq/internal/api/response_types" "github.com/RHEcosystemAppEng/cluster-iq/internal/config" @@ -30,6 +31,9 @@ const ( apiClusterEndpoint = "/clusters" apiInstanceEndpoint = "/instances" apiExpenseEndpoint = "/expenses" + + // apiRequestTimeout defines the timeout for HTTP POST requests to the API + apiRequestTimeout = 60 * time.Second ) var ( @@ -396,7 +400,12 @@ func (s *Scanner) postScannerInventory() error { func postData(path string, b []byte) error { url := fmt.Sprintf("%s%s", APIURL, path) - request, err := http.NewRequestWithContext(context.TODO(), http.MethodPost, url, bytes.NewBuffer(b)) + + // Create context with timeout for API requests + ctx, cancel := context.WithTimeout(context.Background(), apiRequestTimeout) + defer cancel() + + request, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewBuffer(b)) if err != nil { return err } diff --git a/internal/models/dto/action_dto.go b/internal/models/dto/action_dto.go index 649b3223..553b2563 100644 --- a/internal/models/dto/action_dto.go +++ b/internal/models/dto/action_dto.go @@ -9,19 +9,17 @@ import ( // ActionDTORequest represents the data needed to create or update an action. type ActionDTORequest struct { - ID string `json:"id"` - Type string `json:"type"` - Time time.Time `json:"time"` - CronExp string `json:"cronExpression"` - Operation string `json:"operation"` - Status string `json:"status"` - Enabled bool `json:"enabled"` - ClusterID string `json:"clusterId"` - Region string `json:"region"` - AccountID string `json:"accountId"` - Instances []string `json:"instances"` - Requester string `json:"requester"` - Description *string `json:"description,omitempty"` + ID string `json:"id"` + Type string `json:"type"` + Time time.Time `json:"time"` + CronExp string `json:"cronExpression"` + Operation string `json:"operation"` + Status string `json:"status"` + Enabled bool `json:"enabled"` + ClusterID string `json:"clusterId"` + Region string `json:"region"` + AccountID string `json:"accountId"` + Instances []string `json:"instances"` } // @name ActionRequest func (a ActionDTORequest) ToModelAction() actions.Action { @@ -38,8 +36,8 @@ func (a ActionDTORequest) ToModelAction() actions.Action { actions.ActionOperation(a.Operation), target, actions.ActionStatus(a.Status), - a.Requester, - a.Description, + "", // TODO: Requester missing??? + nil, // TODO: Description missing??? a.Enabled, a.Time, ) @@ -50,8 +48,8 @@ func (a ActionDTORequest) ToModelAction() actions.Action { actions.ActionOperation(a.Operation), target, actions.ActionStatus(a.Status), - a.Requester, - a.Description, + "", // TODO: Requester missing??? + nil, // TODO: Description missing??? a.Enabled, a.CronExp, ) @@ -62,8 +60,8 @@ func (a ActionDTORequest) ToModelAction() actions.Action { actions.ActionOperation(a.Operation), target, actions.ActionStatus(a.Status), - a.Requester, - a.Description, + "", // TODO: Requester missing??? + nil, // TODO: Description missing??? a.Enabled, ) action.ID = a.ID @@ -89,19 +87,17 @@ func ToModelActionList(dtos []ActionDTORequest) *[]actions.Action { // ActionDTOResponse represents the data transfer object for an action response, // containing action details including schedule, cron expression, and target resources. type ActionDTOResponse struct { - ID string `json:"id"` - Type string `json:"type"` - Time time.Time `json:"time"` - CronExp string `json:"cronExpression"` - Operation string `json:"operation"` - Status string `json:"status"` - Enabled bool `json:"enabled"` - ClusterID string `json:"clusterId"` - Region string `json:"region"` - AccountID string `json:"accountId"` - Instances []string `json:"instances"` - Requester string `json:"requester"` - Description *string `json:"description,omitempty"` + ID string `json:"id"` + Type string `json:"type"` + Time time.Time `json:"time"` + CronExp string `json:"cronExpression"` + Operation string `json:"operation"` + Status string `json:"status"` + Enabled bool `json:"enabled"` + ClusterID string `json:"clusterId"` + Region string `json:"region"` + AccountID string `json:"accountId"` + Instances []string `json:"instances"` } // @name ActionResponse // ToModelAction converts ActionDTOResponse to actions.Action @@ -119,8 +115,8 @@ func (a ActionDTOResponse) ToModelAction() actions.Action { actions.ActionOperation(a.Operation), target, actions.ActionStatus(a.Status), - a.Requester, - a.Description, + "", // TODO: Requester missing??? + nil, // TODO: Description missing??? a.Enabled, a.Time, ) @@ -131,8 +127,8 @@ func (a ActionDTOResponse) ToModelAction() actions.Action { actions.ActionOperation(a.Operation), target, actions.ActionStatus(a.Status), - a.Requester, - a.Description, + "", // TODO: Requester missing??? + nil, // TODO: Description missing??? a.Enabled, a.CronExp, ) @@ -143,8 +139,8 @@ func (a ActionDTOResponse) ToModelAction() actions.Action { actions.ActionOperation(a.Operation), target, actions.ActionStatus(a.Status), - a.Requester, - a.Description, + "", // TODO: Requester missing??? + nil, // TODO: Description missing??? a.Enabled, ) action.ID = a.ID From 778a1bcec01101293bb89124a7b922b1be054916 Mon Sep 17 00:00:00 2001 From: r2dedios Date: Mon, 16 Mar 2026 18:16:18 +0100 Subject: [PATCH 19/53] docs(db_client): add deprecation comments to non-context wrapper methods Added deprecation comments to Get, Select, Insert, Update, NamedUpdate, and Delete methods to guide developers toward using WithContext versions for proper timeout and cancellation support. Co-Authored-By: Claude Sonnet 4.5 --- internal/db_client/db_client.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/internal/db_client/db_client.go b/internal/db_client/db_client.go index c9fa1311..a916eb01 100644 --- a/internal/db_client/db_client.go +++ b/internal/db_client/db_client.go @@ -82,6 +82,8 @@ func (d *DBClient) GetWithContext(ctx context.Context, dest interface{}, table s return d.db.GetContext(ctx, dest, query, args...) } +// Get is a convenience wrapper without context. +// Deprecated: Prefer GetWithContext for proper timeout and cancellation support. func (d *DBClient) Get(dest interface{}, table string, opts models.ListOptions, columns ...string) error { return d.GetWithContext(context.TODO(), dest, table, opts, columns...) } @@ -117,6 +119,8 @@ func (d *DBClient) SelectWithContext(ctx context.Context, dest interface{}, tabl return d.db.SelectContext(ctx, dest, query, args...) } +// Select is a convenience wrapper without context. +// Deprecated: Prefer SelectWithContext for proper timeout and cancellation support. func (d *DBClient) Select(dest interface{}, table string, opts models.ListOptions, orderColumn string, columns ...string) error { return d.SelectWithContext(context.TODO(), dest, table, opts, orderColumn, columns...) } @@ -192,6 +196,8 @@ func (d *DBClient) InsertWithContext(ctx context.Context, query string, data int return nil } +// Insert is a convenience wrapper without context. +// Deprecated: Prefer InsertWithContext for proper timeout and cancellation support. func (d *DBClient) Insert(query string, data interface{}) error { return d.InsertWithContext(context.TODO(), query, data) } @@ -224,6 +230,8 @@ func (d *DBClient) UpdateWithContext(ctx context.Context, query string, data int return nil } +// Update is a convenience wrapper without context. +// Deprecated: Prefer UpdateWithContext for proper timeout and cancellation support. func (d *DBClient) Update(query string, data interface{}) error { return d.UpdateWithContext(context.TODO(), query, data) } @@ -256,6 +264,8 @@ func (d *DBClient) NamedUpdateWithContext(ctx context.Context, query string, dat return nil } +// NamedUpdate is a convenience wrapper without context. +// Deprecated: Prefer NamedUpdateWithContext for proper timeout and cancellation support. func (d *DBClient) NamedUpdate(query string, data interface{}) error { return d.NamedUpdateWithContext(context.TODO(), query, data) } @@ -303,6 +313,8 @@ func (d *DBClient) DeleteWithContext(ctx context.Context, table string, opts mod return nil } +// Delete is a convenience wrapper without context. +// Deprecated: Prefer DeleteWithContext for proper timeout and cancellation support. func (d *DBClient) Delete(table string, opts models.ListOptions) error { return d.DeleteWithContext(context.TODO(), table, opts) } From f092a1ffcfc101ce3ca3c508fd056dc905cd7927 Mon Sep 17 00:00:00 2001 From: r2dedios Date: Tue, 17 Mar 2026 11:23:17 +0100 Subject: [PATCH 20/53] fix(dto): propagate conversion errors and add GoDoc comments Signed-off-by: r2dedios --- internal/api/handlers/account_handler.go | 11 ++++++++++- internal/api/handlers/cluster_handler.go | 11 ++++++++++- internal/api/handlers/instance_handler.go | 11 ++++++++++- internal/models/dto/account_dto.go | 24 +++++++++++++---------- internal/models/dto/account_dto_test.go | 11 +++++++---- internal/models/dto/cluster_dto.go | 23 +++++++++++++--------- internal/models/dto/cluster_dto_test.go | 11 +++++++---- internal/models/dto/expense_dto.go | 2 +- internal/models/dto/instance_dto.go | 23 +++++++++++++--------- internal/models/dto/instance_dto_test.go | 11 +++++++---- 10 files changed, 94 insertions(+), 44 deletions(-) diff --git a/internal/api/handlers/account_handler.go b/internal/api/handlers/account_handler.go index 23be32c5..0f559089 100644 --- a/internal/api/handlers/account_handler.go +++ b/internal/api/handlers/account_handler.go @@ -229,7 +229,16 @@ func (h *AccountHandler) Create(c *gin.Context) { return } - if err := h.service.Create(c.Request.Context(), *dto.ToInventoryAccountList(newAccountsDTO)); err != nil { + accounts, err := dto.ToInventoryAccountList(newAccountsDTO) + if err != nil { + h.logger.Error("error converting accounts DTOs", zap.Error(err)) + c.JSON(http.StatusBadRequest, responsetypes.GenericErrorResponse{ + Message: "Invalid account data: " + err.Error(), + }) + return + } + + if err := h.service.Create(c.Request.Context(), *accounts); err != nil { h.logger.Error("error creating accounts", zap.Error(err)) c.JSON(http.StatusInternalServerError, responsetypes.GenericErrorResponse{ Message: "Failed to create accounts: " + err.Error(), diff --git a/internal/api/handlers/cluster_handler.go b/internal/api/handlers/cluster_handler.go index 4e53b6a7..9c69765d 100644 --- a/internal/api/handlers/cluster_handler.go +++ b/internal/api/handlers/cluster_handler.go @@ -201,7 +201,16 @@ func (h *ClusterHandler) Create(c *gin.Context) { return } - if err := h.service.Create(c.Request.Context(), *dto.ToInventoryClusterList(newClusterDTOs)); err != nil { + clusters, err := dto.ToInventoryClusterList(newClusterDTOs) + if err != nil { + h.logger.Error("error converting cluster DTOs", zap.Error(err)) + c.JSON(http.StatusBadRequest, responsetypes.GenericErrorResponse{ + Message: "Invalid cluster data: " + err.Error(), + }) + return + } + + if err := h.service.Create(c.Request.Context(), *clusters); err != nil { h.logger.Error("error creating a cluster", zap.Error(err)) c.JSON(http.StatusInternalServerError, responsetypes.GenericErrorResponse{ Message: "Failed to create clusters: " + err.Error(), diff --git a/internal/api/handlers/instance_handler.go b/internal/api/handlers/instance_handler.go index 8192260f..c91c20fb 100644 --- a/internal/api/handlers/instance_handler.go +++ b/internal/api/handlers/instance_handler.go @@ -155,7 +155,16 @@ func (h *InstanceHandler) Create(c *gin.Context) { return } - if err := h.service.Create(c.Request.Context(), *dto.ToInventoryInstanceList(newInstanceDTOs)); err != nil { + instances, err := dto.ToInventoryInstanceList(newInstanceDTOs) + if err != nil { + h.logger.Error("error converting instance DTOs", zap.Error(err)) + c.JSON(http.StatusBadRequest, responsetypes.GenericErrorResponse{ + Message: "Invalid instance data: " + err.Error(), + }) + return + } + + if err := h.service.Create(c.Request.Context(), *instances); err != nil { h.logger.Error("error creating instances", zap.Error(err)) c.JSON(http.StatusInternalServerError, responsetypes.GenericErrorResponse{ Message: "Failed to create instances: " + err.Error(), diff --git a/internal/models/dto/account_dto.go b/internal/models/dto/account_dto.go index c6ef18ba..4af9dbdf 100644 --- a/internal/models/dto/account_dto.go +++ b/internal/models/dto/account_dto.go @@ -15,8 +15,9 @@ type AccountDTORequest struct { CreatedAt time.Time `json:"createdAt"` } // @name AccountRequest -// TODO: comments -func (a AccountDTORequest) ToInventoryAccount() *inventory.Account { +// ToInventoryAccount converts an AccountDTORequest to an inventory.Account. +// Returns an error if the account cannot be created. +func (a AccountDTORequest) ToInventoryAccount() (*inventory.Account, error) { account, err := inventory.NewAccount( a.AccountID, a.AccountName, @@ -25,21 +26,24 @@ func (a AccountDTORequest) ToInventoryAccount() *inventory.Account { "", ) if err != nil { - // TODO: Propagate error - return nil + return nil, err } account.LastScanTimestamp = a.LastScanTimestamp - return account + return account, nil } -func ToInventoryAccountList(dtos []AccountDTORequest) *[]inventory.Account { - accounts := make([]inventory.Account, len(dtos)) - for i, dto := range dtos { - accounts[i] = *dto.ToInventoryAccount() +func ToInventoryAccountList(dtos []AccountDTORequest) (*[]inventory.Account, error) { + accounts := make([]inventory.Account, 0, len(dtos)) + for _, dto := range dtos { + account, err := dto.ToInventoryAccount() + if err != nil { + return nil, err + } + accounts = append(accounts, *account) } - return &accounts + return &accounts, nil } func ToAccountDTORequest(account inventory.Account) *AccountDTORequest { diff --git a/internal/models/dto/account_dto_test.go b/internal/models/dto/account_dto_test.go index 378d36ac..95ab30e4 100644 --- a/internal/models/dto/account_dto_test.go +++ b/internal/models/dto/account_dto_test.go @@ -11,7 +11,7 @@ import ( // TestAccountDTORequest_ToInventoryAccount verifies DTO to inventory.Account conversion. func TestAccountDTORequest_ToInventoryAccount(t *testing.T) { t.Run("Valid DTO", func(t *testing.T) { testAccountDTORequest_ToInventoryAccount_Correct(t) }) - t.Run("Invalid DTO returns nil", func(t *testing.T) { testAccountDTORequest_ToInventoryAccount_Invalid(t) }) + t.Run("Invalid DTO returns error", func(t *testing.T) { testAccountDTORequest_ToInventoryAccount_Invalid(t) }) } func testAccountDTORequest_ToInventoryAccount_Correct(t *testing.T) { @@ -25,8 +25,9 @@ func testAccountDTORequest_ToInventoryAccount_Correct(t *testing.T) { CreatedAt: now.Add(-time.Hour), } - account := dto.ToInventoryAccount() + account, err := dto.ToInventoryAccount() + assert.NoError(t, err) assert.NotNil(t, account) assert.Equal(t, dto.AccountID, account.AccountID) assert.Equal(t, dto.AccountName, account.AccountName) @@ -41,7 +42,8 @@ func testAccountDTORequest_ToInventoryAccount_Invalid(t *testing.T) { Provider: inventory.AWSProvider, } - account := dto.ToInventoryAccount() + account, err := dto.ToInventoryAccount() + assert.Error(t, err) assert.Nil(t, account) } @@ -68,8 +70,9 @@ func testToInventoryAccountList_Correct(t *testing.T) { }, } - accounts := ToInventoryAccountList(dtos) + accounts, err := ToInventoryAccountList(dtos) + assert.NoError(t, err) assert.NotNil(t, accounts) assert.Len(t, *accounts, 2) diff --git a/internal/models/dto/cluster_dto.go b/internal/models/dto/cluster_dto.go index c69d3bb0..24719819 100644 --- a/internal/models/dto/cluster_dto.go +++ b/internal/models/dto/cluster_dto.go @@ -22,7 +22,9 @@ type ClusterDTORequest struct { Owner string `json:"owner"` } // @name ClusterRequest -func (c ClusterDTORequest) ToInventoryCluster() *inventory.Cluster { +// ToInventoryCluster converts a ClusterDTORequest to an inventory.Cluster. +// Returns an error if the cluster cannot be created. +func (c ClusterDTORequest) ToInventoryCluster() (*inventory.Cluster, error) { cluster, err := inventory.NewCluster( c.ClusterName, c.InfraID, @@ -32,8 +34,7 @@ func (c ClusterDTORequest) ToInventoryCluster() *inventory.Cluster { c.Owner, ) if err != nil { - // TODO: Propagate error - return nil + return nil, err } cluster.LastScanTimestamp = c.LastScanTimestamp @@ -42,16 +43,20 @@ func (c ClusterDTORequest) ToInventoryCluster() *inventory.Cluster { cluster.AccountID = c.AccountID cluster.Age = c.Age - return cluster + return cluster, nil } -func ToInventoryClusterList(dtos []ClusterDTORequest) *[]inventory.Cluster { - clusters := make([]inventory.Cluster, len(dtos)) - for i, dto := range dtos { - clusters[i] = *dto.ToInventoryCluster() +func ToInventoryClusterList(dtos []ClusterDTORequest) (*[]inventory.Cluster, error) { + clusters := make([]inventory.Cluster, 0, len(dtos)) + for _, dto := range dtos { + cluster, err := dto.ToInventoryCluster() + if err != nil { + return nil, err + } + clusters = append(clusters, *cluster) } - return &clusters + return &clusters, nil } func ToClusterDTORequest(cluster inventory.Cluster) *ClusterDTORequest { diff --git a/internal/models/dto/cluster_dto_test.go b/internal/models/dto/cluster_dto_test.go index c5566600..a526a7ff 100644 --- a/internal/models/dto/cluster_dto_test.go +++ b/internal/models/dto/cluster_dto_test.go @@ -11,7 +11,7 @@ import ( // TestClusterDTORequest_ToInventoryCluster verifies DTO to inventory.Cluster conversion. func TestClusterDTORequest_ToInventoryCluster(t *testing.T) { t.Run("Valid DTO", func(t *testing.T) { testClusterDTORequest_ToInventoryCluster_Correct(t) }) - t.Run("Invalid DTO returns nil", func(t *testing.T) { testClusterDTORequest_ToInventoryCluster_Invalid(t) }) + t.Run("Invalid DTO returns error", func(t *testing.T) { testClusterDTORequest_ToInventoryCluster_Invalid(t) }) } func testClusterDTORequest_ToInventoryCluster_Correct(t *testing.T) { @@ -32,8 +32,9 @@ func testClusterDTORequest_ToInventoryCluster_Correct(t *testing.T) { Owner: "owner", } - cluster := dto.ToInventoryCluster() + cluster, err := dto.ToInventoryCluster() + assert.NoError(t, err) assert.NotNil(t, cluster) assert.Equal(t, dto.ClusterName, cluster.ClusterName) @@ -61,7 +62,8 @@ func testClusterDTORequest_ToInventoryCluster_Invalid(t *testing.T) { Owner: "owner", } - cluster := dto.ToInventoryCluster() + cluster, err := dto.ToInventoryCluster() + assert.Error(t, err) assert.Nil(t, cluster) } @@ -100,8 +102,9 @@ func testToInventoryClusterList_Correct(t *testing.T) { }, } - clusters := ToInventoryClusterList(dtos) + clusters, err := ToInventoryClusterList(dtos) + assert.NoError(t, err) assert.NotNil(t, clusters) assert.Len(t, *clusters, 2) diff --git a/internal/models/dto/expense_dto.go b/internal/models/dto/expense_dto.go index 86ce06e9..b9dfc064 100644 --- a/internal/models/dto/expense_dto.go +++ b/internal/models/dto/expense_dto.go @@ -13,7 +13,7 @@ type ExpenseDTORequest struct { Date time.Time `json:"date"` } // @name ExpenseRequest -// TODO: comments +// ToInventoryExpense converts an ExpenseDTORequest to an inventory.Expense. func (e ExpenseDTORequest) ToInventoryExpense() *inventory.Expense { return inventory.NewExpense( e.InstanceID, diff --git a/internal/models/dto/instance_dto.go b/internal/models/dto/instance_dto.go index 7e1a3227..f9ab0092 100644 --- a/internal/models/dto/instance_dto.go +++ b/internal/models/dto/instance_dto.go @@ -22,7 +22,9 @@ type InstanceDTORequest struct { Tags []TagDTORequest `json:"tags"` } // @name InstanceRequest -func (i InstanceDTORequest) ToInventoryInstance() *inventory.Instance { +// ToInventoryInstance converts an InstanceDTORequest to an inventory.Instance. +// Returns an error if the instance cannot be created. +func (i InstanceDTORequest) ToInventoryInstance() (*inventory.Instance, error) { instance, err := inventory.NewInstance( i.InstanceID, i.InstanceName, @@ -34,23 +36,26 @@ func (i InstanceDTORequest) ToInventoryInstance() *inventory.Instance { i.CreatedAt, ) if err != nil { - // TODO: Propagate error - return nil + return nil, err } instance.LastScanTimestamp = i.LastScanTimestamp instance.ClusterID = i.ClusterID - return instance + return instance, nil } -func ToInventoryInstanceList(dtos []InstanceDTORequest) *[]inventory.Instance { - instances := make([]inventory.Instance, len(dtos)) - for i, dto := range dtos { - instances[i] = *dto.ToInventoryInstance() +func ToInventoryInstanceList(dtos []InstanceDTORequest) (*[]inventory.Instance, error) { + instances := make([]inventory.Instance, 0, len(dtos)) + for _, dto := range dtos { + instance, err := dto.ToInventoryInstance() + if err != nil { + return nil, err + } + instances = append(instances, *instance) } - return &instances + return &instances, nil } func ToInstanceDTORequest(instance inventory.Instance) *InstanceDTORequest { diff --git a/internal/models/dto/instance_dto_test.go b/internal/models/dto/instance_dto_test.go index 0791d0d4..539b0a98 100644 --- a/internal/models/dto/instance_dto_test.go +++ b/internal/models/dto/instance_dto_test.go @@ -11,7 +11,7 @@ import ( // TestInstanceDTORequest_ToInventoryInstance verifies DTO to inventory.Instance conversion. func TestInstanceDTORequest_ToInventoryInstance(t *testing.T) { t.Run("Valid DTO", func(t *testing.T) { testInstanceDTORequest_ToInventoryInstance_Correct(t) }) - t.Run("Invalid DTO returns nil", func(t *testing.T) { testInstanceDTORequest_ToInventoryInstance_Invalid(t) }) + t.Run("Invalid DTO returns error", func(t *testing.T) { testInstanceDTORequest_ToInventoryInstance_Invalid(t) }) } func testInstanceDTORequest_ToInventoryInstance_Correct(t *testing.T) { @@ -37,8 +37,9 @@ func testInstanceDTORequest_ToInventoryInstance_Correct(t *testing.T) { }, } - instance := dto.ToInventoryInstance() + instance, err := dto.ToInventoryInstance() + assert.NoError(t, err) assert.NotNil(t, instance) assert.Equal(t, dto.InstanceID, instance.InstanceID) @@ -71,7 +72,8 @@ func testInstanceDTORequest_ToInventoryInstance_Invalid(t *testing.T) { Tags: []TagDTORequest{}, } - instance := dto.ToInventoryInstance() + instance, err := dto.ToInventoryInstance() + assert.Error(t, err) assert.Nil(t, instance) } @@ -110,8 +112,9 @@ func testToInventoryInstanceList_Correct(t *testing.T) { }, } - instances := ToInventoryInstanceList(dtos) + instances, err := ToInventoryInstanceList(dtos) + assert.NoError(t, err) assert.NotNil(t, instances) assert.Len(t, *instances, 2) assert.Equal(t, "i-1", (*instances)[0].InstanceID) From ef4915a177bdd0afd4e35d92309c63b72cb8f171 Mon Sep 17 00:00:00 2001 From: r2dedios Date: Tue, 17 Mar 2026 11:30:31 +0100 Subject: [PATCH 21/53] fix(aws): continue processing instances on conversion error instead of skipping silently Signed-off-by: r2dedios --- internal/cloud_providers/aws/aws-ec2.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/internal/cloud_providers/aws/aws-ec2.go b/internal/cloud_providers/aws/aws-ec2.go index 977c18f9..dde8e6c7 100644 --- a/internal/cloud_providers/aws/aws-ec2.go +++ b/internal/cloud_providers/aws/aws-ec2.go @@ -206,11 +206,12 @@ func (c *AWSEC2Connection) GetInstances() ([]inventory.Instance, error) { var instances []inventory.Instance for _, reser := range reservations { for _, instance := range reser.Instances { - // TODO: Should the loop break in case or error, or continue? newInstance, err := EC2InstanceToInventoryInstance(instance) - if err == nil { - instances = append(instances, *newInstance) + if err != nil { + // Skip instance on conversion error and continue processing remaining instances + continue } + instances = append(instances, *newInstance) } } From 30ff42606fa7b9240ed2e10a855f50b32cdb6672 Mon Sep 17 00:00:00 2001 From: r2dedios Date: Fri, 20 Mar 2026 14:08:28 +0100 Subject: [PATCH 22/53] feat(api): implement PATCH endpoint for accounts --- internal/api/handlers/account_handler.go | 50 +++++++++-- internal/models/dto/account_dto.go | 6 ++ internal/repositories/account_repository.go | 66 ++++++++++++++ internal/services/account_service.go | 13 +++ .../api_accounts_integration_test.go | 86 ++++++++++++++++--- 5 files changed, 203 insertions(+), 18 deletions(-) diff --git a/internal/api/handlers/account_handler.go b/internal/api/handlers/account_handler.go index 0f559089..23aba5c0 100644 --- a/internal/api/handlers/account_handler.go +++ b/internal/api/handlers/account_handler.go @@ -288,16 +288,54 @@ func (h *AccountHandler) Delete(c *gin.Context) { // Update applies partial updates to an existing account. // // @Summary Update an account -// @Description Patch an existing account by ID. +// @Description Patch mutable fields of an account (accountName). // @Tags Accounts // @Accept json // @Produce json // @Param id path string true "Account ID" -// @Param account body dto.AccountDTORequest true "Partial account payload" -// @Success 200 {object} nil -// @Failure 501 {object} nil "Not Implemented" +// @Param account body dto.AccountPatchRequest true "Partial account payload" +// @Success 200 {object} db.AccountDBResponse +// @Failure 400 {object} responsetypes.GenericErrorResponse +// @Failure 404 {object} responsetypes.GenericErrorResponse +// @Failure 500 {object} responsetypes.GenericErrorResponse // @Router /accounts/{id} [patch] func (h *AccountHandler) Update(c *gin.Context) { - // TODO: Implement partial update semantics - c.PureJSON(http.StatusNotImplemented, nil) + accountID := c.Param("id") + + var patchRequest dto.AccountPatchRequest + if err := c.ShouldBindJSON(&patchRequest); err != nil { + h.logger.Error("Bad request for account update - invalid JSON body", + zap.String("account_id", accountID), + zap.Error(err)) + c.JSON(http.StatusBadRequest, responsetypes.GenericErrorResponse{ + Message: "Invalid request body: " + err.Error(), + }) + return + } + + if err := h.service.Update(c.Request.Context(), accountID, patchRequest); err != nil { + h.logger.Error("error updating account", zap.String("account_id", accountID), zap.Error(err)) + if errors.Is(err, repositories.ErrNotFound) { + c.JSON(http.StatusNotFound, responsetypes.GenericErrorResponse{ + Message: "Account not found", + }) + return + } + c.JSON(http.StatusInternalServerError, responsetypes.GenericErrorResponse{ + Message: "Failed to update account: " + err.Error(), + }) + return + } + + // Fetch updated account to return + updatedAccount, err := h.service.GetByID(c.Request.Context(), accountID) + if err != nil { + h.logger.Error("error fetching updated account", zap.String("account_id", accountID), zap.Error(err)) + c.JSON(http.StatusInternalServerError, responsetypes.GenericErrorResponse{ + Message: "Account updated but failed to fetch result", + }) + return + } + + c.JSON(http.StatusOK, (&convert.ConverterImpl{}).ToAccountDTO(updatedAccount)) } diff --git a/internal/models/dto/account_dto.go b/internal/models/dto/account_dto.go index 4af9dbdf..9d254b97 100644 --- a/internal/models/dto/account_dto.go +++ b/internal/models/dto/account_dto.go @@ -69,3 +69,9 @@ type AccountDTOResponse struct { LastMonthCost float64 `json:"lastMonthCost"` CurrentMonthSoFarCost float64 `json:"currentMonthSoFarCost"` } // @name AccountResponse + +// AccountPatchRequest represents mutable fields for partial account updates. +// Only fields present in the request will be updated (using pointers to distinguish null from empty). +type AccountPatchRequest struct { + AccountName *string `json:"accountName,omitempty"` +} // @name AccountPatchRequest diff --git a/internal/repositories/account_repository.go b/internal/repositories/account_repository.go index b013cf4a..5e4c283c 100644 --- a/internal/repositories/account_repository.go +++ b/internal/repositories/account_repository.go @@ -11,6 +11,7 @@ import ( "github.com/RHEcosystemAppEng/cluster-iq/internal/inventory" "github.com/RHEcosystemAppEng/cluster-iq/internal/models" "github.com/RHEcosystemAppEng/cluster-iq/internal/models/db" + "github.com/RHEcosystemAppEng/cluster-iq/internal/models/dto" ) const ( @@ -52,6 +53,7 @@ type AccountRepository interface { GetExpenseUpdateInstances(ctx context.Context, accountID string) ([]db.InstanceDBResponse, error) GetScannerTimestamp(ctx context.Context) (time.Time, error) CreateAccount(ctx context.Context, accounts []inventory.Account) error + UpdateAccount(ctx context.Context, accountID string, patch dto.AccountPatchRequest) error DeleteAccount(ctx context.Context, accountID string) error } @@ -189,6 +191,70 @@ func (r *accountRepositoryImpl) CreateAccount(ctx context.Context, accounts []in return nil } +// UpdateAccount updates mutable fields of an existing account in the database. +// Only non-nil fields in the patch request will be updated. +func (r *accountRepositoryImpl) UpdateAccount(ctx context.Context, accountID string, patch dto.AccountPatchRequest) error { + // Build dynamic UPDATE query with positional parameters + query := "UPDATE accounts SET " + args := make([]interface{}, 0) + argCount := 1 + + if patch.AccountName != nil { + query += fmt.Sprintf("account_name = $%d", argCount) + args = append(args, *patch.AccountName) + argCount++ + } + + // If no fields to update, return early + if len(args) == 0 { + return nil + } + + query += fmt.Sprintf(" WHERE account_id = $%d", argCount) + args = append(args, accountID) + + // Execute update in a transaction + tx, err := r.db.NewTx(ctx) + if err != nil { + return err + } + defer func() { + if err != nil { + _ = tx.Rollback() + } + }() + + if _, err = tx.ExecContext(ctx, query, args...); err != nil { + return fmt.Errorf("exec UPDATE error: %w", err) + } + + if err = tx.Commit(); err != nil { + return fmt.Errorf("commit UPDATE error: %w", err) + } + + // Refresh materialized view after transaction commits + // Note: REFRESH MATERIALIZED VIEW must run outside the transaction + tx2, err := r.db.NewTx(ctx) + if err != nil { + return fmt.Errorf("failed to create transaction for refresh: %w", err) + } + defer func() { + if err != nil { + _ = tx2.Rollback() + } + }() + + if _, err = tx2.ExecContext(ctx, "REFRESH MATERIALIZED VIEW m_accounts_full_view"); err != nil { + return fmt.Errorf("refresh materialized view error: %w", err) + } + + if err = tx2.Commit(); err != nil { + return fmt.Errorf("commit refresh error: %w", err) + } + + return nil +} + // DeleteAccount deletes an account from the database by its ID. // // Parameters: diff --git a/internal/services/account_service.go b/internal/services/account_service.go index e20f2486..cec66588 100644 --- a/internal/services/account_service.go +++ b/internal/services/account_service.go @@ -6,6 +6,7 @@ import ( "github.com/RHEcosystemAppEng/cluster-iq/internal/inventory" "github.com/RHEcosystemAppEng/cluster-iq/internal/models" "github.com/RHEcosystemAppEng/cluster-iq/internal/models/db" + "github.com/RHEcosystemAppEng/cluster-iq/internal/models/dto" "github.com/RHEcosystemAppEng/cluster-iq/internal/repositories" ) @@ -16,6 +17,7 @@ type AccountService interface { GetAccountClustersByID(ctx context.Context, accountID string) ([]db.ClusterDBResponse, error) GetExpenseUpdateInstances(ctx context.Context, accountID string) ([]db.InstanceDBResponse, error) Create(ctx context.Context, accounts []inventory.Account) error + Update(ctx context.Context, accountID string, patch dto.AccountPatchRequest) error Delete(ctx context.Context, accountID string) error } @@ -61,6 +63,17 @@ func (s *accountServiceImpl) Create(ctx context.Context, accounts []inventory.Ac return s.repo.CreateAccount(ctx, accounts) } +// Update updates mutable fields of an existing account. +func (s *accountServiceImpl) Update(ctx context.Context, accountID string, patch dto.AccountPatchRequest) error { + // Verify account exists before updating + _, err := s.repo.GetAccountByID(ctx, accountID) + if err != nil { + return err + } + + return s.repo.UpdateAccount(ctx, accountID, patch) +} + // Delete removes an account by its name. func (s *accountServiceImpl) Delete(ctx context.Context, accountID string) error { return s.repo.DeleteAccount(ctx, accountID) diff --git a/test/integration/api_accounts_integration_test.go b/test/integration/api_accounts_integration_test.go index 180bb327..0aa7449c 100644 --- a/test/integration/api_accounts_integration_test.go +++ b/test/integration/api_accounts_integration_test.go @@ -8,8 +8,6 @@ import ( "time" responsetypes "github.com/RHEcosystemAppEng/cluster-iq/internal/api/response_types" - - "github.com/RHEcosystemAppEng/cluster-iq/internal/inventory" "github.com/RHEcosystemAppEng/cluster-iq/internal/models/dto" ) @@ -34,7 +32,8 @@ func TestAPIAccounts(t *testing.T) { t.Run("Test Post One Account", func(t *testing.T) { testPostOneAccount(t) }) t.Run("Test Post Multiple Accounts", func(t *testing.T) { testPostMultipleAccounts(t) }) t.Run("Test Post Wrong Accounts", func(t *testing.T) { testPostWrongAccount(t) }) - t.Run("Test Patch Account", func(t *testing.T) { testPatchAccount(t) }) + t.Run("Test Patch Account Success", func(t *testing.T) { testPatchAccount_Success(t) }) + t.Run("Test Patch Account Not Found", func(t *testing.T) { testPatchAccount_NotFound(t) }) t.Run("Test Delete Account Success", func(t *testing.T) { testDeleteAccount_Exists(t) }) t.Run("Test Delete Account Not Found", func(t *testing.T) { testDeleteAccount_NoExists(t) }) } @@ -361,26 +360,28 @@ func testPostWrongAccount(t *testing.T) { } } -func testPatchAccount(t *testing.T) { - expectedHTTPCode := http.StatusNotImplemented +func testPatchAccount_Success(t *testing.T) { + expectedHTTPCode := http.StatusOK + expectedAccountID := "gcp-project-1" + expectedAccountName := "updated-gcp-project-name" - patchAccount := dto.AccountDTORequest{ - AccountID: "ACC-001", - AccountName: "test-account-003", - Provider: inventory.AWSProvider, - LastScanTimestamp: time.Now(), + // Create patch request with only the fields to update + newAccountName := expectedAccountName + patchAccount := dto.AccountPatchRequest{ + AccountName: &newAccountName, } patchBody, err := json.Marshal(patchAccount) if err != nil { - t.Fatalf("Failed to marshal updated account: %v", err) + t.Fatalf("Failed to marshal patch request: %v", err) } // Preparing PATCH request - req, err := http.NewRequest(http.MethodPatch, APIAccountsURL+"/ACC-003", bytes.NewBuffer(patchBody)) + req, err := http.NewRequest(http.MethodPatch, APIAccountsURL+"/"+expectedAccountID, bytes.NewBuffer(patchBody)) if err != nil { t.Fatalf("Failed to create request: %v", err) } + req.Header.Set("Content-Type", "application/json") // Executing PATCH request client := &http.Client{} @@ -392,6 +393,67 @@ func testPatchAccount(t *testing.T) { // Check response code checkHTTPResponseCode(t, resp, expectedHTTPCode) + + // Decode the JSON response + var response dto.AccountDTOResponse + if err := json.NewDecoder(resp.Body).Decode(&response); err != nil { + t.Fatalf("Failed to decode response body: %v", err) + } + + // Verify the account was updated + if response.AccountID != expectedAccountID { + t.Fatalf("Expected AccountID: '%s', got: '%s'", expectedAccountID, response.AccountID) + } + + if response.AccountName != expectedAccountName { + t.Fatalf("Expected AccountName: '%s', got: '%s'", expectedAccountName, response.AccountName) + } +} + +func testPatchAccount_NotFound(t *testing.T) { + expectedHTTPCode := http.StatusNotFound + expectedMsg := "Account not found" + nonExistentAccountID := "non-existent-account" + + // Create patch request + newAccountName := "should-not-be-applied" + patchAccount := dto.AccountPatchRequest{ + AccountName: &newAccountName, + } + + patchBody, err := json.Marshal(patchAccount) + if err != nil { + t.Fatalf("Failed to marshal patch request: %v", err) + } + + // Preparing PATCH request + req, err := http.NewRequest(http.MethodPatch, APIAccountsURL+"/"+nonExistentAccountID, bytes.NewBuffer(patchBody)) + if err != nil { + t.Fatalf("Failed to create request: %v", err) + } + req.Header.Set("Content-Type", "application/json") + + // Executing PATCH request + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + t.Fatalf("Failed to execute request: %v", err) + } + defer resp.Body.Close() + + // Check response code + checkHTTPResponseCode(t, resp, expectedHTTPCode) + + // Decode the JSON response + var response responsetypes.GenericErrorResponse + if err := json.NewDecoder(resp.Body).Decode(&response); err != nil { + t.Fatalf("Failed to decode response body: %v", err) + } + + // Verify error message + if response.Message != expectedMsg { + t.Fatalf("Expected Message: '%s', got: '%s'", expectedMsg, response.Message) + } } func testDeleteAccount_Exists(t *testing.T) { From c81eec2987a0345ab7d4ab138aca23e10c7aafa6 Mon Sep 17 00:00:00 2001 From: r2dedios Date: Fri, 20 Mar 2026 14:08:31 +0100 Subject: [PATCH 23/53] feat(api): implement PATCH endpoint for clusters --- internal/api/handlers/cluster_handler.go | 50 ++++++++-- internal/models/dto/cluster_dto.go | 7 ++ internal/repositories/cluster_repository.go | 75 ++++++++++++++- internal/services/cluster_service.go | 14 ++- .../api_clusters_integration_test.go | 92 ++++++++++++++++--- 5 files changed, 213 insertions(+), 25 deletions(-) diff --git a/internal/api/handlers/cluster_handler.go b/internal/api/handlers/cluster_handler.go index 9c69765d..e9b1cd38 100644 --- a/internal/api/handlers/cluster_handler.go +++ b/internal/api/handlers/cluster_handler.go @@ -379,16 +379,54 @@ func (h *ClusterHandler) GetTags(c *gin.Context) { // Update applies partial updates to a cluster. // // @Summary Update a cluster -// @Description Patch mutable fields of a cluster. +// @Description Patch mutable fields of a cluster (consoleLink, owner). // @Tags Clusters // @Accept json // @Produce json // @Param id path string true "Cluster ID" -// @Param cluster body dto.ClusterDTOResponse true "Partial cluster payload" -// @Success 200 {object} nil -// @Failure 501 {object} nil "Not Implemented" +// @Param cluster body dto.ClusterPatchRequest true "Partial cluster payload" +// @Success 200 {object} dto.ClusterDTOResponse +// @Failure 400 {object} responsetypes.GenericErrorResponse +// @Failure 404 {object} responsetypes.GenericErrorResponse +// @Failure 500 {object} responsetypes.GenericErrorResponse // @Router /clusters/{id} [patch] func (h *ClusterHandler) Update(c *gin.Context) { - // TODO: Implement partial update strategy - c.PureJSON(http.StatusNotImplemented, nil) + clusterID := c.Param("id") + + var patchRequest dto.ClusterPatchRequest + if err := c.ShouldBindJSON(&patchRequest); err != nil { + h.logger.Error("Bad request for cluster update - invalid JSON body", + zap.String("cluster_id", clusterID), + zap.Error(err)) + c.JSON(http.StatusBadRequest, responsetypes.GenericErrorResponse{ + Message: "Invalid request body: " + err.Error(), + }) + return + } + + if err := h.service.Update(c.Request.Context(), clusterID, patchRequest); err != nil { + h.logger.Error("error updating cluster", zap.String("cluster_id", clusterID), zap.Error(err)) + if errors.Is(err, repositories.ErrNotFound) { + c.JSON(http.StatusNotFound, responsetypes.GenericErrorResponse{ + Message: "Cluster not found", + }) + return + } + c.JSON(http.StatusInternalServerError, responsetypes.GenericErrorResponse{ + Message: "Failed to update cluster: " + err.Error(), + }) + return + } + + // Fetch updated cluster to return + updatedCluster, err := h.service.Get(c.Request.Context(), clusterID) + if err != nil { + h.logger.Error("error fetching updated cluster", zap.String("cluster_id", clusterID), zap.Error(err)) + c.JSON(http.StatusInternalServerError, responsetypes.GenericErrorResponse{ + Message: "Cluster updated but failed to fetch result", + }) + return + } + + c.JSON(http.StatusOK, (&convert.ConverterImpl{}).ToClusterDTO(*updatedCluster)) } diff --git a/internal/models/dto/cluster_dto.go b/internal/models/dto/cluster_dto.go index 24719819..0e0a9c15 100644 --- a/internal/models/dto/cluster_dto.go +++ b/internal/models/dto/cluster_dto.go @@ -107,3 +107,10 @@ type ClusterDTOResponse struct { LastMonthCost float64 `json:"lastMonthCost"` CurrentMonthSoFarCost float64 `json:"currentMonthSoFarCost"` } // @name ClusterResponse + +// ClusterPatchRequest represents mutable fields for partial cluster updates. +// Only fields present in the request will be updated (using pointers to distinguish null from empty). +type ClusterPatchRequest struct { + ConsoleLink *string `json:"consoleLink,omitempty"` + Owner *string `json:"owner,omitempty"` +} // @name ClusterPatchRequest diff --git a/internal/repositories/cluster_repository.go b/internal/repositories/cluster_repository.go index 0673f709..1f864457 100644 --- a/internal/repositories/cluster_repository.go +++ b/internal/repositories/cluster_repository.go @@ -5,6 +5,7 @@ import ( "database/sql" "errors" "fmt" + "strings" dbclient "github.com/RHEcosystemAppEng/cluster-iq/internal/db_client" "github.com/RHEcosystemAppEng/cluster-iq/internal/inventory" @@ -74,7 +75,7 @@ type ClusterRepository interface { GetInstancesOnCluster(ctx context.Context, clusterID string) ([]db.InstanceDBResponse, error) GetClustersOverview(ctx context.Context) (inventory.ClustersSummary, error) CreateClusters(ctx context.Context, clusters []inventory.Cluster) error - UpdateCluster(ctx context.Context, cluster dto.ClusterDTORequest) error + UpdateCluster(ctx context.Context, clusterID string, patch dto.ClusterPatchRequest) error UpdateClusterStatusByClusterID(ctx context.Context, status string, clusterID string) error DeleteCluster(ctx context.Context, id string) error } @@ -294,9 +295,75 @@ func (r *clusterRepositoryImpl) CreateClusters(ctx context.Context, clusters []i return nil } -// UpdateCluster updates an existing cluster's details in the database. -func (r *clusterRepositoryImpl) UpdateCluster(_ context.Context, _ dto.ClusterDTORequest) error { - // TODO +// UpdateCluster updates mutable fields of an existing cluster in the database. +// Only non-nil fields in the patch request will be updated. +func (r *clusterRepositoryImpl) UpdateCluster(ctx context.Context, clusterID string, patch dto.ClusterPatchRequest) error { + // Build dynamic UPDATE query with positional parameters + query := "UPDATE clusters SET " + args := make([]interface{}, 0) + updateFields := make([]string, 0) + argCount := 1 + + if patch.ConsoleLink != nil { + updateFields = append(updateFields, fmt.Sprintf("console_link = $%d", argCount)) + args = append(args, *patch.ConsoleLink) + argCount++ + } + + if patch.Owner != nil { + updateFields = append(updateFields, fmt.Sprintf("owner = $%d", argCount)) + args = append(args, *patch.Owner) + argCount++ + } + + // If no fields to update, return early + if len(updateFields) == 0 { + return nil + } + + query += strings.Join(updateFields, ", ") + query += fmt.Sprintf(" WHERE cluster_id = $%d", argCount) + args = append(args, clusterID) + + // Execute update in a transaction + tx, err := r.db.NewTx(ctx) + if err != nil { + return err + } + defer func() { + if err != nil { + _ = tx.Rollback() + } + }() + + if _, err = tx.ExecContext(ctx, query, args...); err != nil { + return fmt.Errorf("exec UPDATE error: %w", err) + } + + if err = tx.Commit(); err != nil { + return fmt.Errorf("commit UPDATE error: %w", err) + } + + // Refresh materialized view after transaction commits + // Note: REFRESH MATERIALIZED VIEW must run outside the transaction + tx2, err := r.db.NewTx(ctx) + if err != nil { + return fmt.Errorf("failed to create transaction for refresh: %w", err) + } + defer func() { + if err != nil { + _ = tx2.Rollback() + } + }() + + if _, err = tx2.ExecContext(ctx, "REFRESH MATERIALIZED VIEW m_clusters_full_view"); err != nil { + return fmt.Errorf("refresh materialized view error: %w", err) + } + + if err = tx2.Commit(); err != nil { + return fmt.Errorf("commit refresh error: %w", err) + } + return nil } diff --git a/internal/services/cluster_service.go b/internal/services/cluster_service.go index 04609e4e..ec7e0ec1 100644 --- a/internal/services/cluster_service.go +++ b/internal/services/cluster_service.go @@ -24,7 +24,7 @@ type ClusterService interface { Create(ctx context.Context, clusters []inventory.Cluster) error Delete(ctx context.Context, clusterID string) error GetTags(ctx context.Context, clusterID string) ([]db.TagDBResponse, error) - Update(ctx context.Context, cluster dto.ClusterDTORequest) error + Update(ctx context.Context, clusterID string, patch dto.ClusterPatchRequest) error } var _ ClusterService = (*clusterServiceImpl)(nil) @@ -142,7 +142,13 @@ func (s *clusterServiceImpl) GetTags(ctx context.Context, clusterID string) ([]d return s.repo.GetClusterTags(ctx, clusterID) } -// Update updates an existing cluster. -func (s *clusterServiceImpl) Update(ctx context.Context, cluster dto.ClusterDTORequest) error { - return s.repo.UpdateCluster(ctx, cluster) +// Update updates mutable fields of an existing cluster. +func (s *clusterServiceImpl) Update(ctx context.Context, clusterID string, patch dto.ClusterPatchRequest) error { + // Verify cluster exists before updating + _, err := s.repo.GetClusterByID(ctx, clusterID) + if err != nil { + return err + } + + return s.repo.UpdateCluster(ctx, clusterID, patch) } diff --git a/test/integration/api_clusters_integration_test.go b/test/integration/api_clusters_integration_test.go index 646d7244..c87b8d59 100644 --- a/test/integration/api_clusters_integration_test.go +++ b/test/integration/api_clusters_integration_test.go @@ -8,7 +8,6 @@ import ( "time" responsetypes "github.com/RHEcosystemAppEng/cluster-iq/internal/api/response_types" - "github.com/RHEcosystemAppEng/cluster-iq/internal/inventory" "github.com/RHEcosystemAppEng/cluster-iq/internal/models/dto" ) @@ -38,7 +37,8 @@ func TestClusters(t *testing.T) { t.Run("Test Post One Cluster", func(t *testing.T) { testPostOneCluster(t) }) t.Run("Test Post Multiple Clusters", func(t *testing.T) { testPostMultipleClusters(t) }) t.Run("Test Post Wrong Cluster", func(t *testing.T) { testPostWrongCluster(t) }) - t.Run("Test Patch Cluster", func(t *testing.T) { testPatchCluster(t) }) + t.Run("Test Patch Cluster Success", func(t *testing.T) { testPatchCluster_Success(t) }) + t.Run("Test Patch Cluster Not Found", func(t *testing.T) { testPatchCluster_NotFound(t) }) t.Run("Test Delete Cluster Success", func(t *testing.T) { testDeleteCluster_Exists(t) }) t.Run("Test Delete Cluster Not Found", func(t *testing.T) { testDeleteCluster_NoExists(t) }) } @@ -513,26 +513,31 @@ func testPostWrongCluster(t *testing.T) { } } -func testPatchCluster(t *testing.T) { - expectedHTTPCode := http.StatusNotImplemented +func testPatchCluster_Success(t *testing.T) { + expectedHTTPCode := http.StatusOK + expectedClusterID := "aws-cluster-1-aws-infra-1" + expectedConsoleLink := "https://updated-console.example.com" + expectedOwner := "Updated Owner Name" - patchCluster := dto.ClusterDTORequest{ - ClusterID: "test-cluster-infra-2345", - ClusterName: "test-Cluster-003", - Provider: inventory.AWSProvider, - LastScanTimestamp: time.Now(), + // Create patch request with only the fields to update + newConsoleLink := expectedConsoleLink + newOwner := expectedOwner + patchCluster := dto.ClusterPatchRequest{ + ConsoleLink: &newConsoleLink, + Owner: &newOwner, } patchBody, err := json.Marshal(patchCluster) if err != nil { - t.Fatalf("Failed to marshal updated Cluster: %v", err) + t.Fatalf("Failed to marshal patch request: %v", err) } // Preparing PATCH request - req, err := http.NewRequest(http.MethodPatch, APIClustersURL+"/aws-cluster-1", bytes.NewBuffer(patchBody)) + req, err := http.NewRequest(http.MethodPatch, APIClustersURL+"/"+expectedClusterID, bytes.NewBuffer(patchBody)) if err != nil { t.Fatalf("Failed to create request: %v", err) } + req.Header.Set("Content-Type", "application/json") // Executing PATCH request client := &http.Client{} @@ -544,6 +549,71 @@ func testPatchCluster(t *testing.T) { // Check response code checkHTTPResponseCode(t, resp, expectedHTTPCode) + + // Decode the JSON response + var response dto.ClusterDTOResponse + if err := json.NewDecoder(resp.Body).Decode(&response); err != nil { + t.Fatalf("Failed to decode response body: %v", err) + } + + // Verify the cluster was updated + if response.ClusterID != expectedClusterID { + t.Fatalf("Expected ClusterID: '%s', got: '%s'", expectedClusterID, response.ClusterID) + } + + if response.ConsoleLink != expectedConsoleLink { + t.Fatalf("Expected ConsoleLink: '%s', got: '%s'", expectedConsoleLink, response.ConsoleLink) + } + + if response.Owner != expectedOwner { + t.Fatalf("Expected Owner: '%s', got: '%s'", expectedOwner, response.Owner) + } +} + +func testPatchCluster_NotFound(t *testing.T) { + expectedHTTPCode := http.StatusNotFound + expectedMsg := "Cluster not found" + nonExistentClusterID := "non-existent-cluster" + + // Create patch request + newConsoleLink := "https://should-not-be-applied.com" + patchCluster := dto.ClusterPatchRequest{ + ConsoleLink: &newConsoleLink, + } + + patchBody, err := json.Marshal(patchCluster) + if err != nil { + t.Fatalf("Failed to marshal patch request: %v", err) + } + + // Preparing PATCH request + req, err := http.NewRequest(http.MethodPatch, APIClustersURL+"/"+nonExistentClusterID, bytes.NewBuffer(patchBody)) + if err != nil { + t.Fatalf("Failed to create request: %v", err) + } + req.Header.Set("Content-Type", "application/json") + + // Executing PATCH request + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + t.Fatalf("Failed to execute request: %v", err) + } + defer resp.Body.Close() + + // Check response code + checkHTTPResponseCode(t, resp, expectedHTTPCode) + + // Decode the JSON response + var response responsetypes.GenericErrorResponse + if err := json.NewDecoder(resp.Body).Decode(&response); err != nil { + t.Fatalf("Failed to decode response body: %v", err) + } + + // Verify error message + if response.Message != expectedMsg { + t.Fatalf("Expected Message: '%s', got: '%s'", expectedMsg, response.Message) + } } func testDeleteCluster_Exists(t *testing.T) { From 966aa93edf86c8c9ccbd324f2f8dc3e44643d192 Mon Sep 17 00:00:00 2001 From: r2dedios Date: Fri, 20 Mar 2026 14:08:32 +0100 Subject: [PATCH 24/53] test(dto): add error handling tests for ToInventoryList converters --- internal/models/dto/account_dto_test.go | 21 ++++++++++++++++++ internal/models/dto/cluster_dto_test.go | 21 ++++++++++++++++++ internal/models/dto/instance_dto_test.go | 27 ++++++++++++++++++++++++ 3 files changed, 69 insertions(+) diff --git a/internal/models/dto/account_dto_test.go b/internal/models/dto/account_dto_test.go index 95ab30e4..55ab185c 100644 --- a/internal/models/dto/account_dto_test.go +++ b/internal/models/dto/account_dto_test.go @@ -50,6 +50,7 @@ func testAccountDTORequest_ToInventoryAccount_Invalid(t *testing.T) { // TestToInventoryAccountList verifies slice conversion from DTOs to inventory.Account. func TestToInventoryAccountList(t *testing.T) { t.Run("Multiple DTOs", func(t *testing.T) { testToInventoryAccountList_Correct(t) }) + t.Run("Error on invalid DTO", func(t *testing.T) { testToInventoryAccountList_Error(t) }) } func testToInventoryAccountList_Correct(t *testing.T) { @@ -80,6 +81,26 @@ func testToInventoryAccountList_Correct(t *testing.T) { assert.Equal(t, "acc-2", (*accounts)[1].AccountID) } +func testToInventoryAccountList_Error(t *testing.T) { + dtos := []AccountDTORequest{ + { + AccountID: "acc-1", + AccountName: "account-1", + Provider: inventory.AWSProvider, + }, + { + AccountID: "", // This will cause NewAccount to return error + AccountName: "invalid-account", + Provider: inventory.AWSProvider, + }, + } + + accounts, err := ToInventoryAccountList(dtos) + + assert.Error(t, err) + assert.Nil(t, accounts) +} + // TestToAccountDTORequest verifies inventory.Account to DTO conversion. func TestToAccountDTORequest(t *testing.T) { t.Run("Account to DTO", func(t *testing.T) { testToAccountDTORequest_Correct(t) }) diff --git a/internal/models/dto/cluster_dto_test.go b/internal/models/dto/cluster_dto_test.go index a526a7ff..74a1c367 100644 --- a/internal/models/dto/cluster_dto_test.go +++ b/internal/models/dto/cluster_dto_test.go @@ -70,6 +70,7 @@ func testClusterDTORequest_ToInventoryCluster_Invalid(t *testing.T) { // TestToInventoryClusterList verifies slice conversion from DTOs to inventory.Cluster. func TestToInventoryClusterList(t *testing.T) { t.Run("Multiple DTOs", func(t *testing.T) { testToInventoryClusterList_Correct(t) }) + t.Run("Error on invalid DTO", func(t *testing.T) { testToInventoryClusterList_Error(t) }) } func testToInventoryClusterList_Correct(t *testing.T) { @@ -112,6 +113,26 @@ func testToInventoryClusterList_Correct(t *testing.T) { assert.Equal(t, "c2", (*clusters)[1].ClusterName) } +func testToInventoryClusterList_Error(t *testing.T) { + dtos := []ClusterDTORequest{ + { + ClusterName: "c1", + InfraID: "AAAAA", + Provider: inventory.AWSProvider, + }, + { + ClusterName: "", // This will cause NewCluster to return error + InfraID: "BBBBB", + Provider: inventory.AWSProvider, + }, + } + + clusters, err := ToInventoryClusterList(dtos) + + assert.Error(t, err) + assert.Nil(t, clusters) +} + // TestToClusterDTORequest verifies inventory.Cluster to DTO conversion. func TestToClusterDTORequest(t *testing.T) { t.Run("Cluster to DTO", func(t *testing.T) { testToClusterDTORequest_Correct(t) }) diff --git a/internal/models/dto/instance_dto_test.go b/internal/models/dto/instance_dto_test.go index 539b0a98..4bd9606f 100644 --- a/internal/models/dto/instance_dto_test.go +++ b/internal/models/dto/instance_dto_test.go @@ -80,6 +80,7 @@ func testInstanceDTORequest_ToInventoryInstance_Invalid(t *testing.T) { // TestToInventoryInstanceList verifies slice conversion from DTOs to inventory.Instance. func TestToInventoryInstanceList(t *testing.T) { t.Run("Multiple DTOs", func(t *testing.T) { testToInventoryInstanceList_Correct(t) }) + t.Run("Error on invalid DTO", func(t *testing.T) { testToInventoryInstanceList_Error(t) }) } func testToInventoryInstanceList_Correct(t *testing.T) { @@ -121,6 +122,32 @@ func testToInventoryInstanceList_Correct(t *testing.T) { assert.Equal(t, "i-2", (*instances)[1].InstanceID) } +func testToInventoryInstanceList_Error(t *testing.T) { + now := time.Now().UTC() + + dtos := []InstanceDTORequest{ + { + InstanceID: "i-1", + InstanceName: "n1", + Provider: inventory.AWSProvider, + CreatedAt: now, + Tags: []TagDTORequest{}, + }, + { + InstanceID: "", // This will cause NewInstance to return error + InstanceName: "n2", + Provider: inventory.AWSProvider, + CreatedAt: now, + Tags: []TagDTORequest{}, + }, + } + + instances, err := ToInventoryInstanceList(dtos) + + assert.Error(t, err) + assert.Nil(t, instances) +} + // TestToInstanceDTORequest verifies inventory.Instance to DTO conversion. func TestToInstanceDTORequest(t *testing.T) { t.Run("Instance to DTO", func(t *testing.T) { testToInstanceDTORequest_Correct(t) }) From b73b561a1ab8e09473c96e402b12e78cf5375dd9 Mon Sep 17 00:00:00 2001 From: r2dedios Date: Fri, 20 Mar 2026 14:20:22 +0100 Subject: [PATCH 25/53] docs(claude): condense and streamline CLAUDE.md documentation --- CLAUDE.md | 186 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 186 insertions(+) create mode 100644 CLAUDE.md diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 00000000..f22edcac --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,186 @@ +# CLAUDE.md + +This file provides guidance to Claude Code when working with this repository. + +## Project Overview + +**ClusterIQ** is an inventory and cost estimation platform for OpenShift clusters across multi-cloud environments (currently AWS only). It provides automated discovery, cost tracking, and lifecycle management. + +**Architecture Components:** +1. **Scanner**: CronJob that discovers cloud resources using "Stocker" pattern +2. **API Server**: REST API (Gin framework) for inventory queries and cluster operations +3. **Agent**: gRPC service handling cluster power operations (instant, scheduled, recurring) + +**Repository Structure:** +``` +cmd/ # Application entry points (api, scanner, agent) +internal/ + ├── actions/ # Action system (instant, scheduled, cron) + ├── cloud_providers/ # AWS, GCP, Azure abstractions + ├── cloud_executors/ # Action execution + ├── stocker/ # Resource discovery + ├── repositories/ # Data access layer + ├── services/ # Business logic + ├── api/handlers/ # HTTP handlers + └── models/ # DTO, DB, domain models +db/sql/ # Schemas and migrations +test/integration/ # Integration tests +``` + +## Essential Commands + +**CRITICAL**: Always use Makefile. Never use direct `go build`, `go test`, or `go run`. + +```bash +# Development +make check-dependencies # Check before building +make all # Stop, build, start dev environment +make start-dev # Start services +make stop-dev # Stop services + +# Building +make build # All container images +make local-build # All local binaries + +# Testing +make go-unit-tests +make go-integration-tests +make go-tests # All tests +make lint # Lint entire project +make lint-staged # Lint staged files only + +# Code Generation +make generate-converters # Goverter (DB to DTO) +make swagger-doc # OpenAPI docs +``` + +## Architecture Patterns + +**Layered Architecture:** +- Handlers → Services → Repositories → Database +- Each layer has interfaces for testability +- Repository returns `repositories.ErrNotFound` for missing resources +- Services validate business rules, wrap errors with context +- Handlers map errors to HTTP status codes (404, 400, 500) + +**Key Patterns:** +- **Repository Pattern**: Data access abstraction +- **Service Layer**: Business logic with dependency injection +- **Stocker Pattern**: Cloud resource discovery via `MakeStock()` +- **Action Channel**: Unbuffered channel for decoupled action dispatch +- **Event Tracker**: Audit logging with `EventService.StartTracking()` +- **Functional Options**: AWS connections use `WithEC2()`, `WithRoute53()`, etc. + +## Testing Guidelines + +**Service Layer Mocking:** +```go +// Mock repositories with function pointers +type mockClusterRepository struct { + getClusterByIDFn func(ctx context.Context, clusterID string) (*db.ClusterDBResponse, error) +} + +func (m *mockClusterRepository) GetClusterByID(ctx context.Context, clusterID string) (*db.ClusterDBResponse, error) { + if m.getClusterByIDFn != nil { + return m.getClusterByIDFn(ctx, clusterID) + } + return nil, nil +} +``` + +**gRPC Mocking:** +- Mock the underlying `pb.AgentServiceClient` interface, not the wrapper +- Return actual Go errors: `return nil, errTest` +- DON'T use response error codes for Go errors: `&pb.Response{Error: 1}` returns `nil` error + +**Test Organization:** +- Group by method: `TestServiceName_MethodName(t *testing.T)` +- Use sub-tests: `t.Run("success", func(t *testing.T) { ... })` +- Naming: `testServiceName_MethodName_Scenario` +- All interface methods must be implemented in mocks (even if unused) + +**Coverage Analysis:** +```bash +go test -coverprofile=coverage.out ./internal/services/... +go tool cover -func=coverage.out | grep -v "100.0%" # Find uncovered +go tool cover -html=coverage.out -o coverage.html # Visual +``` + +## Development Workflow + +1. Make code changes +2. Run `make lint-staged` before committing +3. Run relevant tests: `make go-unit-tests` +4. For API changes: update Swagger with `make swagger-doc` +5. For DB changes: create migration in `db/sql/migrations/` +6. For protobuf changes: `make local-build-agent` +7. For goverter changes: `make generate-converters` + +**Commit Convention:** +- Use conventional commits format: `type(scope): brief description` +- Keep messages to single line +- Types: `feat`, `fix`, `refactor`, `test`, `docs`, `chore` +- Examples: + - `feat(api): implement PATCH endpoint for accounts` + - `fix(aws): continue processing instances on conversion error` + - `test(dto): add error handling tests for converters` + +## Critical Implementation Notes + +**Cloud Support:** +- AWS: Fully implemented ✅ +- GCP/Azure: Interface stubs only ❌ + +**Data Consistency:** +- Each repository method = separate transaction +- No distributed transactions +- Eventual consistency for action status updates + +**PATCH Endpoints:** +- Use pointer fields in DTOs to distinguish null from empty +- Dynamic SQL query building with positional parameters +- Verify resource exists before updating +- Refresh materialized views after successful update +- Return updated resource with HTTP 200 + +## Configuration + +Environment variables: +- `CIQ_AGENT_URL` - Agent gRPC endpoint (default: "agent:50051") +- `CIQ_API_URL` - API public endpoint +- `CIQ_DB_URL` - PostgreSQL connection (default: "postgresql://pgsql:5432/clusteriq") +- `CIQ_CREDS_FILE` - Cloud provider credentials file +- `CIQ_LOG_LEVEL` - Log verbosity (default: "INFO") + +## AWS Integration Details + +**Cluster Discovery:** +- OpenShift tags: `kubernetes.io/cluster/-` +- Console URLs via Route53 DNS: `console-openshift-console.apps.*` + +**Instance Timestamps:** +- Primary: Root EBS volume `AttachTime` (via `instance.RootDeviceName`) +- Fallback: Instance `LaunchTime` +- NEVER hardcode device names (varies by instance type) + +**Cost Explorer:** +- 14-day rolling window, DAILY granularity +- Metric: UnblendedCost +- One Expense object per instance per day + +**EC2 Operations:** +- Start: Only targets `stopped` instances +- Stop: Only targets `running` instances +- Prevents idempotency errors + +## Code Generation + +**Goverter**: Type-safe DB ↔ DTO converters +- Definitions: `internal/models/convert/` +- Generated: `internal/models/convert/generated.go` + +**Swagger**: OpenAPI from Go comments +- Annotations in handlers and `cmd/api/server.go` + +**Protobuf**: gRPC code from `cmd/agent/proto/agent.proto` +- Auto-generated during `make local-build-agent` From ecb680637cd28244f49c81c327e976b26edbc400 Mon Sep 17 00:00:00 2001 From: r2dedios Date: Fri, 20 Mar 2026 17:39:45 +0100 Subject: [PATCH 26/53] fix(agent): Solved conccurency access problem when accessing schedule map Signed-off-by: r2dedios --- cmd/agent/agent.go | 11 +++++++- cmd/agent/schedule_agent_service.go | 44 ++++++++++++++++++++++++----- 2 files changed, 47 insertions(+), 8 deletions(-) diff --git a/cmd/agent/agent.go b/cmd/agent/agent.go index cfcc3f5a..4e8424da 100644 --- a/cmd/agent/agent.go +++ b/cmd/agent/agent.go @@ -221,7 +221,16 @@ func (a Agent) signalHandler(signal os.Signal) error { a.logger.Warn("Shutting down server...", zap.String("signal", signal.String())) } - for _, item := range a.sas.schedule { + // Copy schedule map under lock to avoid race condition during iteration + a.sas.mutex.Lock() + scheduleCopy := make(map[string]scheduleItem, len(a.sas.schedule)) + for k, v := range a.sas.schedule { + scheduleCopy[k] = v + } + a.sas.mutex.Unlock() + + // Iterate over the copy (without holding lock) to cancel all scheduled actions + for _, item := range scheduleCopy { cancel := item.cancel action := item.action a.logger.Warn("Cancelling ScheduledAction", zap.String("action_id", action.GetID())) diff --git a/cmd/agent/schedule_agent_service.go b/cmd/agent/schedule_agent_service.go index 1c52f3e3..f2b23382 100644 --- a/cmd/agent/schedule_agent_service.go +++ b/cmd/agent/schedule_agent_service.go @@ -75,12 +75,26 @@ func NewScheduleAgentService(cfg *config.ScheduleAgentServiceConfig, actionsChan } // scheduleNewScheduledAction starts the timing until action's execution timestamp and writes the message on the actions channel to be executed on the ExecutorAgentService +// This is the public version that acquires the mutex before calling the internal implementation. // // Parameters: // - newAction: the new actions.ScheduledAction to be executed // // Returns: func (a *ScheduleAgentService) scheduleNewScheduledAction(newAction *actions.ScheduledAction) { + a.mutex.Lock() + defer a.mutex.Unlock() + a.scheduleNewScheduledActionLocked(newAction) +} + +// scheduleNewScheduledActionLocked is the internal implementation that assumes the mutex is already held. +// This method should only be called from within methods that have already acquired a.mutex. +// +// Parameters: +// - newAction: the new actions.ScheduledAction to be executed +// +// Returns: +func (a *ScheduleAgentService) scheduleNewScheduledActionLocked(newAction *actions.ScheduledAction) { actionID := newAction.GetID() // Check if the duration is negative, which means it refers to a past timestamp @@ -119,6 +133,7 @@ func (a *ScheduleAgentService) scheduleNewScheduledAction(newAction *actions.Sch } // rescheduleScheduleAction Re-schedules the scheduled action considering it's already running +// This method assumes the mutex is already held by the caller (e.g., ScheduleNewActions). // // Parameters: // - newAction: the new actions.ScheduledAction to be executed @@ -132,18 +147,32 @@ func (a *ScheduleAgentService) rescheduleScheduledAction(newAction *actions.Sche // Canceling previous action instance a.schedule[actionID].cancel() - // Re-scheduling action - a.scheduleNewScheduledAction(newAction) + // Re-scheduling action (using locked version since we already have the mutex) + a.scheduleNewScheduledActionLocked(newAction) } } // scheduleNewCronAction starts the timing until action's execution timestamp and writes the message on the actions channel to be executed on the ExecutorAgentService +// This is the public version that acquires the mutex before calling the internal implementation. // // Parameters: -// - newAction: the new actions.ScheduledAction to be executed +// - newAction: the new actions.CronAction to be executed // // Returns: func (a *ScheduleAgentService) scheduleNewCronAction(newAction *actions.CronAction) { + a.mutex.Lock() + defer a.mutex.Unlock() + a.scheduleNewCronActionLocked(newAction) +} + +// scheduleNewCronActionLocked is the internal implementation that assumes the mutex is already held. +// This method should only be called from within methods that have already acquired a.mutex. +// +// Parameters: +// - newAction: the new actions.CronAction to be executed +// +// Returns: +func (a *ScheduleAgentService) scheduleNewCronActionLocked(newAction *actions.CronAction) { actionID := newAction.GetID() // Creating new action context and cancel function @@ -194,10 +223,11 @@ func (a *ScheduleAgentService) scheduleNewCronAction(newAction *actions.CronActi }() } -// rescheduleCronAction Re-schedules the cron action considering it's already running +// rescheduleCronAction Re-schedules the cron action considering it's already running +// This method assumes the mutex is already held by the caller (e.g., ScheduleNewActions). // // Parameters: -// - newAction: the new actions.ScheduledAction to be executed +// - newAction: the new actions.CronAction to be executed // // Returns: func (a *ScheduleAgentService) rescheduleCronAction(newAction *actions.CronAction) { @@ -208,8 +238,8 @@ func (a *ScheduleAgentService) rescheduleCronAction(newAction *actions.CronActio // Canceling previous action instance a.schedule[actionID].cancel() - // Re-scheduling action - a.scheduleNewCronAction(newAction) + // Re-scheduling action (using locked version since we already have the mutex) + a.scheduleNewCronActionLocked(newAction) } } From eed97544cbe6caa30aa25fa0347f5a66624deff5 Mon Sep 17 00:00:00 2001 From: r2dedios Date: Mon, 23 Mar 2026 11:12:10 +0100 Subject: [PATCH 27/53] fix(db): use named returns to prevent rollback after commit Signed-off-by: r2dedios --- internal/db_client/db_client.go | 108 +++++++++++--------- internal/repositories/account_repository.go | 30 +++--- internal/repositories/cluster_repository.go | 30 +++--- 3 files changed, 93 insertions(+), 75 deletions(-) diff --git a/internal/db_client/db_client.go b/internal/db_client/db_client.go index a916eb01..6cbdaaa1 100644 --- a/internal/db_client/db_client.go +++ b/internal/db_client/db_client.go @@ -125,24 +125,23 @@ func (d *DBClient) Select(dest interface{}, table string, opts models.ListOption return d.SelectWithContext(context.TODO(), dest, table, opts, orderColumn, columns...) } -func (d *DBClient) InsertWithReturnWithContext(ctx context.Context, query string, data interface{}) (int64, error) { +func (d *DBClient) InsertWithReturnWithContext(ctx context.Context, query string, data interface{}) (returnedValue int64, err error) { builder := d.NewInsertBuilder().Query(query).Data(data) - tx, err := d.db.BeginTxx(ctx, nil) - if err != nil { - return -1, err + tx, txErr := d.db.BeginTxx(ctx, nil) + if txErr != nil { + return -1, txErr } - // Rollback defer func + // Rollback defer func - only executes if err != nil (transaction not committed) defer func() { if err != nil { if rbErr := tx.Rollback(); rbErr != nil { - d.logger.Error("failed to Rollback INSERT") + d.logger.Error("failed to rollback transaction", zap.Error(rbErr)) } } }() - var returnedValue int64 rows, err := tx.NamedQuery(builder.query, builder.data) if err != nil { return -1, fmt.Errorf("named-exec INSERT error: %w", err) @@ -154,42 +153,47 @@ func (d *DBClient) InsertWithReturnWithContext(ctx context.Context, query string }() if rows.Next() { - if err := rows.Scan(&returnedValue); err != nil { - return -1, fmt.Errorf("scan INSERT return value error %w", err) + if scanErr := rows.Scan(&returnedValue); scanErr != nil { + err = fmt.Errorf("scan INSERT return value error %w", scanErr) + return -1, err } } else { - return -1, fmt.Errorf("sql INSERT did not return any value") + err = fmt.Errorf("sql INSERT did not return any value") + return -1, err } - if err := tx.Commit(); err != nil { + err = tx.Commit() + if err != nil { return -1, fmt.Errorf("commit INSERT error: %w", err) } return returnedValue, nil } -func (d *DBClient) InsertWithContext(ctx context.Context, query string, data interface{}) error { +func (d *DBClient) InsertWithContext(ctx context.Context, query string, data interface{}) (err error) { builder := d.NewInsertBuilder().Query(query).Data(data) - tx, err := d.db.BeginTxx(ctx, nil) - if err != nil { - return err + tx, txErr := d.db.BeginTxx(ctx, nil) + if txErr != nil { + return txErr } - // Rollback defer func + // Rollback defer func - only executes if err != nil (transaction not committed) defer func() { if err != nil { if rbErr := tx.Rollback(); rbErr != nil { - d.logger.Error("failed to Rollback INSERT") + d.logger.Error("failed to rollback transaction", zap.Error(rbErr)) } } }() - if _, err := tx.NamedExecContext(ctx, builder.query, builder.data); err != nil { - return fmt.Errorf("named-exec INSERT error: %w", err) + if _, execErr := tx.NamedExecContext(ctx, builder.query, builder.data); execErr != nil { + err = fmt.Errorf("named-exec INSERT error: %w", execErr) + return err } - if err := tx.Commit(); err != nil { + err = tx.Commit() + if err != nil { return fmt.Errorf("commit INSERT error: %w", err) } @@ -202,28 +206,30 @@ func (d *DBClient) Insert(query string, data interface{}) error { return d.InsertWithContext(context.TODO(), query, data) } -func (d *DBClient) UpdateWithContext(ctx context.Context, query string, data interface{}) error { +func (d *DBClient) UpdateWithContext(ctx context.Context, query string, data interface{}) (err error) { builder := d.NewUpdateBuilder().Query(query).Data(data) - tx, err := d.db.BeginTxx(ctx, nil) - if err != nil { - return err + tx, txErr := d.db.BeginTxx(ctx, nil) + if txErr != nil { + return txErr } - // Rollback defer func + // Rollback defer func - only executes if err != nil (transaction not committed) defer func() { if err != nil { if rbErr := tx.Rollback(); rbErr != nil { - d.logger.Error("failed to Rollback UPDATE") + d.logger.Error("failed to rollback transaction", zap.Error(rbErr)) } } }() - if _, err := tx.ExecContext(ctx, builder.query, builder.data); err != nil { - return fmt.Errorf("exec UPDATE error: %w", err) + if _, execErr := tx.ExecContext(ctx, builder.query, builder.data); execErr != nil { + err = fmt.Errorf("exec UPDATE error: %w", execErr) + return err } - if err := tx.Commit(); err != nil { + err = tx.Commit() + if err != nil { return fmt.Errorf("commit UPDATE error: %w", err) } @@ -236,28 +242,30 @@ func (d *DBClient) Update(query string, data interface{}) error { return d.UpdateWithContext(context.TODO(), query, data) } -func (d *DBClient) NamedUpdateWithContext(ctx context.Context, query string, data interface{}) error { +func (d *DBClient) NamedUpdateWithContext(ctx context.Context, query string, data interface{}) (err error) { builder := d.NewUpdateBuilder().Query(query).Data(data) - tx, err := d.db.BeginTxx(ctx, nil) - if err != nil { - return err + tx, txErr := d.db.BeginTxx(ctx, nil) + if txErr != nil { + return txErr } - // Rollback defer func + // Rollback defer func - only executes if err != nil (transaction not committed) defer func() { if err != nil { if rbErr := tx.Rollback(); rbErr != nil { - d.logger.Error("failed to Rollback UPDATE") + d.logger.Error("failed to rollback transaction", zap.Error(rbErr)) } } }() - if _, err := tx.NamedExecContext(ctx, builder.query, builder.data); err != nil { - return fmt.Errorf("named-exec UPDATE error: %w", err) + if _, execErr := tx.NamedExecContext(ctx, builder.query, builder.data); execErr != nil { + err = fmt.Errorf("named-exec UPDATE error: %w", execErr) + return err } - if err := tx.Commit(); err != nil { + err = tx.Commit() + if err != nil { return fmt.Errorf("commit UPDATE error: %w", err) } @@ -271,7 +279,7 @@ func (d *DBClient) NamedUpdate(query string, data interface{}) error { } // Delete executes a DELETE with a safe transaction pattern.Delete -func (d *DBClient) DeleteWithContext(ctx context.Context, table string, opts models.ListOptions) error { +func (d *DBClient) DeleteWithContext(ctx context.Context, table string, opts models.ListOptions) (err error) { builder := d.NewDeleteBuilder().From(table) // Processing "WHERE" conditions @@ -283,30 +291,32 @@ func (d *DBClient) DeleteWithContext(ctx context.Context, table string, opts mod } // Building query - query, args, err := builder.Build() - if err != nil { + query, args, buildErr := builder.Build() + if buildErr != nil { return fmt.Errorf("error building DELETE query: '%s'", query) } - tx, err := d.db.BeginTxx(ctx, nil) - if err != nil { - return err + tx, txErr := d.db.BeginTxx(ctx, nil) + if txErr != nil { + return txErr } - // Rollback defer func + // Rollback defer func - only executes if err != nil (transaction not committed) defer func() { if err != nil { if rbErr := tx.Rollback(); rbErr != nil { - d.logger.Error("failed to Rollback DELETE") + d.logger.Error("failed to rollback transaction", zap.Error(rbErr)) } } }() - if _, err := tx.ExecContext(ctx, query, args...); err != nil { - return fmt.Errorf("exec DELETE error: %w", err) + if _, execErr := tx.ExecContext(ctx, query, args...); execErr != nil { + err = fmt.Errorf("exec DELETE error: %w", execErr) + return err } - if err := tx.Commit(); err != nil { + err = tx.Commit() + if err != nil { return fmt.Errorf("commit DELETE error: %w", err) } diff --git a/internal/repositories/account_repository.go b/internal/repositories/account_repository.go index 5e4c283c..00ae855d 100644 --- a/internal/repositories/account_repository.go +++ b/internal/repositories/account_repository.go @@ -193,7 +193,7 @@ func (r *accountRepositoryImpl) CreateAccount(ctx context.Context, accounts []in // UpdateAccount updates mutable fields of an existing account in the database. // Only non-nil fields in the patch request will be updated. -func (r *accountRepositoryImpl) UpdateAccount(ctx context.Context, accountID string, patch dto.AccountPatchRequest) error { +func (r *accountRepositoryImpl) UpdateAccount(ctx context.Context, accountID string, patch dto.AccountPatchRequest) (err error) { // Build dynamic UPDATE query with positional parameters query := "UPDATE accounts SET " args := make([]interface{}, 0) @@ -214,9 +214,9 @@ func (r *accountRepositoryImpl) UpdateAccount(ctx context.Context, accountID str args = append(args, accountID) // Execute update in a transaction - tx, err := r.db.NewTx(ctx) - if err != nil { - return err + tx, txErr := r.db.NewTx(ctx) + if txErr != nil { + return txErr } defer func() { if err != nil { @@ -224,19 +224,21 @@ func (r *accountRepositoryImpl) UpdateAccount(ctx context.Context, accountID str } }() - if _, err = tx.ExecContext(ctx, query, args...); err != nil { - return fmt.Errorf("exec UPDATE error: %w", err) + if _, execErr := tx.ExecContext(ctx, query, args...); execErr != nil { + err = fmt.Errorf("exec UPDATE error: %w", execErr) + return err } - if err = tx.Commit(); err != nil { + err = tx.Commit() + if err != nil { return fmt.Errorf("commit UPDATE error: %w", err) } // Refresh materialized view after transaction commits // Note: REFRESH MATERIALIZED VIEW must run outside the transaction - tx2, err := r.db.NewTx(ctx) - if err != nil { - return fmt.Errorf("failed to create transaction for refresh: %w", err) + tx2, tx2Err := r.db.NewTx(ctx) + if tx2Err != nil { + return fmt.Errorf("failed to create transaction for refresh: %w", tx2Err) } defer func() { if err != nil { @@ -244,11 +246,13 @@ func (r *accountRepositoryImpl) UpdateAccount(ctx context.Context, accountID str } }() - if _, err = tx2.ExecContext(ctx, "REFRESH MATERIALIZED VIEW m_accounts_full_view"); err != nil { - return fmt.Errorf("refresh materialized view error: %w", err) + if _, refreshErr := tx2.ExecContext(ctx, "REFRESH MATERIALIZED VIEW m_accounts_full_view"); refreshErr != nil { + err = fmt.Errorf("refresh materialized view error: %w", refreshErr) + return err } - if err = tx2.Commit(); err != nil { + err = tx2.Commit() + if err != nil { return fmt.Errorf("commit refresh error: %w", err) } diff --git a/internal/repositories/cluster_repository.go b/internal/repositories/cluster_repository.go index 1f864457..f09954d8 100644 --- a/internal/repositories/cluster_repository.go +++ b/internal/repositories/cluster_repository.go @@ -297,7 +297,7 @@ func (r *clusterRepositoryImpl) CreateClusters(ctx context.Context, clusters []i // UpdateCluster updates mutable fields of an existing cluster in the database. // Only non-nil fields in the patch request will be updated. -func (r *clusterRepositoryImpl) UpdateCluster(ctx context.Context, clusterID string, patch dto.ClusterPatchRequest) error { +func (r *clusterRepositoryImpl) UpdateCluster(ctx context.Context, clusterID string, patch dto.ClusterPatchRequest) (err error) { // Build dynamic UPDATE query with positional parameters query := "UPDATE clusters SET " args := make([]interface{}, 0) @@ -326,9 +326,9 @@ func (r *clusterRepositoryImpl) UpdateCluster(ctx context.Context, clusterID str args = append(args, clusterID) // Execute update in a transaction - tx, err := r.db.NewTx(ctx) - if err != nil { - return err + tx, txErr := r.db.NewTx(ctx) + if txErr != nil { + return txErr } defer func() { if err != nil { @@ -336,19 +336,21 @@ func (r *clusterRepositoryImpl) UpdateCluster(ctx context.Context, clusterID str } }() - if _, err = tx.ExecContext(ctx, query, args...); err != nil { - return fmt.Errorf("exec UPDATE error: %w", err) + if _, execErr := tx.ExecContext(ctx, query, args...); execErr != nil { + err = fmt.Errorf("exec UPDATE error: %w", execErr) + return err } - if err = tx.Commit(); err != nil { + err = tx.Commit() + if err != nil { return fmt.Errorf("commit UPDATE error: %w", err) } // Refresh materialized view after transaction commits // Note: REFRESH MATERIALIZED VIEW must run outside the transaction - tx2, err := r.db.NewTx(ctx) - if err != nil { - return fmt.Errorf("failed to create transaction for refresh: %w", err) + tx2, tx2Err := r.db.NewTx(ctx) + if tx2Err != nil { + return fmt.Errorf("failed to create transaction for refresh: %w", tx2Err) } defer func() { if err != nil { @@ -356,11 +358,13 @@ func (r *clusterRepositoryImpl) UpdateCluster(ctx context.Context, clusterID str } }() - if _, err = tx2.ExecContext(ctx, "REFRESH MATERIALIZED VIEW m_clusters_full_view"); err != nil { - return fmt.Errorf("refresh materialized view error: %w", err) + if _, refreshErr := tx2.ExecContext(ctx, "REFRESH MATERIALIZED VIEW m_clusters_full_view"); refreshErr != nil { + err = fmt.Errorf("refresh materialized view error: %w", refreshErr) + return err } - if err = tx2.Commit(); err != nil { + err = tx2.Commit() + if err != nil { return fmt.Errorf("commit refresh error: %w", err) } From b57531557493e47a97f8ec3f0aede6aad2551d65 Mon Sep 17 00:00:00 2001 From: r2dedios Date: Mon, 23 Mar 2026 12:27:04 +0100 Subject: [PATCH 28/53] fix(scanner): handle cluster addition errors in instance processing Signed-off-by: r2dedios --- internal/stocker/aws_stocker_ec2.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/internal/stocker/aws_stocker_ec2.go b/internal/stocker/aws_stocker_ec2.go index 887ac4f9..6b6bb636 100644 --- a/internal/stocker/aws_stocker_ec2.go +++ b/internal/stocker/aws_stocker_ec2.go @@ -59,9 +59,16 @@ func (s *AWSStocker) processInstances(instances []inventory.Instance) { } if !s.Account.IsClusterInAccount(cluster.ClusterID) { - _ = s.Account.AddCluster(cluster) + if err := s.Account.AddCluster(cluster); err != nil { + s.logger.Error("error adding cluster to account during instance processing", + zap.String("account_id", s.Account.AccountID), + zap.String("cluster_id", cluster.ClusterID), + zap.Error(err)) + continue + } } + // At this point, the cluster is guaranteed to exist in s.Account.Clusters if err := s.Account.Clusters[cluster.ClusterID].AddInstance(&instance); err != nil { s.logger.Error("error adding instance to cluster during instance processing", zap.String("account_id", s.Account.AccountID), From 40a89651c2be121f0b179005698c86ab64e46bf1 Mon Sep 17 00:00:00 2001 From: r2dedios Date: Mon, 23 Mar 2026 13:18:21 +0100 Subject: [PATCH 29/53] tests: Fixed integration tests errors by restoring static values after modifying the DB Signed-off-by: r2dedios --- internal/repositories/account_repository.go | 2 +- internal/repositories/cluster_repository.go | 2 +- .../api_accounts_integration_test.go | 29 +++++++++++++++++ .../api_clusters_integration_test.go | 32 +++++++++++++++++++ .../api_overview_integration_test.go | 4 +-- 5 files changed, 65 insertions(+), 4 deletions(-) diff --git a/internal/repositories/account_repository.go b/internal/repositories/account_repository.go index 00ae855d..c578a6f7 100644 --- a/internal/repositories/account_repository.go +++ b/internal/repositories/account_repository.go @@ -114,7 +114,7 @@ func (r *accountRepositoryImpl) GetAccountByID(ctx context.Context, accountID st }, } - if err := r.db.GetWithContext(ctx, &account, SelectAccountsMView, opts, "*"); err != nil { + if err := r.db.GetWithContext(ctx, &account, SelectAccountsView, opts, "*"); err != nil { if errors.Is(err, sql.ErrNoRows) { return account, ErrNotFound } diff --git a/internal/repositories/cluster_repository.go b/internal/repositories/cluster_repository.go index f09954d8..b7da9037 100644 --- a/internal/repositories/cluster_repository.go +++ b/internal/repositories/cluster_repository.go @@ -137,7 +137,7 @@ func (r *clusterRepositoryImpl) GetClusterByID(ctx context.Context, clusterID st }, } - if err := r.db.GetWithContext(ctx, &cluster, SelectClustersFullMView, opts, "*"); err != nil { + if err := r.db.GetWithContext(ctx, &cluster, SelectClustersFullView, opts, "*"); err != nil { if errors.Is(err, sql.ErrNoRows) { return nil, ErrNotFound } diff --git a/test/integration/api_accounts_integration_test.go b/test/integration/api_accounts_integration_test.go index 0aa7449c..037e86e7 100644 --- a/test/integration/api_accounts_integration_test.go +++ b/test/integration/api_accounts_integration_test.go @@ -309,6 +309,18 @@ func testPostMultipleAccounts(t *testing.T) { }, } + // Cleanup created accounts after test + defer func() { + for _, acc := range payload { + req, _ := http.NewRequest(http.MethodDelete, APIAccountsURL+"/"+acc.AccountID, nil) + client := &http.Client{} + resp, _ := client.Do(req) + if resp != nil { + resp.Body.Close() + } + } + }() + // Posting test data resp := postAccounts(t, payload, expectedHTTPCode) defer resp.Body.Close() @@ -363,8 +375,25 @@ func testPostWrongAccount(t *testing.T) { func testPatchAccount_Success(t *testing.T) { expectedHTTPCode := http.StatusOK expectedAccountID := "gcp-project-1" + originalAccountName := "gcp-project-demo" expectedAccountName := "updated-gcp-project-name" + // Restore original account name after test to prevent interference with other tests + defer func() { + restoreName := originalAccountName + restorePatch := dto.AccountPatchRequest{ + AccountName: &restoreName, + } + restoreBody, _ := json.Marshal(restorePatch) + req, _ := http.NewRequest(http.MethodPatch, APIAccountsURL+"/"+expectedAccountID, bytes.NewBuffer(restoreBody)) + req.Header.Set("Content-Type", "application/json") + client := &http.Client{} + resp, _ := client.Do(req) + if resp != nil { + resp.Body.Close() + } + }() + // Create patch request with only the fields to update newAccountName := expectedAccountName patchAccount := dto.AccountPatchRequest{ diff --git a/test/integration/api_clusters_integration_test.go b/test/integration/api_clusters_integration_test.go index c87b8d59..cd1b440c 100644 --- a/test/integration/api_clusters_integration_test.go +++ b/test/integration/api_clusters_integration_test.go @@ -455,6 +455,18 @@ func testPostMultipleClusters(t *testing.T) { }, } + // Cleanup created clusters after test + defer func() { + for _, cluster := range payload { + req, _ := http.NewRequest(http.MethodDelete, APIClustersURL+"/"+cluster.ClusterID, nil) + client := &http.Client{} + resp, _ := client.Do(req) + if resp != nil { + resp.Body.Close() + } + } + }() + // Posting test data resp := postClusters(t, payload, expectedHTTPCode) defer resp.Body.Close() @@ -516,9 +528,29 @@ func testPostWrongCluster(t *testing.T) { func testPatchCluster_Success(t *testing.T) { expectedHTTPCode := http.StatusOK expectedClusterID := "aws-cluster-1-aws-infra-1" + originalConsoleLink := "https://console.aws/1" + originalOwner := "team-aws" expectedConsoleLink := "https://updated-console.example.com" expectedOwner := "Updated Owner Name" + // Restore original values after test to prevent interference with other tests + defer func() { + restoreConsoleLink := originalConsoleLink + restoreOwner := originalOwner + restorePatch := dto.ClusterPatchRequest{ + ConsoleLink: &restoreConsoleLink, + Owner: &restoreOwner, + } + restoreBody, _ := json.Marshal(restorePatch) + req, _ := http.NewRequest(http.MethodPatch, APIClustersURL+"/"+expectedClusterID, bytes.NewBuffer(restoreBody)) + req.Header.Set("Content-Type", "application/json") + client := &http.Client{} + resp, _ := client.Do(req) + if resp != nil { + resp.Body.Close() + } + }() + // Create patch request with only the fields to update newConsoleLink := expectedConsoleLink newOwner := expectedOwner diff --git a/test/integration/api_overview_integration_test.go b/test/integration/api_overview_integration_test.go index 7052ef26..3ab7390a 100644 --- a/test/integration/api_overview_integration_test.go +++ b/test/integration/api_overview_integration_test.go @@ -44,11 +44,11 @@ func testGetOverview(t *testing.T) { ClusterCount: 2, }, GCP: dto.ProviderDetails{ - AccountCount: 2, + AccountCount: 1, ClusterCount: 2, }, Azure: dto.ProviderDetails{ - AccountCount: 2, + AccountCount: 1, ClusterCount: 2, }, }, From c517fdd9ed1200905ffe10712f2e0d22e63d38fc Mon Sep 17 00:00:00 2001 From: r2dedios Date: Wed, 29 Apr 2026 11:45:57 +0200 Subject: [PATCH 30/53] chore(deps): update Go dependencies --- go.mod | 29 +++++++++++---------- go.sum | 82 +++++++++++++++++++++++++++++++--------------------------- 2 files changed, 59 insertions(+), 52 deletions(-) diff --git a/go.mod b/go.mod index cbdfb09a..5dea2485 100644 --- a/go.mod +++ b/go.mod @@ -7,14 +7,15 @@ require ( github.com/caarlos0/env/v11 v11.3.1 github.com/gin-contrib/zap v0.1.0 github.com/gin-gonic/gin v1.9.1 + github.com/jmattheis/goverter v1.9.4 github.com/jmoiron/sqlx v1.3.5 github.com/lib/pq v1.10.9 github.com/robfig/cron/v3 v3.0.1 - github.com/stretchr/testify v1.9.0 + github.com/stretchr/testify v1.11.1 github.com/swaggo/swag v1.16.6 go.uber.org/zap v1.25.0 - google.golang.org/grpc v1.69.4 - google.golang.org/protobuf v1.35.1 + google.golang.org/grpc v1.79.3 + google.golang.org/protobuf v1.36.10 gopkg.in/ini.v1 v1.67.0 ) @@ -23,6 +24,7 @@ require ( github.com/PuerkitoBio/purell v1.1.1 // indirect github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect github.com/bytedance/sonic v1.9.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect github.com/dave/jennifer v1.6.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect @@ -36,7 +38,6 @@ require ( github.com/go-playground/universal-translator v0.18.1 // indirect github.com/go-playground/validator/v10 v10.14.0 // indirect github.com/goccy/go-json v0.10.2 // indirect - github.com/jmattheis/goverter v1.9.3 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect @@ -51,18 +52,18 @@ require ( github.com/pmezard/go-difflib v1.0.0 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.2.11 // indirect - go.opentelemetry.io/otel v1.31.0 // indirect - go.opentelemetry.io/otel/trace v1.31.0 // indirect + go.opentelemetry.io/otel v1.39.0 // indirect + go.opentelemetry.io/otel/trace v1.39.0 // indirect go.uber.org/multierr v1.10.0 // indirect golang.org/x/arch v0.3.0 // indirect - golang.org/x/crypto v0.45.0 // indirect - golang.org/x/mod v0.29.0 // indirect - golang.org/x/net v0.47.0 // indirect - golang.org/x/sync v0.18.0 // indirect - golang.org/x/sys v0.38.0 // indirect - golang.org/x/text v0.31.0 // indirect - golang.org/x/tools v0.38.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20241015192408-796eee8c2d53 // indirect + golang.org/x/crypto v0.46.0 // indirect + golang.org/x/mod v0.30.0 // indirect + golang.org/x/net v0.48.0 // indirect + golang.org/x/sync v0.19.0 // indirect + golang.org/x/sys v0.39.0 // indirect + golang.org/x/text v0.32.0 // indirect + golang.org/x/tools v0.39.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index d5bb586e..a3be0c5b 100644 --- a/go.sum +++ b/go.sum @@ -14,6 +14,8 @@ github.com/bytedance/sonic v1.9.1 h1:6iJ6NqdoxCDr6mbY8h18oSO+cShGSMRGCEo7F2h0x8s github.com/bytedance/sonic v1.9.1/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U= github.com/caarlos0/env/v11 v11.3.1 h1:cArPWC15hWmEt+gWk7YBi7lEXTXCvpaSdCiZE2X5mCA= github.com/caarlos0/env/v11 v11.3.1/go.mod h1:qupehSf/Y0TUTsxKywqRt/vJjN5nz6vauiYEUUr8P4U= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY= github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 h1:qSGYFH7+jGhDF8vLC+iwCD4WpbV1EBDSzWkJODFLams= github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk= @@ -34,8 +36,8 @@ github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg= github.com/gin-gonic/gin v1.9.1/go.mod h1:hPrL7YrpYKXt5YId3A/Tnip5kqbEAP+KLuI3SUcPTeU= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= -github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= @@ -70,13 +72,13 @@ github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= -github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/jmattheis/goverter v1.9.3 h1:dTGAI9sqNgVsd9vkHDKpVJCE/Aw6WlIokqrAdSWP2PI= -github.com/jmattheis/goverter v1.9.3/go.mod h1:1n3q6zf7j58tXcRWHbLFxK2Jk8WQVzr0d3nuaCcRqeg= +github.com/jmattheis/goverter v1.9.4 h1:xdBN4TL/ihGkm1V1TqQ7gmTSVA3G4TI3/C/iK26nJqs= +github.com/jmattheis/goverter v1.9.4/go.mod h1:1n3q6zf7j58tXcRWHbLFxK2Jk8WQVzr0d3nuaCcRqeg= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= @@ -144,8 +146,8 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= -github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/swaggo/swag v1.16.6 h1:qBNcx53ZaX+M5dxVyTrgQ0PJ/ACK+NzhwcbieTt+9yI= github.com/swaggo/swag v1.16.6/go.mod h1:ngP2etMK5a0P3QBizic5MEwpRmluJZPHjXcMoj4Xesg= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= @@ -155,18 +157,20 @@ github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95 github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU= github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/otel v1.10.0/go.mod h1:NbvWjCthWHKBEUMpf0/v8ZRZlni86PpGFEMA9pnQSnQ= -go.opentelemetry.io/otel v1.31.0 h1:NsJcKPIW0D0H3NgzPDHmo0WW6SptzPdqg/L1zsIm2hY= -go.opentelemetry.io/otel v1.31.0/go.mod h1:O0C14Yl9FgkjqcCZAsE053C13OaddMYr/hz6clDkEJE= -go.opentelemetry.io/otel/metric v1.31.0 h1:FSErL0ATQAmYHUIzSezZibnyVlft1ybhy4ozRPcF2fE= -go.opentelemetry.io/otel/metric v1.31.0/go.mod h1:C3dEloVbLuYoX41KpmAhOqNriGbA+qqH6PQ5E5mUfnY= -go.opentelemetry.io/otel/sdk v1.31.0 h1:xLY3abVHYZ5HSfOg3l2E5LUj2Cwva5Y7yGxnSW9H5Gk= -go.opentelemetry.io/otel/sdk v1.31.0/go.mod h1:TfRbMdhvxIIr/B2N2LQW2S5v9m3gOQ/08KsbbO5BPT0= -go.opentelemetry.io/otel/sdk/metric v1.31.0 h1:i9hxxLJF/9kkvfHppyLL55aW7iIJz4JjxTeYusH7zMc= -go.opentelemetry.io/otel/sdk/metric v1.31.0/go.mod h1:CRInTMVvNhUKgSAMbKyTMxqOBC0zgyxzW55lZzX43Y8= +go.opentelemetry.io/otel v1.39.0 h1:8yPrr/S0ND9QEfTfdP9V+SiwT4E0G7Y5MO7p85nis48= +go.opentelemetry.io/otel v1.39.0/go.mod h1:kLlFTywNWrFyEdH0oj2xK0bFYZtHRYUdv1NklR/tgc8= +go.opentelemetry.io/otel/metric v1.39.0 h1:d1UzonvEZriVfpNKEVmHXbdf909uGTOQjA0HF0Ls5Q0= +go.opentelemetry.io/otel/metric v1.39.0/go.mod h1:jrZSWL33sD7bBxg1xjrqyDjnuzTUB0x1nBERXd7Ftcs= +go.opentelemetry.io/otel/sdk v1.39.0 h1:nMLYcjVsvdui1B/4FRkwjzoRVsMK8uL/cj0OyhKzt18= +go.opentelemetry.io/otel/sdk v1.39.0/go.mod h1:vDojkC4/jsTJsE+kh+LXYQlbL8CgrEcwmt1ENZszdJE= +go.opentelemetry.io/otel/sdk/metric v1.39.0 h1:cXMVVFVgsIf2YL6QkRF4Urbr/aMInf+2WKg+sEJTtB8= +go.opentelemetry.io/otel/sdk/metric v1.39.0/go.mod h1:xq9HEVH7qeX69/JnwEfp6fVq5wosJsY1mt4lLfYdVew= go.opentelemetry.io/otel/trace v1.10.0/go.mod h1:Sij3YYczqAdz+EhmGhE6TpTxUO5/F/AzrK+kxfGqySM= -go.opentelemetry.io/otel/trace v1.31.0 h1:ffjsj1aRouKewfr85U2aGagJ46+MvodynlQ1HYdmJys= -go.opentelemetry.io/otel/trace v1.31.0/go.mod h1:TXZkRk7SM2ZQLtR6eoAWQFIHPvzQ06FJAsO1tJg480A= +go.opentelemetry.io/otel/trace v1.39.0 h1:2d2vfpEDmCJ5zVYz7ijaJdOF59xLomrvj7bjt6/qCJI= +go.opentelemetry.io/otel/trace v1.39.0/go.mod h1:88w4/PnZSazkGzz/w84VHpQafiU4EtqqlVdxWy+rNOA= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk= @@ -183,24 +187,24 @@ golang.org/x/arch v0.3.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q= -golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4= +golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU= +golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.29.0 h1:HV8lRxZC4l2cr3Zq1LvtOsi/ThTgWnUk/y64QSs8GwA= -golang.org/x/mod v0.29.0/go.mod h1:NyhrlYXJ2H4eJiRy/WDBO6HMqZQ6q9nk4JzS3NuCK+w= +golang.org/x/mod v0.30.0 h1:fDEXFVZ/fmCKProc/yAXXUijritrDzahmwwefnjoPFk= +golang.org/x/mod v0.30.0/go.mod h1:lAsf5O2EvJeSFMiBxXDki7sCgAxEUcZHXoXMKT4GJKc= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM= -golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= -golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= +golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU= +golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I= -golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= +golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -212,33 +216,35 @@ golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= -golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk= +golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= -golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= +golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU= +golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.38.0 h1:Hx2Xv8hISq8Lm16jvBZ2VQf+RLmbd7wVUsALibYI/IQ= -golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs= +golang.org/x/tools v0.39.0 h1:ik4ho21kwuQln40uelmciQPp9SipgNDdrafrYA4TmQQ= +golang.org/x/tools v0.39.0/go.mod h1:JnefbkDPyD8UU2kI5fuf8ZX4/yUeh9W877ZeBONxUqQ= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20241015192408-796eee8c2d53 h1:X58yt85/IXCx0Y3ZwN6sEIKZzQtDEYaBWrDvErdXrRE= -google.golang.org/genproto/googleapis/rpc v0.0.0-20241015192408-796eee8c2d53/go.mod h1:GX3210XPVPUjJbTUbvwI8f2IpZDMZuPJWDzDuebbviI= -google.golang.org/grpc v1.69.4 h1:MF5TftSMkd8GLw/m0KM6V8CMOCY6NZ1NQDPGFgbTt4A= -google.golang.org/grpc v1.69.4/go.mod h1:vyjdE6jLBI76dgpDojsFGNaHlxdjXN9ghpnd2o7JGZ4= +gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= +gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= +google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217 h1:gRkg/vSppuSQoDjxyiGfN4Upv/h/DQmIR10ZU8dh4Ww= +google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk= +google.golang.org/grpc v1.79.3 h1:sybAEdRIEtvcD68Gx7dmnwjZKlyfuc61Dyo9pGXXkKE= +google.golang.org/grpc v1.79.3/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA= -google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE= +google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From c0f70e6bf543beeed440daf8d1533f38c2c48ed1 Mon Sep 17 00:00:00 2001 From: r2dedios Date: Wed, 29 Apr 2026 11:46:21 +0200 Subject: [PATCH 31/53] docs(api): fix Swagger response type for account PATCH endpoint --- internal/api/handlers/account_handler.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/api/handlers/account_handler.go b/internal/api/handlers/account_handler.go index 23aba5c0..bdccb103 100644 --- a/internal/api/handlers/account_handler.go +++ b/internal/api/handlers/account_handler.go @@ -294,7 +294,7 @@ func (h *AccountHandler) Delete(c *gin.Context) { // @Produce json // @Param id path string true "Account ID" // @Param account body dto.AccountPatchRequest true "Partial account payload" -// @Success 200 {object} db.AccountDBResponse +// @Success 200 {object} dto.AccountDTOResponse // @Failure 400 {object} responsetypes.GenericErrorResponse // @Failure 404 {object} responsetypes.GenericErrorResponse // @Failure 500 {object} responsetypes.GenericErrorResponse From b97aaa078e0cc963be745c3e05d6819c5cdf354b Mon Sep 17 00:00:00 2001 From: r2dedios Date: Wed, 29 Apr 2026 11:46:28 +0200 Subject: [PATCH 32/53] docs(db): fix godoc formatting for deprecated methods --- internal/db_client/db_client.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/internal/db_client/db_client.go b/internal/db_client/db_client.go index 6cbdaaa1..1a0e4523 100644 --- a/internal/db_client/db_client.go +++ b/internal/db_client/db_client.go @@ -83,6 +83,7 @@ func (d *DBClient) GetWithContext(ctx context.Context, dest interface{}, table s } // Get is a convenience wrapper without context. +// // Deprecated: Prefer GetWithContext for proper timeout and cancellation support. func (d *DBClient) Get(dest interface{}, table string, opts models.ListOptions, columns ...string) error { return d.GetWithContext(context.TODO(), dest, table, opts, columns...) @@ -120,6 +121,7 @@ func (d *DBClient) SelectWithContext(ctx context.Context, dest interface{}, tabl } // Select is a convenience wrapper without context. +// // Deprecated: Prefer SelectWithContext for proper timeout and cancellation support. func (d *DBClient) Select(dest interface{}, table string, opts models.ListOptions, orderColumn string, columns ...string) error { return d.SelectWithContext(context.TODO(), dest, table, opts, orderColumn, columns...) @@ -201,6 +203,7 @@ func (d *DBClient) InsertWithContext(ctx context.Context, query string, data int } // Insert is a convenience wrapper without context. +// // Deprecated: Prefer InsertWithContext for proper timeout and cancellation support. func (d *DBClient) Insert(query string, data interface{}) error { return d.InsertWithContext(context.TODO(), query, data) @@ -237,6 +240,7 @@ func (d *DBClient) UpdateWithContext(ctx context.Context, query string, data int } // Update is a convenience wrapper without context. +// // Deprecated: Prefer UpdateWithContext for proper timeout and cancellation support. func (d *DBClient) Update(query string, data interface{}) error { return d.UpdateWithContext(context.TODO(), query, data) @@ -273,6 +277,7 @@ func (d *DBClient) NamedUpdateWithContext(ctx context.Context, query string, dat } // NamedUpdate is a convenience wrapper without context. +// // Deprecated: Prefer NamedUpdateWithContext for proper timeout and cancellation support. func (d *DBClient) NamedUpdate(query string, data interface{}) error { return d.NamedUpdateWithContext(context.TODO(), query, data) @@ -324,6 +329,7 @@ func (d *DBClient) DeleteWithContext(ctx context.Context, table string, opts mod } // Delete is a convenience wrapper without context. +// // Deprecated: Prefer DeleteWithContext for proper timeout and cancellation support. func (d *DBClient) Delete(table string, opts models.ListOptions) error { return d.DeleteWithContext(context.TODO(), table, opts) From 3106ffce652a566875008473c54f9c2769c161aa Mon Sep 17 00:00:00 2001 From: r2dedios Date: Wed, 29 Apr 2026 11:46:34 +0200 Subject: [PATCH 33/53] refactor(repositories): extract materialized view refresh logic --- internal/repositories/account_repository.go | 48 +++++++++++---------- internal/repositories/cluster_repository.go | 48 +++++++++++---------- 2 files changed, 52 insertions(+), 44 deletions(-) diff --git a/internal/repositories/account_repository.go b/internal/repositories/account_repository.go index c578a6f7..4735f778 100644 --- a/internal/repositories/account_repository.go +++ b/internal/repositories/account_repository.go @@ -191,6 +191,31 @@ func (r *accountRepositoryImpl) CreateAccount(ctx context.Context, accounts []in return nil } +// refreshAccountsMView refreshes the accounts materialized view in a separate transaction. +func (r *accountRepositoryImpl) refreshAccountsMView(ctx context.Context) error { + tx, txErr := r.db.NewTx(ctx) + if txErr != nil { + return fmt.Errorf("failed to create transaction for refresh: %w", txErr) + } + + var err error + defer func() { + if err != nil { + _ = tx.Rollback() + } + }() + + if _, err = tx.ExecContext(ctx, "REFRESH MATERIALIZED VIEW m_accounts_full_view"); err != nil { + return fmt.Errorf("refresh materialized view error: %w", err) + } + + if err = tx.Commit(); err != nil { + return fmt.Errorf("commit refresh error: %w", err) + } + + return nil +} + // UpdateAccount updates mutable fields of an existing account in the database. // Only non-nil fields in the patch request will be updated. func (r *accountRepositoryImpl) UpdateAccount(ctx context.Context, accountID string, patch dto.AccountPatchRequest) (err error) { @@ -235,28 +260,7 @@ func (r *accountRepositoryImpl) UpdateAccount(ctx context.Context, accountID str } // Refresh materialized view after transaction commits - // Note: REFRESH MATERIALIZED VIEW must run outside the transaction - tx2, tx2Err := r.db.NewTx(ctx) - if tx2Err != nil { - return fmt.Errorf("failed to create transaction for refresh: %w", tx2Err) - } - defer func() { - if err != nil { - _ = tx2.Rollback() - } - }() - - if _, refreshErr := tx2.ExecContext(ctx, "REFRESH MATERIALIZED VIEW m_accounts_full_view"); refreshErr != nil { - err = fmt.Errorf("refresh materialized view error: %w", refreshErr) - return err - } - - err = tx2.Commit() - if err != nil { - return fmt.Errorf("commit refresh error: %w", err) - } - - return nil + return r.refreshAccountsMView(ctx) } // DeleteAccount deletes an account from the database by its ID. diff --git a/internal/repositories/cluster_repository.go b/internal/repositories/cluster_repository.go index b7da9037..e2fd1cb6 100644 --- a/internal/repositories/cluster_repository.go +++ b/internal/repositories/cluster_repository.go @@ -295,6 +295,31 @@ func (r *clusterRepositoryImpl) CreateClusters(ctx context.Context, clusters []i return nil } +// refreshClustersMView refreshes the clusters materialized view in a separate transaction. +func (r *clusterRepositoryImpl) refreshClustersMView(ctx context.Context) error { + tx, txErr := r.db.NewTx(ctx) + if txErr != nil { + return fmt.Errorf("failed to create transaction for refresh: %w", txErr) + } + + var err error + defer func() { + if err != nil { + _ = tx.Rollback() + } + }() + + if _, err = tx.ExecContext(ctx, "REFRESH MATERIALIZED VIEW m_clusters_full_view"); err != nil { + return fmt.Errorf("refresh materialized view error: %w", err) + } + + if err = tx.Commit(); err != nil { + return fmt.Errorf("commit refresh error: %w", err) + } + + return nil +} + // UpdateCluster updates mutable fields of an existing cluster in the database. // Only non-nil fields in the patch request will be updated. func (r *clusterRepositoryImpl) UpdateCluster(ctx context.Context, clusterID string, patch dto.ClusterPatchRequest) (err error) { @@ -347,28 +372,7 @@ func (r *clusterRepositoryImpl) UpdateCluster(ctx context.Context, clusterID str } // Refresh materialized view after transaction commits - // Note: REFRESH MATERIALIZED VIEW must run outside the transaction - tx2, tx2Err := r.db.NewTx(ctx) - if tx2Err != nil { - return fmt.Errorf("failed to create transaction for refresh: %w", tx2Err) - } - defer func() { - if err != nil { - _ = tx2.Rollback() - } - }() - - if _, refreshErr := tx2.ExecContext(ctx, "REFRESH MATERIALIZED VIEW m_clusters_full_view"); refreshErr != nil { - err = fmt.Errorf("refresh materialized view error: %w", refreshErr) - return err - } - - err = tx2.Commit() - if err != nil { - return fmt.Errorf("commit refresh error: %w", err) - } - - return nil + return r.refreshClustersMView(ctx) } // UpdateClusterStatusByClusterID updates the status of a cluster and all its instances in the database. From 2211c58d93d77f2b1e4d3bec1f40702488ea8811 Mon Sep 17 00:00:00 2001 From: r2dedios Date: Wed, 29 Apr 2026 13:33:05 +0200 Subject: [PATCH 34/53] fix(repositories): use correct query for terminated instances refresh --- internal/repositories/inventory_repository.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/repositories/inventory_repository.go b/internal/repositories/inventory_repository.go index cfedefda..53a22ab8 100644 --- a/internal/repositories/inventory_repository.go +++ b/internal/repositories/inventory_repository.go @@ -32,7 +32,7 @@ func NewInventoryRepository(db *dbclient.DBClient) InventoryRepository { func (r *inventoryRepositoryImpl) Refresh(ctx context.Context) error { // Updating 'Terminated' Instances - if err := r.db.ExecFunc(ctx, UpdateTerminatedClustersQuery); err != nil { + if err := r.db.ExecFunc(ctx, UpdateTerminatedInstancesQuery); err != nil { return err } From a3a7382d0eea1c4100b7cddc78cd4938168313fe Mon Sep 17 00:00:00 2001 From: r2dedios Date: Wed, 29 Apr 2026 13:40:03 +0200 Subject: [PATCH 35/53] fix(api): return HTTP status codes from inventory Refresh handler --- internal/api/handlers/inventory_handler.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/internal/api/handlers/inventory_handler.go b/internal/api/handlers/inventory_handler.go index 15063909..41c62841 100644 --- a/internal/api/handlers/inventory_handler.go +++ b/internal/api/handlers/inventory_handler.go @@ -1,6 +1,9 @@ package handlers import ( + "net/http" + + responsetypes "github.com/RHEcosystemAppEng/cluster-iq/internal/api/response_types" "github.com/RHEcosystemAppEng/cluster-iq/internal/services" "github.com/gin-gonic/gin" "go.uber.org/zap" @@ -23,5 +26,11 @@ func NewInventoryHandler(service services.InventoryService, logger *zap.Logger) func (h *InventoryHandler) Refresh(c *gin.Context) { if err := h.service.Refresh(c); err != nil { h.logger.Error("Error when refreshing Inventory", zap.Error(err)) + c.JSON(http.StatusInternalServerError, responsetypes.GenericErrorResponse{ + Message: "Failed to refresh inventory", + }) + return } + + c.Status(http.StatusOK) } From 73d454ae33d56695538c4e1be570fbb76540e328 Mon Sep 17 00:00:00 2001 From: r2dedios Date: Wed, 29 Apr 2026 14:06:26 +0200 Subject: [PATCH 36/53] fix(api): prevent nil dereferences in action, expense, and scanner handlers --- cmd/scanner/scanner.go | 4 ++- internal/api/handlers/action_handler.go | 40 +++++++++++++++++++++++- internal/api/handlers/expense_handler.go | 10 +++++- 3 files changed, 51 insertions(+), 3 deletions(-) diff --git a/cmd/scanner/scanner.go b/cmd/scanner/scanner.go index c28ca6b1..42b64c68 100644 --- a/cmd/scanner/scanner.go +++ b/cmd/scanner/scanner.go @@ -150,7 +150,9 @@ func (s *Scanner) createStockers() error { zap.String("account_name", account.AccountName), zap.Error(err)) } else { - s.billingStockers = append(s.billingStockers, stocker.NewAWSBillingStocker(account, s.logger, instancesToScan)) + if bs := stocker.NewAWSBillingStocker(account, s.logger, instancesToScan); bs != nil { + s.billingStockers = append(s.billingStockers, bs) + } } } case inventory.GCPProvider: diff --git a/internal/api/handlers/action_handler.go b/internal/api/handlers/action_handler.go index 17b759d7..cb0ad54f 100644 --- a/internal/api/handlers/action_handler.go +++ b/internal/api/handlers/action_handler.go @@ -187,11 +187,26 @@ func (h *ActionHandler) Create(c *gin.Context) { // @Tags Actions // @Param id path string true "Scheduled action ID" // @Success 200 {object} nil +// @Failure 404 {object} responsetypes.GenericErrorResponse // @Failure 500 {object} responsetypes.GenericErrorResponse // @Router /actions/{id}/enable [patch] func (h *ActionHandler) Enable(c *gin.Context) { actionID := c.Param("id") + if _, err := h.service.Get(c.Request.Context(), actionID); err != nil { + h.logger.Error("error enabling action", zap.String("action_id", actionID), zap.Error(err)) + if errors.Is(err, repositories.ErrNotFound) { + c.JSON(http.StatusNotFound, responsetypes.GenericErrorResponse{ + Message: "Scheduled action not found", + }) + return + } + c.JSON(http.StatusInternalServerError, responsetypes.GenericErrorResponse{ + Message: "Failed to enable scheduled action", + }) + return + } + if err := h.service.Enable(c.Request.Context(), actionID); err != nil { h.logger.Error("error enabling action", zap.String("action_id", actionID), zap.Error(err)) c.JSON(http.StatusInternalServerError, responsetypes.GenericErrorResponse{ @@ -210,11 +225,26 @@ func (h *ActionHandler) Enable(c *gin.Context) { // @Tags Actions // @Param id path string true "Scheduled action ID" // @Success 200 {object} nil +// @Failure 404 {object} responsetypes.GenericErrorResponse // @Failure 500 {object} responsetypes.GenericErrorResponse // @Router /actions/{id}/disable [patch] func (h *ActionHandler) Disable(c *gin.Context) { actionID := c.Param("id") + if _, err := h.service.Get(c.Request.Context(), actionID); err != nil { + h.logger.Error("error disabling action", zap.String("action_id", actionID), zap.Error(err)) + if errors.Is(err, repositories.ErrNotFound) { + c.JSON(http.StatusNotFound, responsetypes.GenericErrorResponse{ + Message: "Scheduled action not found", + }) + return + } + c.JSON(http.StatusInternalServerError, responsetypes.GenericErrorResponse{ + Message: "Failed to disable scheduled action", + }) + return + } + if err := h.service.Disable(c.Request.Context(), actionID); err != nil { h.logger.Error("error disabling action", zap.String("action_id", actionID), zap.Error(err)) c.JSON(http.StatusInternalServerError, responsetypes.GenericErrorResponse{ @@ -281,7 +311,15 @@ func (h *ActionHandler) Update(c *gin.Context) { return } - if err := h.service.Update(c.Request.Context(), actionDTO.ToModelAction()); err != nil { + action := actionDTO.ToModelAction() + if action == nil { + c.JSON(http.StatusBadRequest, responsetypes.GenericErrorResponse{ + Message: "Unknown action type: " + actionDTO.Type, + }) + return + } + + if err := h.service.Update(c.Request.Context(), action); err != nil { h.logger.Error("error updating action", zap.Error(err)) c.JSON(http.StatusInternalServerError, responsetypes.GenericErrorResponse{ Message: "Failed to update action: " + err.Error(), diff --git a/internal/api/handlers/expense_handler.go b/internal/api/handlers/expense_handler.go index fabf6cf4..ce3d0034 100644 --- a/internal/api/handlers/expense_handler.go +++ b/internal/api/handlers/expense_handler.go @@ -114,7 +114,15 @@ func (h *ExpenseHandler) Create(c *gin.Context) { return } - if err := h.service.Create(c.Request.Context(), *dto.ToInventoryExpenseList(expenseDTOs)); err != nil { + expenses := dto.ToInventoryExpenseList(expenseDTOs) + if expenses == nil { + c.JSON(http.StatusBadRequest, responsetypes.GenericErrorResponse{ + Message: "Failed to convert expense data", + }) + return + } + + if err := h.service.Create(c.Request.Context(), *expenses); err != nil { h.logger.Error("error creating expense", zap.Error(err)) c.JSON(http.StatusInternalServerError, responsetypes.GenericErrorResponse{ Message: "Failed to create expenses: " + err.Error(), From 52e3b86803b716b70f8d00804d2f5aa4cbb4954a Mon Sep 17 00:00:00 2001 From: r2dedios Date: Wed, 29 Apr 2026 14:06:32 +0200 Subject: [PATCH 37/53] fix(repositories): add ErrNoClustersInAccount and fix wrong error message --- internal/repositories/account_repository.go | 11 ++++++++++- internal/repositories/errors.go | 3 +++ internal/repositories/instance_repository.go | 2 +- 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/internal/repositories/account_repository.go b/internal/repositories/account_repository.go index 4735f778..e3e221d9 100644 --- a/internal/repositories/account_repository.go +++ b/internal/repositories/account_repository.go @@ -132,6 +132,10 @@ func (r *accountRepositoryImpl) GetAccountByID(ctx context.Context, accountID st // - A slice of inventory.Account objects (usually containing one element). // - An error if the query fails. func (r *accountRepositoryImpl) GetAccountClustersByID(ctx context.Context, accountID string) ([]db.ClusterDBResponse, error) { + if _, err := r.GetAccountByID(ctx, accountID); err != nil { + return nil, err + } + clusters := []db.ClusterDBResponse{} opts := models.ListOptions{ @@ -144,10 +148,15 @@ func (r *accountRepositoryImpl) GetAccountClustersByID(ctx context.Context, acco if err := r.db.SelectWithContext(ctx, &clusters, SelectClustersFullMView, opts, "*"); err != nil { if errors.Is(err, sql.ErrNoRows) { - return clusters, ErrNotFound + return clusters, nil } return clusters, err } + + if len(clusters) == 0 { + return clusters, ErrNoClustersInAccount + } + return clusters, nil } diff --git a/internal/repositories/errors.go b/internal/repositories/errors.go index 8eb3555c..5e014292 100644 --- a/internal/repositories/errors.go +++ b/internal/repositories/errors.go @@ -4,3 +4,6 @@ import "errors" // ErrNotFound is returned when a resource is not found in the database. var ErrNotFound = errors.New("requested resource not found") + +// ErrNoClustersInAccount is returned when an account exists but has no associated clusters. +var ErrNoClustersInAccount = errors.New("no clusters found for this account") diff --git a/internal/repositories/instance_repository.go b/internal/repositories/instance_repository.go index 2aa5a911..14570d85 100644 --- a/internal/repositories/instance_repository.go +++ b/internal/repositories/instance_repository.go @@ -144,7 +144,7 @@ func (r *instanceRepositoryImpl) GetInstancesOverview(ctx context.Context) (inve "COUNT(CASE WHEN status = 'Stopped' THEN 1 END) AS stopped", "COUNT(CASE WHEN status = 'Terminated' THEN 1 END) AS archived", ); err != nil { - return countsDB, fmt.Errorf("failed to list clusters: %w", err) + return countsDB, fmt.Errorf("failed to list instances: %w", err) } return countsDB, nil From 7df56de06255f8bebfd8a75e93411452671ce568 Mon Sep 17 00:00:00 2001 From: r2dedios Date: Wed, 29 Apr 2026 14:06:37 +0200 Subject: [PATCH 38/53] fix(dto): add requester/description fields to action DTOs and fix JSON tag --- internal/models/dto/action_dto.go | 74 +++++++++++++++-------------- internal/models/dto/instance_dto.go | 2 +- 2 files changed, 40 insertions(+), 36 deletions(-) diff --git a/internal/models/dto/action_dto.go b/internal/models/dto/action_dto.go index 553b2563..37f4b770 100644 --- a/internal/models/dto/action_dto.go +++ b/internal/models/dto/action_dto.go @@ -9,17 +9,19 @@ import ( // ActionDTORequest represents the data needed to create or update an action. type ActionDTORequest struct { - ID string `json:"id"` - Type string `json:"type"` - Time time.Time `json:"time"` - CronExp string `json:"cronExpression"` - Operation string `json:"operation"` - Status string `json:"status"` - Enabled bool `json:"enabled"` - ClusterID string `json:"clusterId"` - Region string `json:"region"` - AccountID string `json:"accountId"` - Instances []string `json:"instances"` + ID string `json:"id"` + Type string `json:"type"` + Time time.Time `json:"time"` + CronExp string `json:"cronExpression"` + Operation string `json:"operation"` + Status string `json:"status"` + Enabled bool `json:"enabled"` + ClusterID string `json:"clusterId"` + Region string `json:"region"` + AccountID string `json:"accountId"` + Instances []string `json:"instances"` + Requester string `json:"requester"` + Description *string `json:"description"` } // @name ActionRequest func (a ActionDTORequest) ToModelAction() actions.Action { @@ -36,8 +38,8 @@ func (a ActionDTORequest) ToModelAction() actions.Action { actions.ActionOperation(a.Operation), target, actions.ActionStatus(a.Status), - "", // TODO: Requester missing??? - nil, // TODO: Description missing??? + a.Requester, + a.Description, a.Enabled, a.Time, ) @@ -48,8 +50,8 @@ func (a ActionDTORequest) ToModelAction() actions.Action { actions.ActionOperation(a.Operation), target, actions.ActionStatus(a.Status), - "", // TODO: Requester missing??? - nil, // TODO: Description missing??? + a.Requester, + a.Description, a.Enabled, a.CronExp, ) @@ -60,8 +62,8 @@ func (a ActionDTORequest) ToModelAction() actions.Action { actions.ActionOperation(a.Operation), target, actions.ActionStatus(a.Status), - "", // TODO: Requester missing??? - nil, // TODO: Description missing??? + a.Requester, + a.Description, a.Enabled, ) action.ID = a.ID @@ -87,17 +89,19 @@ func ToModelActionList(dtos []ActionDTORequest) *[]actions.Action { // ActionDTOResponse represents the data transfer object for an action response, // containing action details including schedule, cron expression, and target resources. type ActionDTOResponse struct { - ID string `json:"id"` - Type string `json:"type"` - Time time.Time `json:"time"` - CronExp string `json:"cronExpression"` - Operation string `json:"operation"` - Status string `json:"status"` - Enabled bool `json:"enabled"` - ClusterID string `json:"clusterId"` - Region string `json:"region"` - AccountID string `json:"accountId"` - Instances []string `json:"instances"` + ID string `json:"id"` + Type string `json:"type"` + Time time.Time `json:"time"` + CronExp string `json:"cronExpression"` + Operation string `json:"operation"` + Status string `json:"status"` + Enabled bool `json:"enabled"` + ClusterID string `json:"clusterId"` + Region string `json:"region"` + AccountID string `json:"accountId"` + Instances []string `json:"instances"` + Requester string `json:"requester"` + Description *string `json:"description"` } // @name ActionResponse // ToModelAction converts ActionDTOResponse to actions.Action @@ -115,8 +119,8 @@ func (a ActionDTOResponse) ToModelAction() actions.Action { actions.ActionOperation(a.Operation), target, actions.ActionStatus(a.Status), - "", // TODO: Requester missing??? - nil, // TODO: Description missing??? + a.Requester, + a.Description, a.Enabled, a.Time, ) @@ -127,8 +131,8 @@ func (a ActionDTOResponse) ToModelAction() actions.Action { actions.ActionOperation(a.Operation), target, actions.ActionStatus(a.Status), - "", // TODO: Requester missing??? - nil, // TODO: Description missing??? + a.Requester, + a.Description, a.Enabled, a.CronExp, ) @@ -139,8 +143,8 @@ func (a ActionDTOResponse) ToModelAction() actions.Action { actions.ActionOperation(a.Operation), target, actions.ActionStatus(a.Status), - "", // TODO: Requester missing??? - nil, // TODO: Description missing??? + a.Requester, + a.Description, a.Enabled, ) action.ID = a.ID @@ -150,7 +154,7 @@ func (a ActionDTOResponse) ToModelAction() actions.Action { } } -// ToModelActionList converts a slice of ActionDTOResponse to a slice of actions.Action +// ToModelActionListFromResponse converts a slice of ActionDTOResponse to a slice of actions.Action func ToModelActionListFromResponse(dtos []ActionDTOResponse) ([]actions.Action, error) { resultActions := make([]actions.Action, 0, len(dtos)) for _, dto := range dtos { diff --git a/internal/models/dto/instance_dto.go b/internal/models/dto/instance_dto.go index f9ab0092..db14a09b 100644 --- a/internal/models/dto/instance_dto.go +++ b/internal/models/dto/instance_dto.go @@ -96,7 +96,7 @@ type InstanceDTOResponse struct { ClusterID string `json:"clusterId"` ClusterName string `json:"clusterName"` LastScanTimestamp time.Time `json:"lastScanTimestamp"` - CreatedAt time.Time `json:"creationTimestamp"` + CreatedAt time.Time `json:"createdAt"` Age int `json:"age"` TotalCost float64 `json:"totalCost"` Last15DaysCost float64 `json:"last15DaysCost"` From 2fe2124bc492447baf5043d49c4ccdc3187dc083 Mon Sep 17 00:00:00 2001 From: r2dedios Date: Wed, 29 Apr 2026 14:08:13 +0200 Subject: [PATCH 39/53] fix(clients): propagate context in ProcessInstantAction --- internal/clients/agent.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/clients/agent.go b/internal/clients/agent.go index 590902f3..769635e5 100644 --- a/internal/clients/agent.go +++ b/internal/clients/agent.go @@ -47,15 +47,15 @@ func NewAPIGRPCClient(agentURL string, logger *zap.Logger) (*APIGRPCClient, erro }, nil } -func (a APIGRPCClient) ProcessInstantAction(action *actions.InstantAction) error { +func (a APIGRPCClient) ProcessInstantAction(ctx context.Context, action *actions.InstantAction) error { if action.GetDescription() == nil { action.Description = &DefaultInstantActionDescription } switch action.Operation { case actions.PowerOff: - return a.PowerOffCluster(context.Background(), action) + return a.PowerOffCluster(ctx, action) case actions.PowerOn: - return a.PowerOnCluster(context.Background(), action) + return a.PowerOnCluster(ctx, action) default: return fmt.Errorf("received InstantAction with unknown Operation") } From d5f2467042f67513cdd7e336ca420b2be4cff94a Mon Sep 17 00:00:00 2001 From: r2dedios Date: Wed, 29 Apr 2026 14:22:37 +0200 Subject: [PATCH 40/53] docs(services): fix duplicate comment on GetInstances method --- internal/services/cluster_service.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/services/cluster_service.go b/internal/services/cluster_service.go index ec7e0ec1..e292de79 100644 --- a/internal/services/cluster_service.go +++ b/internal/services/cluster_service.go @@ -59,7 +59,7 @@ func (s *clusterServiceImpl) Get(ctx context.Context, clusterID string) (*db.Clu return s.repo.GetClusterByID(ctx, clusterID) } -// Get retrieves a single cluster by its ID. +// GetInstances retrieves all instances belonging to a cluster. func (s *clusterServiceImpl) GetInstances(ctx context.Context, clusterID string) ([]db.InstanceDBResponse, error) { return s.repo.GetInstancesOnCluster(ctx, clusterID) } From d21541fc19ce32b45c6d1d42f763295debdb8c6e Mon Sep 17 00:00:00 2001 From: r2dedios Date: Wed, 29 Apr 2026 14:22:42 +0200 Subject: [PATCH 41/53] fix(repositories): use conditional rollback in action Create transaction --- internal/repositories/action_repository.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/internal/repositories/action_repository.go b/internal/repositories/action_repository.go index 0efd533e..d79771aa 100644 --- a/internal/repositories/action_repository.go +++ b/internal/repositories/action_repository.go @@ -194,14 +194,18 @@ func (r *actionRepositoryImpl) GetByID(ctx context.Context, actionID string) (db // - An error if the insert fails // // TODO: Temporal fix returning TX from DBClient to manage both insertions in the same sql transaction -func (r *actionRepositoryImpl) Create(ctx context.Context, newActions []actions.Action) error { +func (r *actionRepositoryImpl) Create(ctx context.Context, newActions []actions.Action) (err error) { schedActions, cronActions := actions.SplitActionsByType(newActions) tx, err := r.db.NewTx(ctx) if err != nil { return fmt.Errorf("failed to begin transaction: %w", err) } - defer func() { _ = tx.Rollback() }() + defer func() { + if err != nil { + _ = tx.Rollback() + } + }() // Writing Scheduled Actions if len(schedActions) > 0 { From fcbfc362ec7aca51a8db580a0f4e27c6a4fc2b4b Mon Sep 17 00:00:00 2001 From: r2dedios Date: Wed, 29 Apr 2026 14:22:48 +0200 Subject: [PATCH 42/53] fix(services): wrap repository errors with service-layer context --- internal/services/account_service.go | 31 +++++++++++++++++-------- internal/services/action_service.go | 32 +++++++++++++++++++++----- internal/services/event_service.go | 16 +++++++++---- internal/services/expense_service.go | 12 ++++++++-- internal/services/instance_service.go | 18 ++++++++++++--- internal/services/inventory_service.go | 6 ++++- 6 files changed, 90 insertions(+), 25 deletions(-) diff --git a/internal/services/account_service.go b/internal/services/account_service.go index cec66588..4066da40 100644 --- a/internal/services/account_service.go +++ b/internal/services/account_service.go @@ -2,6 +2,7 @@ package services import ( "context" + "fmt" "github.com/RHEcosystemAppEng/cluster-iq/internal/inventory" "github.com/RHEcosystemAppEng/cluster-iq/internal/models" @@ -46,21 +47,30 @@ func (s *accountServiceImpl) GetByID(ctx context.Context, accountID string) (db. return s.repo.GetAccountByID(ctx, accountID) } -// GetAccountClustersByID retrieves a single account by its name. -// It returns an error if no account or more than one account is found. +// GetAccountClustersByID retrieves the clusters belonging to an account. func (s *accountServiceImpl) GetAccountClustersByID(ctx context.Context, accountID string) ([]db.ClusterDBResponse, error) { - return s.repo.GetAccountClustersByID(ctx, accountID) + clusters, err := s.repo.GetAccountClustersByID(ctx, accountID) + if err != nil { + return clusters, fmt.Errorf("get clusters for account %s: %w", accountID, err) + } + return clusters, nil } -// GetExpenseUpdateInstances retrieves a single account by its name. -// It returns an error if no account or more than one account is found. +// GetExpenseUpdateInstances retrieves instances with outdated billing information. func (s *accountServiceImpl) GetExpenseUpdateInstances(ctx context.Context, accountID string) ([]db.InstanceDBResponse, error) { - return s.repo.GetExpenseUpdateInstances(ctx, accountID) + instances, err := s.repo.GetExpenseUpdateInstances(ctx, accountID) + if err != nil { + return instances, fmt.Errorf("get expense update instances for account %s: %w", accountID, err) + } + return instances, nil } // Create creates one or more new accounts. func (s *accountServiceImpl) Create(ctx context.Context, accounts []inventory.Account) error { - return s.repo.CreateAccount(ctx, accounts) + if err := s.repo.CreateAccount(ctx, accounts); err != nil { + return fmt.Errorf("create accounts: %w", err) + } + return nil } // Update updates mutable fields of an existing account. @@ -74,7 +84,10 @@ func (s *accountServiceImpl) Update(ctx context.Context, accountID string, patch return s.repo.UpdateAccount(ctx, accountID, patch) } -// Delete removes an account by its name. +// Delete removes an account by its ID. func (s *accountServiceImpl) Delete(ctx context.Context, accountID string) error { - return s.repo.DeleteAccount(ctx, accountID) + if err := s.repo.DeleteAccount(ctx, accountID); err != nil { + return fmt.Errorf("delete account %s: %w", accountID, err) + } + return nil } diff --git a/internal/services/action_service.go b/internal/services/action_service.go index 077b5a0f..1d9ad5c9 100644 --- a/internal/services/action_service.go +++ b/internal/services/action_service.go @@ -2,6 +2,7 @@ package services import ( "context" + "fmt" "github.com/RHEcosystemAppEng/cluster-iq/internal/actions" "github.com/RHEcosystemAppEng/cluster-iq/internal/models" @@ -40,30 +41,49 @@ func (s *actionServiceImpl) List(ctx context.Context, options models.ListOptions // Get retrieves a single scheduled action by its ID. func (s *actionServiceImpl) Get(ctx context.Context, actionID string) (db.ActionDBResponse, error) { - return s.repo.GetByID(ctx, actionID) + action, err := s.repo.GetByID(ctx, actionID) + if err != nil { + return action, fmt.Errorf("get action %s: %w", actionID, err) + } + return action, nil } // Create creates new scheduled actions. func (s *actionServiceImpl) Create(ctx context.Context, newActions []actions.Action) error { - return s.repo.Create(ctx, newActions) + if err := s.repo.Create(ctx, newActions); err != nil { + return fmt.Errorf("create actions: %w", err) + } + return nil } // Enable enables a scheduled action. func (s *actionServiceImpl) Enable(ctx context.Context, actionID string) error { - return s.repo.Enable(ctx, actionID) + if err := s.repo.Enable(ctx, actionID); err != nil { + return fmt.Errorf("enable action %s: %w", actionID, err) + } + return nil } // Disable disables a scheduled action. func (s *actionServiceImpl) Disable(ctx context.Context, actionID string) error { - return s.repo.Disable(ctx, actionID) + if err := s.repo.Disable(ctx, actionID); err != nil { + return fmt.Errorf("disable action %s: %w", actionID, err) + } + return nil } // Delete removes a scheduled action by its ID. func (s *actionServiceImpl) Delete(ctx context.Context, actionID string) error { - return s.repo.Delete(ctx, actionID) + if err := s.repo.Delete(ctx, actionID); err != nil { + return fmt.Errorf("delete action %s: %w", actionID, err) + } + return nil } // Update updates an action. func (s *actionServiceImpl) Update(ctx context.Context, action actions.Action) error { - return s.repo.Update(ctx, action) + if err := s.repo.Update(ctx, action); err != nil { + return fmt.Errorf("update action %s: %w", action.GetID(), err) + } + return nil } diff --git a/internal/services/event_service.go b/internal/services/event_service.go index c11d95ba..c593506c 100644 --- a/internal/services/event_service.go +++ b/internal/services/event_service.go @@ -2,6 +2,7 @@ package services import ( "context" + "fmt" "github.com/RHEcosystemAppEng/cluster-iq/internal/events" "github.com/RHEcosystemAppEng/cluster-iq/internal/models" @@ -40,12 +41,19 @@ func (s *eventServiceImpl) ListClusterEvents(ctx context.Context, opts models.Li return s.repo.ListClusterEvents(ctx, opts) } -// Add creates a new audit event. +// Create creates a new audit event. func (s *eventServiceImpl) Create(ctx context.Context, event events.Event) (int64, error) { - return s.repo.CreateEvent(ctx, event) + id, err := s.repo.CreateEvent(ctx, event) + if err != nil { + return id, fmt.Errorf("create event: %w", err) + } + return id, nil } -// UpdateStatus updates the status of an existing audit event. +// Update updates the result of an existing audit event. func (s *eventServiceImpl) Update(ctx context.Context, eventID int64, result string) error { - return s.repo.UpdateEventStatus(ctx, eventID, result) + if err := s.repo.UpdateEventStatus(ctx, eventID, result); err != nil { + return fmt.Errorf("update event %d: %w", eventID, err) + } + return nil } diff --git a/internal/services/expense_service.go b/internal/services/expense_service.go index 49996219..1420a779 100644 --- a/internal/services/expense_service.go +++ b/internal/services/expense_service.go @@ -4,6 +4,7 @@ package services import ( "context" + "fmt" "github.com/RHEcosystemAppEng/cluster-iq/internal/inventory" "github.com/RHEcosystemAppEng/cluster-iq/internal/models" @@ -39,10 +40,17 @@ func (s *expenseServiceImpl) List(ctx context.Context, options models.ListOption // GetByInstanceID retrieves all expenses associated with a specific instance. func (s *expenseServiceImpl) GetByInstanceID(ctx context.Context, instanceID string) ([]db.ExpenseDBResponse, error) { - return s.repo.GetExpensesByInstance(ctx, instanceID) + expenses, err := s.repo.GetExpensesByInstance(ctx, instanceID) + if err != nil { + return expenses, fmt.Errorf("get expenses for instance %s: %w", instanceID, err) + } + return expenses, nil } // Create creates new expense records. func (s *expenseServiceImpl) Create(ctx context.Context, expenses []inventory.Expense) error { - return s.repo.Create(ctx, expenses) + if err := s.repo.Create(ctx, expenses); err != nil { + return fmt.Errorf("create expenses: %w", err) + } + return nil } diff --git a/internal/services/instance_service.go b/internal/services/instance_service.go index 7dddea36..af735327 100644 --- a/internal/services/instance_service.go +++ b/internal/services/instance_service.go @@ -2,6 +2,7 @@ package services import ( "context" + "fmt" "github.com/RHEcosystemAppEng/cluster-iq/internal/inventory" "github.com/RHEcosystemAppEng/cluster-iq/internal/models" @@ -38,15 +39,26 @@ func (s *instanceServiceImpl) List(ctx context.Context, opts models.ListOptions) // Get retrieves a single instance by its ID. func (s *instanceServiceImpl) Get(ctx context.Context, instanceID string) (db.InstanceDBResponse, error) { - return s.repo.GetInstanceByID(ctx, instanceID) + instance, err := s.repo.GetInstanceByID(ctx, instanceID) + if err != nil { + return instance, fmt.Errorf("get instance %s: %w", instanceID, err) + } + return instance, nil } // GetSummary retrieves a summary of instance counts. func (s *instanceServiceImpl) GetSummary(ctx context.Context) (inventory.InstancesSummary, error) { - return s.repo.GetInstancesOverview(ctx) + summary, err := s.repo.GetInstancesOverview(ctx) + if err != nil { + return summary, fmt.Errorf("get instances summary: %w", err) + } + return summary, nil } // Create creates new instances. func (s *instanceServiceImpl) Create(ctx context.Context, instances []inventory.Instance) error { - return s.repo.CreateInstances(ctx, instances) + if err := s.repo.CreateInstances(ctx, instances); err != nil { + return fmt.Errorf("create instances: %w", err) + } + return nil } diff --git a/internal/services/inventory_service.go b/internal/services/inventory_service.go index 53671176..35b8e379 100644 --- a/internal/services/inventory_service.go +++ b/internal/services/inventory_service.go @@ -2,6 +2,7 @@ package services import ( "context" + "fmt" "github.com/RHEcosystemAppEng/cluster-iq/internal/repositories" ) @@ -27,5 +28,8 @@ func NewInventoryService(repo repositories.InventoryRepository) InventoryService // Refresh updates the views and inventory results func (s *inventoryServiceImpl) Refresh(ctx context.Context) error { - return s.repo.Refresh(ctx) + if err := s.repo.Refresh(ctx); err != nil { + return fmt.Errorf("refresh inventory: %w", err) + } + return nil } From 85eef3ce5fa60930d0321e591d5f5eb381e54cd3 Mon Sep 17 00:00:00 2001 From: r2dedios Date: Wed, 29 Apr 2026 14:23:20 +0200 Subject: [PATCH 43/53] fix(aws): guard against nil ResourceTagSet in Route53 zone discovery --- internal/cloud_providers/aws/aws-route53.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/cloud_providers/aws/aws-route53.go b/internal/cloud_providers/aws/aws-route53.go index 1ffc8d10..3dcf909e 100644 --- a/internal/cloud_providers/aws/aws-route53.go +++ b/internal/cloud_providers/aws/aws-route53.go @@ -50,7 +50,7 @@ func (c *AWSRoute53Connection) GetZonesWithTags() ([]HostedZone, error) { ResourceType: &hztype, ResourceId: aws.String(*zone.Id), }) - if err != nil { + if err != nil || tags.ResourceTagSet == nil { continue } zonesWithTags = append(zonesWithTags, HostedZone{ From 078d4726171382b60aa27ca2b51adaef0719e885 Mon Sep 17 00:00:00 2001 From: r2dedios Date: Wed, 29 Apr 2026 14:23:50 +0200 Subject: [PATCH 44/53] docs(api): fix Swagger annotations for action Update and inventory Refresh --- internal/api/handlers/action_handler.go | 8 ++++---- internal/api/handlers/inventory_handler.go | 8 ++++++++ 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/internal/api/handlers/action_handler.go b/internal/api/handlers/action_handler.go index cb0ad54f..72157afd 100644 --- a/internal/api/handlers/action_handler.go +++ b/internal/api/handlers/action_handler.go @@ -288,15 +288,15 @@ func (h *ActionHandler) Delete(c *gin.Context) { c.Status(http.StatusNoContent) } -// Update applies partial updates to an existing actions. +// Update applies partial updates to an existing action. // -// @Summary Update an actions -// @Description Patch an existing actions by ID. +// @Summary Update an action +// @Description Patch an existing action. // @Tags Actions // @Accept json // @Produce json // @Param action body dto.ActionDTORequest true "Partial action payload" -// @Success 200 {object} nil +// @Success 200 {object} responsetypes.PostResponse // @Failure 400 {object} responsetypes.GenericErrorResponse // @Failure 500 {object} responsetypes.GenericErrorResponse // @Router /actions [patch] diff --git a/internal/api/handlers/inventory_handler.go b/internal/api/handlers/inventory_handler.go index 41c62841..b75f12e0 100644 --- a/internal/api/handlers/inventory_handler.go +++ b/internal/api/handlers/inventory_handler.go @@ -23,6 +23,14 @@ func NewInventoryHandler(service services.InventoryService, logger *zap.Logger) } } +// Refresh triggers a refresh of materialized views and terminated resource status. +// +// @Summary Refresh inventory +// @Description Refresh materialized views and update terminated resource status. +// @Tags Inventory +// @Success 200 +// @Failure 500 {object} responsetypes.GenericErrorResponse +// @Router /inventory [post] func (h *InventoryHandler) Refresh(c *gin.Context) { if err := h.service.Refresh(c); err != nil { h.logger.Error("Error when refreshing Inventory", zap.Error(err)) From b0a1e6cc96f55a64d8aff4156567f590dc42395a Mon Sep 17 00:00:00 2001 From: r2dedios Date: Wed, 29 Apr 2026 14:24:01 +0200 Subject: [PATCH 45/53] docs: fix broken link, duplicate env var, and update CLAUDE.md --- CLAUDE.md | 9 +++++---- README.md | 5 ++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index f22edcac..c2558487 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -23,7 +23,7 @@ internal/ ├── services/ # Business logic ├── api/handlers/ # HTTP handlers └── models/ # DTO, DB, domain models -db/sql/ # Schemas and migrations +db/sql/ # Schema definitions (init.sql, cron.sql) test/integration/ # Integration tests ``` @@ -60,7 +60,8 @@ make swagger-doc # OpenAPI docs - Handlers → Services → Repositories → Database - Each layer has interfaces for testability - Repository returns `repositories.ErrNotFound` for missing resources -- Services validate business rules, wrap errors with context +- Repository returns `repositories.ErrNoClustersInAccount` when an account exists but has no clusters +- Services wrap all errors with `fmt.Errorf` context before returning to handlers - Handlers map errors to HTTP status codes (404, 400, 500) **Key Patterns:** @@ -112,7 +113,7 @@ go tool cover -html=coverage.out -o coverage.html # Visual 2. Run `make lint-staged` before committing 3. Run relevant tests: `make go-unit-tests` 4. For API changes: update Swagger with `make swagger-doc` -5. For DB changes: create migration in `db/sql/migrations/` +5. For DB changes: update `db/sql/init.sql` or add data migration in `doc/releases/` 6. For protobuf changes: `make local-build-agent` 7. For goverter changes: `make generate-converters` @@ -147,7 +148,7 @@ go tool cover -html=coverage.out -o coverage.html # Visual Environment variables: - `CIQ_AGENT_URL` - Agent gRPC endpoint (default: "agent:50051") -- `CIQ_API_URL` - API public endpoint +- `CIQ_API_URL` - API endpoint used by Scanner and Agent (not by the API server itself) - `CIQ_DB_URL` - PostgreSQL connection (default: "postgresql://pgsql:5432/clusteriq") - `CIQ_CREDS_FILE` - Cloud provider credentials file - `CIQ_LOG_LEVEL` - Log verbosity (default: "INFO") diff --git a/README.md b/README.md index 5e5da8fc..7221a035 100644 --- a/README.md +++ b/README.md @@ -157,7 +157,7 @@ For deploying ClusterIQ in local for development purposes, check the following ## DB Backup For backing up or restoring the ClusterIQ database, check the following -[document](./doc/db-backup.md) +[document](./doc/developers/db-backup.md) This document also describes how to manage data migration when a new release of ClusterIQ changes DB data structure. @@ -170,8 +170,7 @@ Available configuration via Env Vars: | CIQ_AGENT_POLLING_SECONDS_INTERVAL | integer (Default: 30) | ClusterIQ Agent polling time (seconds) | | CIQ_AGENT_URL | string (Default: "agent:50051") | ClusterIQ Agent listen URL | | CIQ_API_LISTEN_URL | string (Default: "0.0.0.0:8080") | ClusterIQ API listen URL | -| CIQ_API_URL | string (Default: "") | ClusterIQ API public endpoint | -| CIQ_AGENT_LISTEN_URL | string (Default: "0.0.0.0:50051") | ClusterIQ Agent listen URL | +| CIQ_API_URL | string (Default: "") | ClusterIQ API endpoint (used by Scanner) | | CIQ_DB_URL | string (Default: "postgresql://pgsql:5432/clusteriq") | ClusterIQ DB URL | | CIQ_CREDS_FILE | string (Default: "") | Cloud providers accounts credentials file | | CIQ_LOG_LEVEL | string (Default: "INFO") | ClusterIQ Logs verbosity mode | From 5c41fc85050aee499b18b716c142fb50efc28add Mon Sep 17 00:00:00 2001 From: r2dedios Date: Wed, 29 Apr 2026 16:23:18 +0200 Subject: [PATCH 46/53] fix(db): cascade delete events on cluster/instance removal --- db/sql/init.sql | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/db/sql/init.sql b/db/sql/init.sql index 4fd4293e..19d7ecec 100644 --- a/db/sql/init.sql +++ b/db/sql/init.sql @@ -208,6 +208,33 @@ CREATE INDEX IF NOT EXISTS ix_events_type_id_time ON events (resource_type, reso -- Default expenses partition. The rest of expenses will be created by pg_cron CREATE TABLE events_default PARTITION OF events DEFAULT; +-- Cascade delete events when clusters are deleted +CREATE OR REPLACE FUNCTION delete_cluster_events() +RETURNS TRIGGER AS $$ +BEGIN + DELETE FROM events WHERE resource_type = 'Cluster'::RESOURCE_TYPE AND resource_id = OLD.id; + RETURN OLD; +END; +$$ LANGUAGE plpgsql; + +CREATE TRIGGER trg_delete_cluster_events + BEFORE DELETE ON clusters + FOR EACH ROW + EXECUTE FUNCTION delete_cluster_events(); + +-- Cascade delete events when instances are deleted +CREATE OR REPLACE FUNCTION delete_instance_events() +RETURNS TRIGGER AS $$ +BEGIN + DELETE FROM events WHERE resource_type = 'Instance'::RESOURCE_TYPE AND resource_id = OLD.id; + RETURN OLD; +END; +$$ LANGUAGE plpgsql; + +CREATE TRIGGER trg_delete_instance_events + BEFORE DELETE ON instances + FOR EACH ROW + EXECUTE FUNCTION delete_instance_events(); -- ############################################################################# From 7b22fa44779bad3a6f3f48e4aba0694f6eab3e80 Mon Sep 17 00:00:00 2001 From: r2dedios Date: Wed, 29 Apr 2026 16:23:24 +0200 Subject: [PATCH 47/53] fix(events): handle nullable resource_id from event views --- db/test_files/load_example_data.sql | 2 +- internal/models/convert/generated.go | 9 ++++++--- internal/models/convert/mapper.go | 1 + internal/models/db/events.go | 2 +- internal/models/db/events_test.go | 23 +++++++++++++++-------- internal/models/dto/event_dto.go | 2 +- internal/models/dto/event_dto_test.go | 10 ++++++---- 7 files changed, 31 insertions(+), 18 deletions(-) diff --git a/db/test_files/load_example_data.sql b/db/test_files/load_example_data.sql index 33498c64..7367fd47 100644 --- a/db/test_files/load_example_data.sql +++ b/db/test_files/load_example_data.sql @@ -243,7 +243,7 @@ SELECT (ARRAY['scanner','agent','api','scheduler','user'])[1 + floor(random()*5)], (ARRAY['scan','PowerOn','PowerOff','RestartCluster','Terminate'])[1 + floor(random()*5)], (1 + floor(random()*20))::int AS resource_id, - CASE WHEN random() < 0.6 THEN 'instance' ELSE 'cluster' END AS resource_type, + CASE WHEN random() < 0.6 THEN 'Instance'::RESOURCE_TYPE ELSE 'Cluster'::RESOURCE_TYPE END AS resource_type, (ARRAY['Pending','Running','Failed','Success','Unknown'])[1 + floor(random()*5)]::ACTION_STATUS AS result, 'auto-generated dev event' AS description, (ARRAY['info','warning','error','notice'])[1 + floor(random()*4)] AS severity diff --git a/internal/models/convert/generated.go b/internal/models/convert/generated.go index bfdd9bf9..1384b4bf 100644 --- a/internal/models/convert/generated.go +++ b/internal/models/convert/generated.go @@ -97,15 +97,18 @@ func (c *ConverterImpl) ToClusterEventDTO(source db.ClusterEventDBResponse) dto. var dtoClusterEventDTOResponse dto.ClusterEventDTOResponse dtoClusterEventDTOResponse.ID = source.ID dtoClusterEventDTOResponse.Action = source.Action - dtoClusterEventDTOResponse.ResourceID = source.ResourceID + if source.ResourceID != nil { + xstring := *source.ResourceID + dtoClusterEventDTOResponse.ResourceID = &xstring + } dtoClusterEventDTOResponse.ResourceType = source.ResourceType dtoClusterEventDTOResponse.EventTimestamp = Time(source.EventTimestamp) dtoClusterEventDTOResponse.Result = actions.ActionStatus(source.Result) dtoClusterEventDTOResponse.Severity = source.Severity dtoClusterEventDTOResponse.TriggeredBy = source.TriggeredBy if source.Description != nil { - xstring := *source.Description - dtoClusterEventDTOResponse.Description = &xstring + xstring2 := *source.Description + dtoClusterEventDTOResponse.Description = &xstring2 } return dtoClusterEventDTOResponse } diff --git a/internal/models/convert/mapper.go b/internal/models/convert/mapper.go index b6d6e29d..975953a6 100644 --- a/internal/models/convert/mapper.go +++ b/internal/models/convert/mapper.go @@ -45,6 +45,7 @@ type Converter interface { ToSystemEventDTOs(src []db.SystemEventDBResponse) []dto.SystemEventDTOResponse // Action + // goverter:ignore Requester Description ToActionDTO(src db.ActionDBResponse) dto.ActionDTOResponse ToActionDTOs(src []db.ActionDBResponse) []dto.ActionDTOResponse diff --git a/internal/models/db/events.go b/internal/models/db/events.go index 6da33ef1..f787924c 100644 --- a/internal/models/db/events.go +++ b/internal/models/db/events.go @@ -13,7 +13,7 @@ type ClusterEventDBResponse struct { EventTimestamp time.Time `db:"event_timestamp"` TriggeredBy string `db:"triggered_by"` Action string `db:"action"` - ResourceID string `db:"resource_id"` + ResourceID *string `db:"resource_id"` ResourceType string `db:"resource_type"` Result actions.ActionStatus `db:"result"` Description *string `db:"description,omitempty"` diff --git a/internal/models/db/events_test.go b/internal/models/db/events_test.go index 597a649d..fdd73855 100644 --- a/internal/models/db/events_test.go +++ b/internal/models/db/events_test.go @@ -24,6 +24,7 @@ func TestClusterEventDBResponse_ToClusterEventDTOResponse(t *testing.T) { func testClusterEventDBResponse_ToClusterEventDTOResponse_Correct(t *testing.T) { now := time.Now().UTC() desc := "desc" + resID := "cluster-1" conv := &convert.ConverterImpl{} model := db.ClusterEventDBResponse{ @@ -31,7 +32,7 @@ func testClusterEventDBResponse_ToClusterEventDTOResponse_Correct(t *testing.T) EventTimestamp: now, TriggeredBy: "api", Action: "START", - ResourceID: "cluster-1", + ResourceID: &resID, ResourceType: inventory.ClusterResourceType, Result: "Success", Description: &desc, @@ -53,6 +54,7 @@ func testClusterEventDBResponse_ToClusterEventDTOResponse_Correct(t *testing.T) func testClusterEventDBResponse_ToClusterEventDTOResponse_NilDescription(t *testing.T) { now := time.Now().UTC() + resID := "cluster-2" conv := &convert.ConverterImpl{} model := db.ClusterEventDBResponse{ @@ -60,7 +62,7 @@ func testClusterEventDBResponse_ToClusterEventDTOResponse_NilDescription(t *test EventTimestamp: now, TriggeredBy: "scanner", Action: "STOP", - ResourceID: "cluster-2", + ResourceID: &resID, ResourceType: inventory.ClusterResourceType, Result: "Failed", Description: nil, @@ -84,9 +86,11 @@ func testToClusterEventDTOResponseList_Correct(t *testing.T) { now := time.Now().UTC() conv := &convert.ConverterImpl{} + c1 := "c1" + c2 := "c2" models := []db.ClusterEventDBResponse{ - {ID: 1, EventTimestamp: now, TriggeredBy: "api", Action: "START", ResourceID: "c1", ResourceType: inventory.ClusterResourceType, Result: "Success", Severity: "Info"}, - {ID: 2, EventTimestamp: now.Add(-time.Minute), TriggeredBy: "agent", Action: "STOP", ResourceID: "c2", ResourceType: inventory.ClusterResourceType, Result: "Failed", Severity: "Error"}, + {ID: 1, EventTimestamp: now, TriggeredBy: "api", Action: "START", ResourceID: &c1, ResourceType: inventory.ClusterResourceType, Result: "Success", Severity: "Info"}, + {ID: 2, EventTimestamp: now.Add(-time.Minute), TriggeredBy: "agent", Action: "STOP", ResourceID: &c2, ResourceType: inventory.ClusterResourceType, Result: "Failed", Severity: "Error"}, } dtos := conv.ToClusterEventDTOs(models) @@ -106,6 +110,7 @@ func TestSystemEventDBResponse_ToSystemEventDTOResponse(t *testing.T) { func testSystemEventDBResponse_ToSystemEventDTOResponse_Correct(t *testing.T) { now := time.Now().UTC() desc := "desc" + resID := "cluster-10" conv := &convert.ConverterImpl{} model := db.SystemEventDBResponse{ @@ -114,7 +119,7 @@ func testSystemEventDBResponse_ToSystemEventDTOResponse_Correct(t *testing.T) { EventTimestamp: now, TriggeredBy: "scheduler", Action: "START", - ResourceID: "cluster-10", + ResourceID: &resID, ResourceType: inventory.ClusterResourceType, Result: "Pending", Description: &desc, @@ -130,7 +135,7 @@ func testSystemEventDBResponse_ToSystemEventDTOResponse_Correct(t *testing.T) { assert.Equal(t, now, dto.EventTimestamp) assert.Equal(t, "scheduler", dto.TriggeredBy) assert.Equal(t, "START", dto.Action) - assert.Equal(t, "cluster-10", dto.ResourceID) + assert.Equal(t, &resID, dto.ResourceID) assert.Equal(t, inventory.ClusterResourceType, dto.ResourceType) assert.Equal(t, actions.StatusPending, dto.Result) assert.Equal(t, &desc, dto.Description) @@ -150,6 +155,8 @@ func testToSystemEventDTOResponseList_Correct(t *testing.T) { now := time.Now().UTC() conv := &convert.ConverterImpl{} + sc1 := "c1" + sc2 := "c2" models := []db.SystemEventDBResponse{ { ClusterEventDBResponse: db.ClusterEventDBResponse{ @@ -157,7 +164,7 @@ func testToSystemEventDTOResponseList_Correct(t *testing.T) { EventTimestamp: now, TriggeredBy: "api", Action: "START", - ResourceID: "c1", + ResourceID: &sc1, ResourceType: inventory.ClusterResourceType, Result: "Success", Severity: "Info", @@ -171,7 +178,7 @@ func testToSystemEventDTOResponseList_Correct(t *testing.T) { EventTimestamp: now.Add(-time.Minute), TriggeredBy: "agent", Action: "STOP", - ResourceID: "c2", + ResourceID: &sc2, ResourceType: inventory.ClusterResourceType, Result: "Failed", Severity: "Error", diff --git a/internal/models/dto/event_dto.go b/internal/models/dto/event_dto.go index 0b1d483e..210fd143 100644 --- a/internal/models/dto/event_dto.go +++ b/internal/models/dto/event_dto.go @@ -38,7 +38,7 @@ func (e EventDTORequest) ToModelEvent() *events.Event { type ClusterEventDTOResponse struct { ID int64 `json:"id"` Action string `json:"action"` - ResourceID string `json:"resourceId"` + ResourceID *string `json:"resourceId"` ResourceType string `json:"resourceType"` EventTimestamp time.Time `json:"timestamp"` Result actions.ActionStatus `json:"result"` diff --git a/internal/models/dto/event_dto_test.go b/internal/models/dto/event_dto_test.go index f9a733a2..b6a90d7a 100644 --- a/internal/models/dto/event_dto_test.go +++ b/internal/models/dto/event_dto_test.go @@ -79,12 +79,13 @@ func TestEventDTOResponse_Struct(t *testing.T) { func testClusterEventDTOResponse_Struct(t *testing.T) { desc := "desc" + resID := "cluster-1" now := time.Now() dto := ClusterEventDTOResponse{ ID: 1, Action: "START", - ResourceID: "cluster-1", + ResourceID: &resID, ResourceType: inventory.ClusterResourceType, EventTimestamp: now, Result: "Success", @@ -95,7 +96,7 @@ func testClusterEventDTOResponse_Struct(t *testing.T) { assert.Equal(t, int64(1), dto.ID) assert.Equal(t, "START", dto.Action) - assert.Equal(t, "cluster-1", dto.ResourceID) + assert.Equal(t, &resID, dto.ResourceID) assert.Equal(t, inventory.ClusterResourceType, dto.ResourceType) assert.Equal(t, now, dto.EventTimestamp) assert.Equal(t, actions.StatusSuccess, dto.Result) @@ -106,13 +107,14 @@ func testClusterEventDTOResponse_Struct(t *testing.T) { func testSystemEventDTOResponse_Struct(t *testing.T) { desc := "desc" + resID := "cluster-2" now := time.Now() dto := SystemEventDTOResponse{ ClusterEventDTOResponse: ClusterEventDTOResponse{ ID: 2, Action: "STOP", - ResourceID: "cluster-2", + ResourceID: &resID, ResourceType: inventory.ClusterResourceType, EventTimestamp: now, Result: "Failed", @@ -126,7 +128,7 @@ func testSystemEventDTOResponse_Struct(t *testing.T) { assert.Equal(t, int64(2), dto.ID) assert.Equal(t, "STOP", dto.Action) - assert.Equal(t, "cluster-2", dto.ResourceID) + assert.Equal(t, &resID, dto.ResourceID) assert.Equal(t, inventory.ClusterResourceType, dto.ResourceType) assert.Equal(t, now, dto.EventTimestamp) assert.Equal(t, actions.StatusFailed, dto.Result) From 10e36ea2439a8448609aee9be2bd399629d95bb9 Mon Sep 17 00:00:00 2001 From: r2dedios Date: Wed, 29 Apr 2026 16:24:49 +0200 Subject: [PATCH 48/53] docs(claude): clarify Makefile usage for compilation checks --- CLAUDE.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CLAUDE.md b/CLAUDE.md index c2558487..ddb5d76f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -30,6 +30,7 @@ test/integration/ # Integration tests ## Essential Commands **CRITICAL**: Always use Makefile. Never use direct `go build`, `go test`, or `go run`. +This includes compilation checks after code changes — use `make local-build` instead of `go build ./...`. ```bash # Development From 456671101186672e924bf66130bdbcb9eeb41e46 Mon Sep 17 00:00:00 2001 From: r2dedios Date: Wed, 29 Apr 2026 16:35:20 +0200 Subject: [PATCH 49/53] fix(ci): pin golangci-lint version to match local environment --- .github/workflows/validate-pr.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/validate-pr.yaml b/.github/workflows/validate-pr.yaml index 50880b43..3466a19d 100644 --- a/.github/workflows/validate-pr.yaml +++ b/.github/workflows/validate-pr.yaml @@ -25,6 +25,7 @@ jobs: - name: Run golangci-lint uses: golangci/golangci-lint-action@v8 with: + version: v2.6.2 only-new-issues: true args: --whole-files From f611b2a5e1b82b3b76146f38aaae94dfc2070a85 Mon Sep 17 00:00:00 2001 From: r2dedios Date: Wed, 29 Apr 2026 16:41:32 +0200 Subject: [PATCH 50/53] fix(db): replace NamedQuery with PrepareNamed to ensure resource cleanup --- internal/db_client/db_client.go | 22 +++++----------------- 1 file changed, 5 insertions(+), 17 deletions(-) diff --git a/internal/db_client/db_client.go b/internal/db_client/db_client.go index 1a0e4523..27edb834 100644 --- a/internal/db_client/db_client.go +++ b/internal/db_client/db_client.go @@ -1,5 +1,3 @@ -// TODO: Placeholder for the SQL client to fix linter issues -// TODO: Add actual implementation in next PR package dbclient import ( @@ -144,24 +142,14 @@ func (d *DBClient) InsertWithReturnWithContext(ctx context.Context, query string } }() - rows, err := tx.NamedQuery(builder.query, builder.data) + stmt, err := tx.PrepareNamed(builder.query) if err != nil { - return -1, fmt.Errorf("named-exec INSERT error: %w", err) + return -1, fmt.Errorf("prepare INSERT error: %w", err) } - defer func() { - if cerr := rows.Close(); cerr != nil { - d.logger.Error("failed to close rows after insert") - } - }() + defer stmt.Close() - if rows.Next() { - if scanErr := rows.Scan(&returnedValue); scanErr != nil { - err = fmt.Errorf("scan INSERT return value error %w", scanErr) - return -1, err - } - } else { - err = fmt.Errorf("sql INSERT did not return any value") - return -1, err + if err = stmt.Get(&returnedValue, builder.data); err != nil { + return -1, fmt.Errorf("exec INSERT error: %w", err) } err = tx.Commit() From 781e53fa1c157bf9154cfe06d6f09dd551f2ecd6 Mon Sep 17 00:00:00 2001 From: r2dedios Date: Wed, 29 Apr 2026 16:53:02 +0200 Subject: [PATCH 51/53] test(integration): fix assertions after service error wrapping and terminated instances fix --- db/test_files/integration_tests_data.sql | 28 +++++++++---------- .../api_accounts_integration_test.go | 2 +- .../api_instances_integration_test.go | 2 +- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/db/test_files/integration_tests_data.sql b/db/test_files/integration_tests_data.sql index cbf25640..aaddeaf3 100644 --- a/db/test_files/integration_tests_data.sql +++ b/db/test_files/integration_tests_data.sql @@ -20,20 +20,20 @@ INSERT INTO clusters (cluster_id, cluster_name, infra_id, provider, status, regi -- ## Instances ## INSERT INTO instances (instance_id, instance_name, cluster_id, provider, instance_type, availability_zone, status, last_scan_ts, created_at, age) VALUES - ('id-0123456789X', 'aws-instance-1a', 1, 'AWS', 't3.micro', 'us-east-1a', 'Running', '2025-08-01 12:00:00+00', '2025-02-10 00:00:00+00', 170), - ('id-1123456789X', 'aws-instance-1b', 1, 'AWS', 't3.medium', 'us-east-1b', 'Stopped', '2025-08-01 12:00:00+00', '2025-02-11 00:00:00+00', 169), - ('id-2123456789X', 'aws-instance-2a', 2, 'AWS', 'm6g.large', 'us-east-2a', 'Running', '2025-08-01 12:00:00+00', '2025-03-15 00:00:00+00', 140), - ('id-3123456789X', 'aws-instance-2b', 2, 'AWS', 'c6i.large', 'us-east-2b', 'Unknown', '2025-08-01 12:00:00+00', '2025-03-20 00:00:00+00', 135), - - ('id-0123456789Y', 'gcp-instance-1a', 3, 'GCP', 'e2-small', 'europe-west1-b', 'Running', '2025-08-02 12:00:00+00', '2025-02-05 00:00:00+00', 175), - ('id-1123456789Y', 'gcp-instance-1b', 3, 'GCP', 'e2-medium', 'europe-west1-c', 'Stopped', '2025-08-02 12:00:00+00', '2025-02-06 00:00:00+00', 174), - ('id-2123456789Y', 'gcp-instance-2a', 4, 'GCP', 'n2-standard', 'europe-west2-a','Running', '2025-08-02 12:00:00+00', '2025-03-10 00:00:00+00', 150), - ('id-3123456789Y', 'gcp-instance-2b', 4, 'GCP', 'n2-standard', 'europe-west2-b','Unknown', '2025-08-02 12:00:00+00', '2025-03-11 00:00:00+00', 149), - - ('id-0123456789Z', 'az-instance-1a', 5, 'Azure', 'B1s', 'westeurope-1', 'Running', '2025-08-03 12:00:00+00', '2025-02-01 00:00:00+00', 180), - ('id-1123456789Z', 'az-instance-1b', 5, 'Azure', 'B2s', 'westeurope-2', 'Stopped', '2025-08-03 12:00:00+00', '2025-02-02 00:00:00+00', 179), - ('id-2123456789Z', 'az-instance-2a', 6, 'Azure', 'D2s_v3', 'westeurope-1', 'Running', '2025-08-03 12:00:00+00', '2025-03-05 00:00:00+00', 160), - ('id-3123456789Z', 'az-instance-2b', 6, 'Azure', 'D2s_v3', 'westeurope-2', 'Unknown', '2025-08-03 12:00:00+00', '2025-03-06 00:00:00+00', 159); + ('id-0123456789X', 'aws-instance-1a', 1, 'AWS', 't3.micro', 'us-east-1a', 'Running', NOW(), '2025-02-10 00:00:00+00', 170), + ('id-1123456789X', 'aws-instance-1b', 1, 'AWS', 't3.medium', 'us-east-1b', 'Stopped', NOW(), '2025-02-11 00:00:00+00', 169), + ('id-2123456789X', 'aws-instance-2a', 2, 'AWS', 'm6g.large', 'us-east-2a', 'Running', NOW(), '2025-03-15 00:00:00+00', 140), + ('id-3123456789X', 'aws-instance-2b', 2, 'AWS', 'c6i.large', 'us-east-2b', 'Unknown', NOW(), '2025-03-20 00:00:00+00', 135), + + ('id-0123456789Y', 'gcp-instance-1a', 3, 'GCP', 'e2-small', 'europe-west1-b', 'Running', NOW(), '2025-02-05 00:00:00+00', 175), + ('id-1123456789Y', 'gcp-instance-1b', 3, 'GCP', 'e2-medium', 'europe-west1-c', 'Stopped', NOW(), '2025-02-06 00:00:00+00', 174), + ('id-2123456789Y', 'gcp-instance-2a', 4, 'GCP', 'n2-standard', 'europe-west2-a','Running', NOW(), '2025-03-10 00:00:00+00', 150), + ('id-3123456789Y', 'gcp-instance-2b', 4, 'GCP', 'n2-standard', 'europe-west2-b','Unknown', NOW(), '2025-03-11 00:00:00+00', 149), + + ('id-0123456789Z', 'az-instance-1a', 5, 'Azure', 'B1s', 'westeurope-1', 'Running', NOW(), '2025-02-01 00:00:00+00', 180), + ('id-1123456789Z', 'az-instance-1b', 5, 'Azure', 'B2s', 'westeurope-2', 'Stopped', NOW(), '2025-02-02 00:00:00+00', 179), + ('id-2123456789Z', 'az-instance-2a', 6, 'Azure', 'D2s_v3', 'westeurope-1', 'Running', NOW(), '2025-03-05 00:00:00+00', 160), + ('id-3123456789Z', 'az-instance-2b', 6, 'Azure', 'D2s_v3', 'westeurope-2', 'Unknown', NOW(), '2025-03-06 00:00:00+00', 159); -- ## Instance Tags ## INSERT INTO tags (key, value, instance_id) VALUES diff --git a/test/integration/api_accounts_integration_test.go b/test/integration/api_accounts_integration_test.go index 037e86e7..9b3c86f4 100644 --- a/test/integration/api_accounts_integration_test.go +++ b/test/integration/api_accounts_integration_test.go @@ -343,7 +343,7 @@ func testPostMultipleAccounts(t *testing.T) { func testPostWrongAccount(t *testing.T) { expectedHTTPCode := http.StatusInternalServerError - expectedMsg := "Failed to create accounts: named-exec INSERT error: pq: invalid input value for enum cloud_provider: \"Provider\"" + expectedMsg := "Failed to create accounts: create accounts: named-exec INSERT error: pq: invalid input value for enum cloud_provider: \"Provider\"" ts, _ := time.Parse(time.RFC3339, "2025-08-02T10:00:00+00:00") payload := []dto.AccountDTORequest{ diff --git a/test/integration/api_instances_integration_test.go b/test/integration/api_instances_integration_test.go index b7757cc4..126d878c 100644 --- a/test/integration/api_instances_integration_test.go +++ b/test/integration/api_instances_integration_test.go @@ -451,7 +451,7 @@ func testPostInstancesWithTags(t *testing.T) { func testPostInstancesWrongValues(t *testing.T) { expectedHTTPCode := http.StatusInternalServerError - expectedMsg := "Failed to create instances: named-exec INSERT error: pq: invalid input value for enum cloud_provider: \"PROVIDER\"" + expectedMsg := "Failed to create instances: create instances: named-exec INSERT error: pq: invalid input value for enum cloud_provider: \"PROVIDER\"" payload := []dto.InstanceDTORequest{ { InstanceID: "error-instance", From 8686a71df3afa7fe26ba02171f78e400b84faf4b Mon Sep 17 00:00:00 2001 From: r2dedios Date: Tue, 26 May 2026 11:31:41 +0200 Subject: [PATCH 52/53] fix: address multiple runtime bugs found during release-0.5.1 audit - fix(scanner): correct reversed MD5 hash copy for credential tracking - fix(agent): eliminate data race on shared err variable across goroutines - fix(clients): store gRPC connection for proper lifecycle management - fix(repositories): wrap CreateInstances in single atomic transaction - fix(db): require WHERE clause in DeleteBuilder for safety - fix(aws): guard against nil EC2 tag keys/values with aws.StringValue - fix(repositories): use SELECT * for expense update instances view Co-Authored-By: Claude Opus 4.6 --- cmd/agent/agent.go | 7 +++---- cmd/api/server.go | 5 +++++ cmd/scanner/scanner.go | 2 +- internal/clients/agent.go | 8 +++++++ internal/cloud_providers/aws/aws-tags.go | 3 ++- internal/db_client/delete_builder.go | 8 +++---- internal/repositories/account_repository.go | 2 +- internal/repositories/instance_repository.go | 22 ++++++++++++++------ 8 files changed, 40 insertions(+), 17 deletions(-) diff --git a/cmd/agent/agent.go b/cmd/agent/agent.go index 4e8424da..4adea821 100644 --- a/cmd/agent/agent.go +++ b/cmd/agent/agent.go @@ -106,14 +106,13 @@ func NewAgent(cfg *config.AgentConfig, logger *zap.Logger) (*Agent, error) { // StartAgentServices starts every AgentService on a separate thread(go-routine) func (a *Agent) StartAgentServices() error { - var err error errChan := make(chan error, AgentServicesCount) // Starting InstantAgentService a.wg.Add(1) go func() { defer a.wg.Done() - if err = a.ias.Start(); err != nil { + if err := a.ias.Start(); err != nil { errChan <- fmt.Errorf("instant AgentService (gRPC) failed: %w", err) return } @@ -124,7 +123,7 @@ func (a *Agent) StartAgentServices() error { a.wg.Add(1) go func() { defer a.wg.Done() - if err = a.sas.Start(); err != nil { + if err := a.sas.Start(); err != nil { errChan <- fmt.Errorf("scheduled Agent Service failed: %w", err) return } @@ -135,7 +134,7 @@ func (a *Agent) StartAgentServices() error { a.wg.Add(1) go func() { defer a.wg.Done() - if err = a.eas.Start(); err != nil { + if err := a.eas.Start(); err != nil { errChan <- fmt.Errorf("executor Agent Service failed: %w", err) return } diff --git a/cmd/api/server.go b/cmd/api/server.go index 391e9f8e..e5bd8b3d 100644 --- a/cmd/api/server.go +++ b/cmd/api/server.go @@ -186,6 +186,11 @@ func main() { if err != nil { logger.Fatal("Failed to create gRPC client", zap.Error(err)) } + defer func() { + if err := agentClient.Close(); err != nil { + logger.Error("error closing gRPC client", zap.Error(err)) + } + }() // Initializing repositories inventoryRepo := repositories.NewInventoryRepository(dbClient) diff --git a/cmd/scanner/scanner.go b/cmd/scanner/scanner.go index 42b64c68..e7c5ba16 100644 --- a/cmd/scanner/scanner.go +++ b/cmd/scanner/scanner.go @@ -68,7 +68,7 @@ type Scanner struct { func NewScanner(cfg *config.ScannerConfig, logger *zap.Logger) *Scanner { // Calculate Credentials file MD5 checksum for checking on runtime hash := md5.Sum([]byte(cfg.CredentialsFile)) - copy(hash[:], credsFileHash) + credsFileHash = hash[:] tr := &http.Transport{ TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, diff --git a/internal/clients/agent.go b/internal/clients/agent.go index 769635e5..f0d78593 100644 --- a/internal/clients/agent.go +++ b/internal/clients/agent.go @@ -20,6 +20,8 @@ var ( type APIGRPCClient struct { // Client is the gRPC client used to communicate with the Agent service. Client pb.AgentServiceClient + // conn holds the underlying gRPC connection for lifecycle management. + conn *grpc.ClientConn // logger is used for logging gRPC operations and errors. logger *zap.Logger } @@ -43,10 +45,16 @@ func NewAPIGRPCClient(agentURL string, logger *zap.Logger) (*APIGRPCClient, erro return &APIGRPCClient{ Client: pb.NewAgentServiceClient(conn), + conn: conn, logger: logger, }, nil } +// Close closes the underlying gRPC connection. +func (a *APIGRPCClient) Close() error { + return a.conn.Close() +} + func (a APIGRPCClient) ProcessInstantAction(ctx context.Context, action *actions.InstantAction) error { if action.GetDescription() == nil { action.Description = &DefaultInstantActionDescription diff --git a/internal/cloud_providers/aws/aws-tags.go b/internal/cloud_providers/aws/aws-tags.go index baec6bdf..38490b96 100644 --- a/internal/cloud_providers/aws/aws-tags.go +++ b/internal/cloud_providers/aws/aws-tags.go @@ -2,6 +2,7 @@ package cloudprovider import ( "github.com/RHEcosystemAppEng/cluster-iq/internal/inventory" + "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ec2" ) @@ -9,7 +10,7 @@ import ( func ConvertEC2TagtoTag(ec2Tags []*ec2.Tag, instanceID string) []inventory.Tag { var tags []inventory.Tag for _, tag := range ec2Tags { - tags = append(tags, *inventory.NewTag(*tag.Key, *tag.Value, instanceID)) + tags = append(tags, *inventory.NewTag(aws.StringValue(tag.Key), aws.StringValue(tag.Value), instanceID)) } return tags } diff --git a/internal/db_client/delete_builder.go b/internal/db_client/delete_builder.go index 4a8e0370..b339579d 100644 --- a/internal/db_client/delete_builder.go +++ b/internal/db_client/delete_builder.go @@ -29,11 +29,11 @@ func (d *DeleteBuilder) Build() (string, []interface{}, error) { return "", nil, fmt.Errorf("no table defined for DeleteBuilder") } - query := fmt.Sprintf("DELETE FROM %s", d.table) - - if len(d.where) > 0 { - query += " WHERE " + strings.Join(d.where, " AND ") + if len(d.where) == 0 { + return "", nil, fmt.Errorf("DELETE requires at least one WHERE condition for safety") } + query := fmt.Sprintf("DELETE FROM %s WHERE %s", d.table, strings.Join(d.where, " AND ")) + return query, d.args, nil } diff --git a/internal/repositories/account_repository.go b/internal/repositories/account_repository.go index e3e221d9..8d86995d 100644 --- a/internal/repositories/account_repository.go +++ b/internal/repositories/account_repository.go @@ -178,7 +178,7 @@ func (r *accountRepositoryImpl) GetExpenseUpdateInstances(ctx context.Context, a }, } - if err := r.db.SelectWithContext(ctx, &instances, SelectInstancesPendingExpenseUpdateView, opts, "instance_id", "instance_id"); err != nil { + if err := r.db.SelectWithContext(ctx, &instances, SelectInstancesPendingExpenseUpdateView, opts, "instance_id"); err != nil { return instances, fmt.Errorf("failed to list instances pending of expense update: %w", err) } diff --git a/internal/repositories/instance_repository.go b/internal/repositories/instance_repository.go index 14570d85..3d001c80 100644 --- a/internal/repositories/instance_repository.go +++ b/internal/repositories/instance_repository.go @@ -157,9 +157,19 @@ func (r *instanceRepositoryImpl) GetInstancesOverview(ctx context.Context) (inve // // Returns: // - An error if the transaction fails. -func (r *instanceRepositoryImpl) CreateInstances(ctx context.Context, instances []inventory.Instance) error { - if err := r.db.InsertWithContext(ctx, InsertInstancesQuery, instances); err != nil { - return err +func (r *instanceRepositoryImpl) CreateInstances(ctx context.Context, instances []inventory.Instance) (err error) { + tx, err := r.db.NewTx(ctx) + if err != nil { + return fmt.Errorf("failed to begin transaction: %w", err) + } + defer func() { + if err != nil { + _ = tx.Rollback() + } + }() + + if _, err = tx.NamedExecContext(ctx, InsertInstancesQuery, instances); err != nil { + return fmt.Errorf("failed to insert instances: %w", err) } newTags := []inventory.Tag{} @@ -171,12 +181,12 @@ func (r *instanceRepositoryImpl) CreateInstances(ctx context.Context, instances } if len(newTags) > 0 { - if err := r.db.InsertWithContext(ctx, InsertTagsQuery, newTags); err != nil { - return err + if _, err = tx.NamedExecContext(ctx, InsertTagsQuery, newTags); err != nil { + return fmt.Errorf("failed to insert tags: %w", err) } } - return nil + return tx.Commit() } // DeleteInstance deletes an instance and its associated tags from the database. From d1f3aa78c54ea297f26a5f4b1bb916fb2d79ff91 Mon Sep 17 00:00:00 2001 From: r2dedios Date: Tue, 26 May 2026 13:01:48 +0200 Subject: [PATCH 53/53] docs(db): rewrite v0.5-to-v0.5.1 migration guide --- .../v0.5.1/data-migration-0.5-0.5.1.md | 424 ++++++++++++------ 1 file changed, 287 insertions(+), 137 deletions(-) diff --git a/doc/releases/v0.5.1/data-migration-0.5-0.5.1.md b/doc/releases/v0.5.1/data-migration-0.5-0.5.1.md index a0b02a4f..ad6f5ac9 100644 --- a/doc/releases/v0.5.1/data-migration-0.5-0.5.1.md +++ b/doc/releases/v0.5.1/data-migration-0.5-0.5.1.md @@ -1,142 +1,292 @@ -# ClusterIQ Data Migration Guide (v0.4.X → v0.5) -This document defines a deterministic, manual procedure to migrate ClusterIQ -data from version `0.5` to `0.5.1`. +# ClusterIQ Data Migration Guide (v0.5 → v0.5.1) -The migration is performed directly in the target database: -- Apply DB schema transformations +This document defines the procedure to migrate a ClusterIQ database from version `0.5` to `0.5.1`. -## Scope -This procedure covers the data migration from ClusterIQ `0.5` to `0.5.1`. +## Changes Summary -This procedure does NOT: -- Deploy ClusterIQ -- Perform application upgrades -- Modify ClusterIQ configuration outside the database -- Execute automated CI/CD tasks -- Origin Database is running in Openshift -- Destination Database is running in the Development setup (podman-compose) +| Object | Change | Risk | +|--------|--------|------| +| `instances.cluster_id` | `INTEGER` → `BIGINT` | Low — type widening, no data loss | +| `events.resource_id` | `INTEGER` → `BIGINT` | Low — type widening, no data loss | +| `events.resource_type` | `TEXT` → `RESOURCE_TYPE` enum | Medium — requires value conversion | +| `events_resource_type_check` | Removed (replaced by enum) | None | +| Cascade delete triggers | New (clusters, instances) | None — additive | +| `cluster_events` view | Updated join conditions | None — recreated in place | +| `system_events` view | Updated join conditions | None — recreated in place | +| `clusters_tags` view | Added WHERE filter | None — recreated in place | ## Preconditions -- Source ClusterIQ version is `0.5` -- pgsql deployment using `v0.5.1` image tag -- Operator has access to OpenShift and PostgreSQL -- `oc`, `psql`, and `podman` CLIs are installed and configured -- No write operations are running against the source database - -## Procedure Checklist - -### Pre-checks - -* [ ] **M0** — Prepare CLI and Namespace - ```sh - # Login into Openshift - oc login … - - # Define Namespace - export CIQ_NAMESPACE="" - oc project $CIQ_NAMESPACE - - # Stop Scanner to prevent updates during the data migration - oc patch cronjob scanner -p '{"spec" : {"suspend" : true }}' --type=merge -n $CIQ_NAMESPACE - ``` - -* [ ] **M1** — Connect to the target Database (assuming it runs in OCP). - ```sh - oc rsh pgsql-0 -n $CIQ_NAMESPACE - - psql -d clusteriq - ``` - -* [ ] **M2** — Check `RESOURCE_TYPE` enum exists. - ```sql - clusteriq=# \dT+ RESOURCE_TYPE - ``` - **Expected result:** - ```sql - List of data types - Schema | Name | Internal name | Size | Elements | Owner | Access privileges | Description - --------+---------------+---------------+------+----------+-------+-------------------+------------- - public | resource_type | resource_type | 4 | Account +| user | | - | | | | Cluster +| | | - | | | | Instance | | | - (1 row) - - ``` - -* [ ] **M3** — Modify instances table. - ```sql - -- Count before modifying - SELECT COUNT(*) FROM instances; - - -- Tx - BEGIN; - - -- 1. Drop FK - ALTER TABLE instances DROP CONSTRAINT instances_cluster_id_fkey; - - -- 2. Change column type - ALTER TABLE instances - ALTER COLUMN cluster_id TYPE BIGINT - USING cluster_id::bigint; - - -- 3. Recreate FK - ALTER TABLE instances - ADD CONSTRAINT instances_cluster_id_fkey - FOREIGN KEY (cluster_id) REFERENCES clusters(id) - ON DELETE CASCADE; - - COMMIT; - - -- Count after modifying must be same result as before - SELECT COUNT(*) FROM instances; - ``` - - **Expected result:** Alter table tx correct - -* [ ] **M4** — Modify events table. - ```sql - -- Count before modifying - SELECT COUNT(*) FROM instances; - - -- Tx - BEGIN; - - -- 1) Drop legacy CHECK constraint (if present) - ALTER TABLE public.events - DROP CONSTRAINT IF EXISTS events_resource_type_check; - - -- 2) resource_id: INTEGER -> BIGINT - ALTER TABLE public.events - ALTER COLUMN resource_id TYPE BIGINT - USING resource_id::bigint; - - -- 3) resource_type: TEXT -> RESOURCE_TYPE (align with enum: Account/Cluster/Instance) - ALTER TABLE public.events - ALTER COLUMN resource_type TYPE public.resource_type - USING ( - CASE resource_type - WHEN 'cluster' THEN 'Cluster'::public.resource_type - WHEN 'Cluster' THEN 'Cluster'::public.resource_type - WHEN 'instance' THEN 'Instance'::public.resource_type - WHEN 'Instance' THEN 'Instance'::public.resource_type - WHEN 'account' THEN 'Account'::public.resource_type - WHEN 'Account' THEN 'Account'::public.resource_type - ELSE resource_type::public.resource_type - END - ); - - -- 4) Keep NOT NULL (it was NOT NULL already, but explicit is fine) - ALTER TABLE public.events - ALTER COLUMN resource_type SET NOT NULL; - - COMMIT; - - -- Count after modifying must be same result as before - SELECT COUNT(*) FROM instances; - ``` - - **Expected result:** Alter table tx correct - -## Postconditions -- Tables modified correctly. -- Views re-created. + +- Source ClusterIQ version is `v0.5` +- Target images are tagged `v0.5.1` +- The `RESOURCE_TYPE` enum already exists (created in v0.5) +- Operator has `oc` and `psql` access +- **No write operations running against the database during migration** + +## Procedure + +### M0 — Prepare environment + +```sh +oc login ... + +export CIQ_NAMESPACE="cluster-iq" +oc project $CIQ_NAMESPACE + +# Stop scanner to prevent writes during migration +oc patch cronjob scanner -p '{"spec":{"suspend":true}}' --type=merge -n $CIQ_NAMESPACE +``` + +### M1 — Connect to database + +```sh +oc rsh pgsql-0 -n $CIQ_NAMESPACE +psql -d clusteriq +``` + +### M2 — Pre-migration snapshot + +Save row counts before any change. These must match after each step. + +```sql +SELECT 'instances' AS tbl, COUNT(*) AS rows FROM instances +UNION ALL +SELECT 'events', COUNT(*) FROM events; +``` + +Record the output. Every verification step below must return these same numbers. + +### M3 — Verify RESOURCE_TYPE enum exists + +```sql +SELECT enumlabel FROM pg_enum +WHERE enumtypid = 'public.resource_type'::regtype +ORDER BY enumsortorder; +``` + +Expected output: + +``` + enumlabel +----------- + Account + Cluster + Instance +``` + +If the enum does not exist, **stop here** — the database is not on v0.5. + +### M4 — Verify event data is safe to convert + +Check that all existing `resource_type` values can map to the enum: + +```sql +SELECT DISTINCT resource_type FROM events +WHERE resource_type NOT IN ('cluster', 'Cluster', 'instance', 'Instance', 'account', 'Account'); +``` + +Expected output: **0 rows**. If any rows appear, those values need manual mapping before continuing. + +### M5 — Alter `instances` table + +```sql +BEGIN; + +ALTER TABLE instances DROP CONSTRAINT instances_cluster_id_fkey; + +ALTER TABLE instances + ALTER COLUMN cluster_id TYPE BIGINT + USING cluster_id::bigint; + +ALTER TABLE instances + ADD CONSTRAINT instances_cluster_id_fkey + FOREIGN KEY (cluster_id) REFERENCES clusters(id) + ON DELETE CASCADE; + +COMMIT; +``` + +Verify: + +```sql +SELECT COUNT(*) FROM instances; +-- Must match M2 +``` + +### M6 — Alter `events` table + +```sql +BEGIN; + +ALTER TABLE events + DROP CONSTRAINT IF EXISTS events_resource_type_check; + +ALTER TABLE events + ALTER COLUMN resource_id TYPE BIGINT + USING resource_id::bigint; + +ALTER TABLE events + ALTER COLUMN resource_type TYPE public.resource_type + USING ( + CASE resource_type + WHEN 'cluster' THEN 'Cluster'::public.resource_type + WHEN 'Cluster' THEN 'Cluster'::public.resource_type + WHEN 'instance' THEN 'Instance'::public.resource_type + WHEN 'Instance' THEN 'Instance'::public.resource_type + WHEN 'account' THEN 'Account'::public.resource_type + WHEN 'Account' THEN 'Account'::public.resource_type + END + ); + +ALTER TABLE events + ALTER COLUMN resource_type SET NOT NULL; + +COMMIT; +``` + +Verify: + +```sql +SELECT COUNT(*) FROM events; +-- Must match M2 +``` + +### M7 — Create cascade delete triggers + +```sql +CREATE OR REPLACE FUNCTION delete_cluster_events() +RETURNS TRIGGER AS $$ +BEGIN + DELETE FROM events WHERE resource_type = 'Cluster'::RESOURCE_TYPE AND resource_id = OLD.id; + RETURN OLD; +END; +$$ LANGUAGE plpgsql; + +CREATE TRIGGER trg_delete_cluster_events + BEFORE DELETE ON clusters + FOR EACH ROW + EXECUTE FUNCTION delete_cluster_events(); + +CREATE OR REPLACE FUNCTION delete_instance_events() +RETURNS TRIGGER AS $$ +BEGIN + DELETE FROM events WHERE resource_type = 'Instance'::RESOURCE_TYPE AND resource_id = OLD.id; + RETURN OLD; +END; +$$ LANGUAGE plpgsql; + +CREATE TRIGGER trg_delete_instance_events + BEFORE DELETE ON instances + FOR EACH ROW + EXECUTE FUNCTION delete_instance_events(); +``` + +Verify: + +```sql +SELECT tgname FROM pg_trigger +WHERE tgname IN ('trg_delete_cluster_events', 'trg_delete_instance_events'); +``` + +Expected: 2 rows. + +### M8 — Recreate views + +The event views must be updated to use the enum type instead of plain text comparisons. + +```sql +CREATE OR REPLACE VIEW cluster_events AS +SELECT + ev.id, + ev.event_timestamp, + ev.triggered_by, + ev.action, + COALESCE(c.cluster_id, i.instance_id) AS resource_id, + ev.resource_type, + ev.result, + ev.description, + ev.severity +FROM events ev +LEFT JOIN clusters c ON ev.resource_type = 'Cluster'::RESOURCE_TYPE AND c.id = ev.resource_id +LEFT JOIN instances i ON ev.resource_type = 'Instance'::RESOURCE_TYPE AND i.id = ev.resource_id +ORDER BY event_timestamp DESC; + +CREATE OR REPLACE VIEW system_events AS +SELECT + ev.id, + ev.event_timestamp, + ev.triggered_by, + ev.action, + COALESCE(c.cluster_id, i.instance_id) AS resource_id, + ev.resource_type, + ev.result, + ev.description, + ev.severity, + acc.account_id, + acc.provider +FROM events ev +LEFT JOIN clusters c ON ev.resource_type = 'Cluster'::RESOURCE_TYPE AND c.id = ev.resource_id +LEFT JOIN instances i ON ev.resource_type = 'Instance'::RESOURCE_TYPE AND i.id = ev.resource_id +LEFT JOIN accounts acc ON acc.id = ( + CASE + WHEN ev.resource_type = 'Cluster'::RESOURCE_TYPE + THEN (SELECT c.account_id FROM clusters c WHERE c.id = ev.resource_id) + WHEN ev.resource_type = 'Instance'::RESOURCE_TYPE + THEN (SELECT c.account_id FROM clusters c WHERE c.id = (SELECT i.cluster_id FROM instances i WHERE i.id = ev.resource_id)) + END +) +ORDER BY ev.event_timestamp DESC; + +CREATE OR REPLACE VIEW clusters_tags AS +SELECT + c.cluster_id, + t.key, + MIN(t.value) AS value +FROM clusters c +JOIN instances i ON i.cluster_id = c.id +JOIN tags t ON t.instance_id = i.id +WHERE t.key != 'Name' AND t.key != 'MachineName' +GROUP BY c.cluster_id, t.key +HAVING COUNT(*) > 1; +``` + +Verify: + +```sql +SELECT viewname FROM pg_views +WHERE schemaname = 'public' + AND viewname IN ('cluster_events', 'system_events', 'clusters_tags'); +``` + +Expected: 3 rows. + +### M9 — Refresh materialized views + +```sql +SELECT refresh_materialized_views(); +``` + +### M10 — Final verification + +```sql +SELECT 'instances' AS tbl, COUNT(*) AS rows FROM instances +UNION ALL +SELECT 'events', COUNT(*) FROM events; +``` + +Must match the counts from M2 exactly. + +### M11 — Resume scanner + +Exit psql and the pod, then: + +```sh +oc patch cronjob scanner -p '{"spec":{"suspend":false}}' --type=merge -n $CIQ_NAMESPACE +``` + +## Rollback + +If any step fails, the transaction (BEGIN/COMMIT) ensures the table is unchanged. +Roll back by fixing the issue and re-running the failed step. No partial state is possible for M5 and M6 since they are wrapped in transactions. + +For M7 and M8 (triggers and views), these are idempotent — re-running them is safe.