From 0a77d002b6d74c8a9e829b3fef95548fd840d85d Mon Sep 17 00:00:00 2001 From: cdarninsuang Date: Fri, 3 Jul 2026 18:30:38 +0000 Subject: [PATCH] fix: use LEFT JOIN in projectBasicSQL to include projects without quota records When quota_per_project_enable is false, Harbor does not create quota or quota_usage rows for new projects. The previous INNER JOIN on those tables silently excluded every project without a quota record from the collector map, causing the member/repo/artifact update functions to log a "project not found" error for each missing project on every scrape. With ~500+ affected projects this produced hundreds of error log writes per scrape, increasing scrape latency and risking Prometheus timeouts. Affected projects also had no metrics emitted for quota, member, repo, or artifact dimensions. Switch to LEFT JOIN with join-condition filtering (moving name='public' and reference='project' predicates into the ON clause) and COALESCE defaults so every non-deleted project is always present in the map. Projects without quota records emit 0 for storage quota/usage. Also cast project_id to text for the quota join rather than casting reference_id to integer, which is safer when the quota table contains non-integer reference IDs from other reference types. Add a test case that creates a project without quota records and asserts it appears in the collector map with empty quota/usage values. Signed-off-by: cdarninsuang --- src/pkg/exporter/project_collector.go | 10 +++---- src/pkg/exporter/project_collector_test.go | 32 ++++++++++++++++------ 2 files changed, 29 insertions(+), 13 deletions(-) diff --git a/src/pkg/exporter/project_collector.go b/src/pkg/exporter/project_collector.go index cdf9538777b..cfb817ad246 100644 --- a/src/pkg/exporter/project_collector.go +++ b/src/pkg/exporter/project_collector.go @@ -32,11 +32,11 @@ var ( FROM project INNER JOIN project_metadata ON project.project_id=project_metadata.project_id WHERE project.deleted=FALSE AND project_metadata.name='public' GROUP BY project_metadata.value;` - projectBasicSQL = `SELECT project.project_id, project.name, project_metadata.value AS public, quota.hard AS quota, quota_usage.used AS usage FROM project - INNER JOIN project_metadata ON (project.project_id = project_metadata.project_id) - INNER JOIN quota ON project.project_id = CAST(quota.reference_id AS Integer) - INNER JOIN quota_usage ON project.project_id = CAST(quota_usage.reference_id AS Integer) - WHERE quota.reference='project' AND quota_usage.reference='project' AND project.deleted=FALSE AND project_metadata.name='public';` + projectBasicSQL = `SELECT project.project_id, project.name, COALESCE(project_metadata.value, 'false') AS public, COALESCE(quota.hard, '{}') AS quota, COALESCE(quota_usage.used, '{}') AS usage FROM project + LEFT JOIN project_metadata ON (project.project_id = project_metadata.project_id AND project_metadata.name='public') + LEFT JOIN quota ON (project.project_id::text = quota.reference_id AND quota.reference='project') + LEFT JOIN quota_usage ON (project.project_id::text = quota_usage.reference_id AND quota_usage.reference='project') + WHERE project.deleted=FALSE;` projectMemberSQL = `SELECT project.project_id, COUNT(project.project_id) AS member_total FROM project INNER JOIN project_member ON project.project_id=project_member.project_id WHERE project.deleted=FALSE AND project_member.entity_type='u' diff --git a/src/pkg/exporter/project_collector_test.go b/src/pkg/exporter/project_collector_test.go index 56edbd2c119..dbda4b1ce61 100644 --- a/src/pkg/exporter/project_collector_test.go +++ b/src/pkg/exporter/project_collector_test.go @@ -29,6 +29,7 @@ var ( eve = models.User{Username: "eve", Password: "password", Email: "eve@test.com"} testPro1 = proModels.Project{OwnerID: 1, Name: "test1", Metadata: map[string]string{"public": "true"}} testPro2 = proModels.Project{OwnerID: 1, Name: "test2", Metadata: map[string]string{"public": "false"}} + testPro3 = proModels.Project{OwnerID: 1, Name: "test3", Metadata: map[string]string{"public": "false"}} rs1 = qtypes.ResourceList{qtypes.ResourceStorage: 100} rs2 = qtypes.ResourceList{qtypes.ResourceStorage: 200} repo1 = model.RepoRecord{Name: "repo1"} @@ -65,8 +66,14 @@ func setupTest(t *testing.T) { if err != nil { t.Errorf("project creating %v", err) } + // testPro3 has no quota records — simulates quota_per_project_enable=false + proID3, err := proctl.Ctl.Create(ctx, &testPro3) + if err != nil { + t.Errorf("project creating %v", err) + } testPro1.ProjectID = proID1 testPro2.ProjectID = proID2 + testPro3.ProjectID = proID3 // Create quota for project quotactl.Ctl.Create(ctx, "project", strconv.Itoa(int(testPro1.ProjectID)), rs1) @@ -125,14 +132,14 @@ func setupTest(t *testing.T) { } func tearDownTest(t *testing.T) { - dao.GetOrmer().Raw("delete from project_member where project_id in (?, ?)", []int64{testPro1.ProjectID, testPro2.ProjectID}).Exec() - dao.GetOrmer().Raw("delete from project_metadata where project_id in (?, ?)", []int64{testPro1.ProjectID, testPro2.ProjectID}).Exec() - dao.GetOrmer().Raw("delete from quota where reference=\"project\" and reference_id in (?, ?)", []int64{testPro1.ProjectID, testPro2.ProjectID}).Exec() - dao.GetOrmer().Raw("delete from quota_usage where reference=\"project\" and reference_id in (?, ?)", []int64{testPro1.ProjectID, testPro2.ProjectID}).Exec() - dao.GetOrmer().Raw("delete from project where project_id in (?, ?)", []int64{testPro1.ProjectID, testPro2.ProjectID}).Exec() - dao.GetOrmer().Raw("delete from artifact where project_id in (?, ?)", []int64{testPro1.ProjectID, testPro2.ProjectID}).Exec() - dao.GetOrmer().Raw("delete from repository where project_id in (?, ?)", []int64{testPro1.ProjectID, testPro2.ProjectID}).Exec() - dao.GetOrmer().Raw("delete from cve_allowlist where project_id in (?, ?)", []int64{testPro1.ProjectID, testPro2.ProjectID}).Exec() + dao.GetOrmer().Raw("delete from project_member where project_id in (?, ?, ?)", []int64{testPro1.ProjectID, testPro2.ProjectID, testPro3.ProjectID}).Exec() + dao.GetOrmer().Raw("delete from project_metadata where project_id in (?, ?, ?)", []int64{testPro1.ProjectID, testPro2.ProjectID, testPro3.ProjectID}).Exec() + dao.GetOrmer().Raw("delete from quota where reference=\"project\" and reference_id in (?, ?, ?)", []int64{testPro1.ProjectID, testPro2.ProjectID, testPro3.ProjectID}).Exec() + dao.GetOrmer().Raw("delete from quota_usage where reference=\"project\" and reference_id in (?, ?, ?)", []int64{testPro1.ProjectID, testPro2.ProjectID, testPro3.ProjectID}).Exec() + dao.GetOrmer().Raw("delete from project where project_id in (?, ?, ?)", []int64{testPro1.ProjectID, testPro2.ProjectID, testPro3.ProjectID}).Exec() + dao.GetOrmer().Raw("delete from artifact where project_id in (?, ?, ?)", []int64{testPro1.ProjectID, testPro2.ProjectID, testPro3.ProjectID}).Exec() + dao.GetOrmer().Raw("delete from repository where project_id in (?, ?, ?)", []int64{testPro1.ProjectID, testPro2.ProjectID, testPro3.ProjectID}).Exec() + dao.GetOrmer().Raw("delete from cve_allowlist where project_id in (?, ?, ?)", []int64{testPro1.ProjectID, testPro2.ProjectID, testPro3.ProjectID}).Exec() dao.GetOrmer().Raw("delete from harbor_user where user_id in (?, ?, ?)", []int{alice.UserID, bob.UserID, eve.UserID}).Exec() } @@ -167,4 +174,13 @@ func (c *ProjectCollectorTestSuite) TestProjectCollector() { c.Equalf(pMap[testPro2.ProjectID].PullTotal, float64(0), "pMap %v", pMap) c.Equalf(pMap[testPro2.ProjectID].Artifact["IMAGE"].ArtifactTotal, float64(1), "pMap %v", pMap) + // testPro3 has no quota records (simulates quota_per_project_enable=false). + // Verify it is still included in the map with zero quota/usage rather than + // being silently dropped and logged as "project not found". + c.Containsf(pMap, testPro3.ProjectID, "project without quota records should appear in pMap") + c.Equalf(pMap[testPro3.ProjectID].Name, testPro3.Name, "pMap %v", pMap) + c.Equalf(strconv.FormatBool(pMap[testPro3.ProjectID].Public), testPro3.Metadata["public"], "pMap %v", pMap) + c.Equalf(pMap[testPro3.ProjectID].Quota, "{}", "project without quota record should have empty quota") + c.Equalf(pMap[testPro3.ProjectID].Usage, "{}", "project without quota record should have empty usage") + }