From 11e7fa97aff2711a55cbd02b23a215e95617ed36 Mon Sep 17 00:00:00 2001 From: egorikas Date: Thu, 11 Jun 2026 21:38:22 +0200 Subject: [PATCH] Expose cgroup v2 memory.swap.events counters Extend the cgroup v2 event reporting with the memory.swap.events counters that aren't yet surfaced by cAdvisor: - container_memory_swap_events_high_total - container_memory_swap_events_max_total - container_memory_swap_events_fail_total Counters are read key-by-key and tolerate missing keys, so memory.swap.high (newer kernels) stays at zero where unavailable. Signed-off-by: egorikas --- container/libcontainer/handler.go | 13 +++++ container/libcontainer/handler_test.go | 58 +++++++++++++++++++ docs/storage/prometheus.md | 5 ++ info/v1/container.go | 8 +++ integration/tests/metrics/prometheus_test.go | 6 ++ metrics/prometheus.go | 21 +++++++ metrics/prometheus_fake.go | 5 ++ metrics/testdata/prometheus_metrics | 9 +++ .../prometheus_metrics_whitelist_filtered | 9 +++ 9 files changed, 134 insertions(+) diff --git a/container/libcontainer/handler.go b/container/libcontainer/handler.go index c875ee4737..96a41ee236 100644 --- a/container/libcontainer/handler.go +++ b/container/libcontainer/handler.go @@ -95,6 +95,7 @@ func (h *Handler) GetStats() (*info.ContainerStats, error) { if cgroups.IsCgroup2UnifiedMode() { setMemoryEvents(h.cgroupManager.Path(""), stats) + setSwapEvents(h.cgroupManager.Path(""), stats) } if h.includedMetrics.Has(container.ProcessSchedulerMetrics) { @@ -897,6 +898,18 @@ func setMemoryEvents(cgroupPath string, ret *info.ContainerStats) { } } +func setSwapEvents(cgroupPath string, ret *info.ContainerStats) { + if val, err := fscommon.GetValueByKey(cgroupPath, "memory.swap.events", "high"); err == nil { + ret.Memory.SwapEvents.High = val + } + if val, err := fscommon.GetValueByKey(cgroupPath, "memory.swap.events", "max"); err == nil { + ret.Memory.SwapEvents.Max = val + } + if val, err := fscommon.GetValueByKey(cgroupPath, "memory.swap.events", "fail"); err == nil { + ret.Memory.SwapEvents.Fail = val + } +} + func setCPUSetStats(s *cgroups.Stats, ret *info.ContainerStats) { ret.CpuSet.MemoryMigrate = s.CPUSetStats.MemoryMigrate } diff --git a/container/libcontainer/handler_test.go b/container/libcontainer/handler_test.go index effb5c11e2..c8c6e853d3 100644 --- a/container/libcontainer/handler_test.go +++ b/container/libcontainer/handler_test.go @@ -443,3 +443,61 @@ func TestSetMemoryEventsFileNotFound(t *testing.T) { assert.Equal(t, uint64(0), ret.Memory.Events.High) assert.Equal(t, uint64(0), ret.Memory.Events.Max) } + +func TestSetSwapEvents(t *testing.T) { + cgroups.TestMode = true + defer func() { cgroups.TestMode = false }() + + testData := []struct { + name string + content string + high uint64 + max uint64 + fail uint64 + }{ + { + "all counters", + "high 2\nmax 9\nfail 4\n", + 2, 9, 4, + }, + { + "zeros", + "high 0\nmax 0\nfail 0\n", + 0, 0, 0, + }, + { + "fail only", + "high 0\nmax 0\nfail 17\n", + 0, 0, 17, + }, + { + "empty file", + "", + 0, 0, 0, + }, + } + + for _, testItem := range testData { + t.Run(testItem.name, func(t *testing.T) { + dir := t.TempDir() + err := os.WriteFile(filepath.Join(dir, "memory.swap.events"), []byte(testItem.content), 0o644) + assert.Nil(t, err) + + var ret info.ContainerStats + setSwapEvents(dir, &ret) + + assert.Equal(t, testItem.high, ret.Memory.SwapEvents.High) + assert.Equal(t, testItem.max, ret.Memory.SwapEvents.Max) + assert.Equal(t, testItem.fail, ret.Memory.SwapEvents.Fail) + }) + } +} + +func TestSetSwapEventsFileNotFound(t *testing.T) { + var ret info.ContainerStats + setSwapEvents("/nonexistent/path", &ret) + + assert.Equal(t, uint64(0), ret.Memory.SwapEvents.High) + assert.Equal(t, uint64(0), ret.Memory.SwapEvents.Max) + assert.Equal(t, uint64(0), ret.Memory.SwapEvents.Fail) +} diff --git a/docs/storage/prometheus.md b/docs/storage/prometheus.md index f7a84ca91f..6d6b3c1859 100644 --- a/docs/storage/prometheus.md +++ b/docs/storage/prometheus.md @@ -56,6 +56,8 @@ Metric name | Type | Description | Unit (where applicable) | option parameter | `container_memory_bandwidth_bytes` | Gauge | Total memory bandwidth usage statistics for container counted with RDT Memory Bandwidth Monitoring (MBM). | bytes | resctrl | `container_memory_bandwidth_local_bytes` | Gauge | Local memory bandwidth usage statistics for container counted with RDT Memory Bandwidth Monitoring (MBM). | bytes | resctrl | `container_memory_cache` | Gauge | Total page cache memory | bytes | memory | +`container_memory_events_high_total` | Counter | Cumulative count of memory.high throttle events (cgroup v2) | | memory | +`container_memory_events_max_total` | Counter | Cumulative count of memory.max limit hit events (cgroup v2) | | memory | `container_memory_failcnt` | Counter | Number of memory usage hits limits | | memory | `container_memory_failures_total` | Counter | Cumulative count of memory allocation failures | | memory | `container_memory_file_dirty_bytes` | Gauge | File cache that has been modified but not yet written back to disk (cgroup v2) | bytes | memory | @@ -68,6 +70,9 @@ Metric name | Type | Description | Unit (where applicable) | option parameter | `container_memory_pgsteal_total` | Counter | Cumulative number of pages reclaimed by the page reclaim algorithm (cgroup v2) | | memory | `container_memory_rss` | Gauge | Size of RSS | bytes | memory | `container_memory_swap` | Gauge | Container swap usage | bytes | memory | +`container_memory_swap_events_fail_total` | Counter | Cumulative count of swap allocation failures (cgroup v2) | | memory | +`container_memory_swap_events_high_total` | Counter | Cumulative count of memory.swap.high throttle events (cgroup v2) | | memory | +`container_memory_swap_events_max_total` | Counter | Cumulative count of memory.swap.max limit hit events (cgroup v2) | | memory | `container_memory_usage_bytes` | Gauge | Current memory usage, including all memory regardless of when it was accessed | bytes | memory | `container_memory_working_set_bytes` | Gauge | Current working set | bytes | memory | `container_memory_workingset_refault_anon_total` | Counter | Cumulative number of refaults of previously evicted anonymous pages (cgroup v2) | | memory | diff --git a/info/v1/container.go b/info/v1/container.go index 1e2b5013c4..709b300955 100644 --- a/info/v1/container.go +++ b/info/v1/container.go @@ -488,6 +488,8 @@ type MemoryStats struct { PSI PSIStats `json:"psi"` Events MemoryEvents `json:"events,omitempty"` + + SwapEvents SwapEvents `json:"swap_events,omitempty"` } type MemoryEvents struct { @@ -495,6 +497,12 @@ type MemoryEvents struct { Max uint64 `json:"max"` } +type SwapEvents struct { + High uint64 `json:"high"` + Max uint64 `json:"max"` + Fail uint64 `json:"fail"` +} + type CPUSetStats struct { MemoryMigrate uint64 `json:"memory_migrate"` } diff --git a/integration/tests/metrics/prometheus_test.go b/integration/tests/metrics/prometheus_test.go index 56cf4d1777..8af1f743b3 100644 --- a/integration/tests/metrics/prometheus_test.go +++ b/integration/tests/metrics/prometheus_test.go @@ -119,6 +119,9 @@ func TestCoreMemoryMetricsExist(t *testing.T) { "container_memory_rss", "container_memory_events_high_total", "container_memory_events_max_total", + "container_memory_swap_events_high_total", + "container_memory_swap_events_max_total", + "container_memory_swap_events_fail_total", } for _, name := range memoryMetrics { @@ -289,6 +292,9 @@ func TestMetricsHaveCorrectTypes(t *testing.T) { "container_network_transmit_bytes_total", "container_memory_events_high_total", "container_memory_events_max_total", + "container_memory_swap_events_high_total", + "container_memory_swap_events_max_total", + "container_memory_swap_events_fail_total", } for _, name := range counterMetrics { if mf, ok := families[name]; ok { diff --git a/metrics/prometheus.go b/metrics/prometheus.go index 7a5186297d..a85fb00e4e 100644 --- a/metrics/prometheus.go +++ b/metrics/prometheus.go @@ -532,6 +532,27 @@ func NewPrometheusCollector(i infoProvider, f ContainerLabelsFunc, includedMetri getValues: func(s *info.ContainerStats) metricValues { return metricValues{{value: float64(s.Memory.Events.Max), timestamp: s.Timestamp}} }, + }, { + name: "container_memory_swap_events_high_total", + help: "Cumulative count of memory.swap.high throttle events for the container", + valueType: prometheus.CounterValue, + getValues: func(s *info.ContainerStats) metricValues { + return metricValues{{value: float64(s.Memory.SwapEvents.High), timestamp: s.Timestamp}} + }, + }, { + name: "container_memory_swap_events_max_total", + help: "Cumulative count of memory.swap.max limit hit events for the container", + valueType: prometheus.CounterValue, + getValues: func(s *info.ContainerStats) metricValues { + return metricValues{{value: float64(s.Memory.SwapEvents.Max), timestamp: s.Timestamp}} + }, + }, { + name: "container_memory_swap_events_fail_total", + help: "Cumulative count of swap allocation failures for the container", + valueType: prometheus.CounterValue, + getValues: func(s *info.ContainerStats) metricValues { + return metricValues{{value: float64(s.Memory.SwapEvents.Fail), timestamp: s.Timestamp}} + }, }, { name: "container_memory_file_dirty_bytes", help: "Number of bytes of file cache that has been modified but not yet written back to disk.", diff --git a/metrics/prometheus_fake.go b/metrics/prometheus_fake.go index c2bcd17f24..30f81a832b 100644 --- a/metrics/prometheus_fake.go +++ b/metrics/prometheus_fake.go @@ -399,6 +399,11 @@ func (p testSubcontainersInfoProvider) GetRequestedContainersInfo(string, v2.Req High: 42, Max: 5, }, + SwapEvents: info.SwapEvents{ + High: 8, + Max: 6, + Fail: 7, + }, }, Hugetlb: map[string]info.HugetlbStats{ "2Mi": { diff --git a/metrics/testdata/prometheus_metrics b/metrics/testdata/prometheus_metrics index 01f29ee0d2..435786c954 100644 --- a/metrics/testdata/prometheus_metrics +++ b/metrics/testdata/prometheus_metrics @@ -169,6 +169,15 @@ container_memory_events_high_total{container_env_foo_env="prod",container_label_ # HELP container_memory_events_max_total Cumulative count of memory.max limit hit events for the container # TYPE container_memory_events_max_total counter container_memory_events_max_total{container_env_foo_env="prod",container_label_foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 5 1395066363000 +# HELP container_memory_swap_events_high_total Cumulative count of memory.swap.high throttle events for the container +# TYPE container_memory_swap_events_high_total counter +container_memory_swap_events_high_total{container_env_foo_env="prod",container_label_foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 8 1395066363000 +# HELP container_memory_swap_events_max_total Cumulative count of memory.swap.max limit hit events for the container +# TYPE container_memory_swap_events_max_total counter +container_memory_swap_events_max_total{container_env_foo_env="prod",container_label_foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 6 1395066363000 +# HELP container_memory_swap_events_fail_total Cumulative count of swap allocation failures for the container +# TYPE container_memory_swap_events_fail_total counter +container_memory_swap_events_fail_total{container_env_foo_env="prod",container_label_foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 7 1395066363000 # HELP container_memory_failcnt Number of memory usage hits limits # TYPE container_memory_failcnt counter container_memory_failcnt{container_env_foo_env="prod",container_label_foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 0 1395066363000 diff --git a/metrics/testdata/prometheus_metrics_whitelist_filtered b/metrics/testdata/prometheus_metrics_whitelist_filtered index 9abb88d2d7..6434d4ee1b 100644 --- a/metrics/testdata/prometheus_metrics_whitelist_filtered +++ b/metrics/testdata/prometheus_metrics_whitelist_filtered @@ -169,6 +169,15 @@ container_memory_events_high_total{container_env_foo_env="prod",id="testcontaine # HELP container_memory_events_max_total Cumulative count of memory.max limit hit events for the container # TYPE container_memory_events_max_total counter container_memory_events_max_total{container_env_foo_env="prod",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 5 1395066363000 +# HELP container_memory_swap_events_high_total Cumulative count of memory.swap.high throttle events for the container +# TYPE container_memory_swap_events_high_total counter +container_memory_swap_events_high_total{container_env_foo_env="prod",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 8 1395066363000 +# HELP container_memory_swap_events_max_total Cumulative count of memory.swap.max limit hit events for the container +# TYPE container_memory_swap_events_max_total counter +container_memory_swap_events_max_total{container_env_foo_env="prod",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 6 1395066363000 +# HELP container_memory_swap_events_fail_total Cumulative count of swap allocation failures for the container +# TYPE container_memory_swap_events_fail_total counter +container_memory_swap_events_fail_total{container_env_foo_env="prod",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 7 1395066363000 # HELP container_memory_failcnt Number of memory usage hits limits # TYPE container_memory_failcnt counter container_memory_failcnt{container_env_foo_env="prod",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 0 1395066363000