From 55311779edba7633ffc88fdd0f94aa9639918ae4 Mon Sep 17 00:00:00 2001 From: Nils Eriksson Date: Thu, 11 Jun 2026 09:08:35 +0100 Subject: [PATCH 1/3] Expose cgroup v2 memory.events OOM metrics --- container/libcontainer/handler.go | 9 +++++++ container/libcontainer/handler_test.go | 27 ++++++++++++------- docs/storage/prometheus.md | 3 +++ info/v1/container.go | 7 +++-- integration/tests/metrics/prometheus_test.go | 6 +++++ metrics/prometheus.go | 21 +++++++++++++++ metrics/prometheus_fake.go | 7 +++-- metrics/testdata/prometheus_metrics | 9 +++++++ .../prometheus_metrics_whitelist_filtered | 9 +++++++ 9 files changed, 85 insertions(+), 13 deletions(-) diff --git a/container/libcontainer/handler.go b/container/libcontainer/handler.go index c875ee4737..5e9301e2b0 100644 --- a/container/libcontainer/handler.go +++ b/container/libcontainer/handler.go @@ -895,6 +895,15 @@ func setMemoryEvents(cgroupPath string, ret *info.ContainerStats) { if val, err := fscommon.GetValueByKey(cgroupPath, "memory.events", "max"); err == nil { ret.Memory.Events.Max = val } + if val, err := fscommon.GetValueByKey(cgroupPath, "memory.events", "oom"); err == nil { + ret.Memory.Events.Oom = val + } + if val, err := fscommon.GetValueByKey(cgroupPath, "memory.events", "oom_kill"); err == nil { + ret.Memory.Events.OomKill = val + } + if val, err := fscommon.GetValueByKey(cgroupPath, "memory.events", "oom_group_kill"); err == nil { + ret.Memory.Events.OomGroupKill = val + } } func setCPUSetStats(s *cgroups.Stats, ret *info.ContainerStats) { diff --git a/container/libcontainer/handler_test.go b/container/libcontainer/handler_test.go index effb5c11e2..5f1065c467 100644 --- a/container/libcontainer/handler_test.go +++ b/container/libcontainer/handler_test.go @@ -394,30 +394,33 @@ func TestSetMemoryEvents(t *testing.T) { defer func() { cgroups.TestMode = false }() testData := []struct { - name string - content string - high uint64 - max uint64 + name string + content string + high uint64 + max uint64 + oom uint64 + oomKill uint64 + oomGroupKill uint64 }{ { "all counters", - "low 0\nhigh 42\nmax 5\noom 1\noom_kill 0\noom_group_kill 0\n", - 42, 5, + "low 0\nhigh 42\nmax 5\noom 3\noom_kill 2\noom_group_kill 1\n", + 42, 5, 3, 2, 1, }, { "zeros", "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0\n", - 0, 0, + 0, 0, 0, 0, 0, }, { "high only", "low 0\nhigh 238\nmax 0\noom 0\noom_kill 0\noom_group_kill 0\n", - 238, 0, + 238, 0, 0, 0, 0, }, { "empty file", "", - 0, 0, + 0, 0, 0, 0, 0, }, } @@ -432,6 +435,9 @@ func TestSetMemoryEvents(t *testing.T) { assert.Equal(t, testItem.high, ret.Memory.Events.High) assert.Equal(t, testItem.max, ret.Memory.Events.Max) + assert.Equal(t, testItem.oom, ret.Memory.Events.Oom) + assert.Equal(t, testItem.oomKill, ret.Memory.Events.OomKill) + assert.Equal(t, testItem.oomGroupKill, ret.Memory.Events.OomGroupKill) }) } } @@ -442,4 +448,7 @@ func TestSetMemoryEventsFileNotFound(t *testing.T) { assert.Equal(t, uint64(0), ret.Memory.Events.High) assert.Equal(t, uint64(0), ret.Memory.Events.Max) + assert.Equal(t, uint64(0), ret.Memory.Events.Oom) + assert.Equal(t, uint64(0), ret.Memory.Events.OomKill) + assert.Equal(t, uint64(0), ret.Memory.Events.OomGroupKill) } diff --git a/docs/storage/prometheus.md b/docs/storage/prometheus.md index f7a84ca91f..7010e86f29 100644 --- a/docs/storage/prometheus.md +++ b/docs/storage/prometheus.md @@ -56,6 +56,9 @@ Metric name | Type | Description | Unit (where applicable) | option parameter | `container_memory_bandwidth_bytes` | Gauge | Total memory bandwidth usage statistics for container counted with RDT Memory Bandwidth Monitoring (MBM). | bytes | resctrl | `container_memory_bandwidth_local_bytes` | Gauge | Local memory bandwidth usage statistics for container counted with RDT Memory Bandwidth Monitoring (MBM). | bytes | resctrl | `container_memory_cache` | Gauge | Total page cache memory | bytes | memory | +`container_memory_events_oom_group_kill_total` | Counter | Cumulative count of memory OOM group kill events for the container | | memory | +`container_memory_events_oom_kill_total` | Counter | Cumulative count of memory OOM kill events for the container | | memory | +`container_memory_events_oom_total` | Counter | Cumulative count of memory allocation failure events for the container | | memory | `container_memory_failcnt` | Counter | Number of memory usage hits limits | | memory | `container_memory_failures_total` | Counter | Cumulative count of memory allocation failures | | memory | `container_memory_file_dirty_bytes` | Gauge | File cache that has been modified but not yet written back to disk (cgroup v2) | bytes | memory | diff --git a/info/v1/container.go b/info/v1/container.go index 1e2b5013c4..fa710cad2f 100644 --- a/info/v1/container.go +++ b/info/v1/container.go @@ -491,8 +491,11 @@ type MemoryStats struct { } type MemoryEvents struct { - High uint64 `json:"high"` - Max uint64 `json:"max"` + High uint64 `json:"high"` + Max uint64 `json:"max"` + Oom uint64 `json:"oom"` + OomKill uint64 `json:"oom_kill"` + OomGroupKill uint64 `json:"oom_group_kill"` } type CPUSetStats struct { diff --git a/integration/tests/metrics/prometheus_test.go b/integration/tests/metrics/prometheus_test.go index 56cf4d1777..0297b642bc 100644 --- a/integration/tests/metrics/prometheus_test.go +++ b/integration/tests/metrics/prometheus_test.go @@ -119,6 +119,9 @@ func TestCoreMemoryMetricsExist(t *testing.T) { "container_memory_rss", "container_memory_events_high_total", "container_memory_events_max_total", + "container_memory_events_oom_total", + "container_memory_events_oom_kill_total", + "container_memory_events_oom_group_kill_total", } for _, name := range memoryMetrics { @@ -289,6 +292,9 @@ func TestMetricsHaveCorrectTypes(t *testing.T) { "container_network_transmit_bytes_total", "container_memory_events_high_total", "container_memory_events_max_total", + "container_memory_events_oom_total", + "container_memory_events_oom_kill_total", + "container_memory_events_oom_group_kill_total", } for _, name := range counterMetrics { if mf, ok := families[name]; ok { diff --git a/metrics/prometheus.go b/metrics/prometheus.go index 7a5186297d..02526fdf07 100644 --- a/metrics/prometheus.go +++ b/metrics/prometheus.go @@ -532,6 +532,27 @@ func NewPrometheusCollector(i infoProvider, f ContainerLabelsFunc, includedMetri getValues: func(s *info.ContainerStats) metricValues { return metricValues{{value: float64(s.Memory.Events.Max), timestamp: s.Timestamp}} }, + }, { + name: "container_memory_events_oom_total", + help: "Cumulative count of memory allocation failure events for the container", + valueType: prometheus.CounterValue, + getValues: func(s *info.ContainerStats) metricValues { + return metricValues{{value: float64(s.Memory.Events.Oom), timestamp: s.Timestamp}} + }, + }, { + name: "container_memory_events_oom_kill_total", + help: "Cumulative count of memory OOM kill events for the container", + valueType: prometheus.CounterValue, + getValues: func(s *info.ContainerStats) metricValues { + return metricValues{{value: float64(s.Memory.Events.OomKill), timestamp: s.Timestamp}} + }, + }, { + name: "container_memory_events_oom_group_kill_total", + help: "Cumulative count of memory OOM group kill events for the container", + valueType: prometheus.CounterValue, + getValues: func(s *info.ContainerStats) metricValues { + return metricValues{{value: float64(s.Memory.Events.OomGroupKill), timestamp: s.Timestamp}} + }, }, { name: "container_memory_file_dirty_bytes", help: "Number of bytes of file cache that has been modified but not yet written back to disk.", diff --git a/metrics/prometheus_fake.go b/metrics/prometheus_fake.go index c2bcd17f24..6dcda802a5 100644 --- a/metrics/prometheus_fake.go +++ b/metrics/prometheus_fake.go @@ -396,8 +396,11 @@ func (p testSubcontainersInfoProvider) GetRequestedContainersInfo(string, v2.Req }, }, Events: info.MemoryEvents{ - High: 42, - Max: 5, + High: 42, + Max: 5, + Oom: 3, + OomKill: 2, + OomGroupKill: 1, }, }, Hugetlb: map[string]info.HugetlbStats{ diff --git a/metrics/testdata/prometheus_metrics b/metrics/testdata/prometheus_metrics index 01f29ee0d2..fe0efaf95a 100644 --- a/metrics/testdata/prometheus_metrics +++ b/metrics/testdata/prometheus_metrics @@ -169,6 +169,15 @@ container_memory_events_high_total{container_env_foo_env="prod",container_label_ # HELP container_memory_events_max_total Cumulative count of memory.max limit hit events for the container # TYPE container_memory_events_max_total counter container_memory_events_max_total{container_env_foo_env="prod",container_label_foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 5 1395066363000 +# HELP container_memory_events_oom_total Cumulative count of memory allocation failure events for the container +# TYPE container_memory_events_oom_total counter +container_memory_events_oom_total{container_env_foo_env="prod",container_label_foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 3 1395066363000 +# HELP container_memory_events_oom_kill_total Cumulative count of memory OOM kill events for the container +# TYPE container_memory_events_oom_kill_total counter +container_memory_events_oom_kill_total{container_env_foo_env="prod",container_label_foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 2 1395066363000 +# HELP container_memory_events_oom_group_kill_total Cumulative count of memory OOM group kill events for the container +# TYPE container_memory_events_oom_group_kill_total counter +container_memory_events_oom_group_kill_total{container_env_foo_env="prod",container_label_foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 1 1395066363000 # HELP container_memory_failcnt Number of memory usage hits limits # TYPE container_memory_failcnt counter container_memory_failcnt{container_env_foo_env="prod",container_label_foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 0 1395066363000 diff --git a/metrics/testdata/prometheus_metrics_whitelist_filtered b/metrics/testdata/prometheus_metrics_whitelist_filtered index 9abb88d2d7..b0064aa9c6 100644 --- a/metrics/testdata/prometheus_metrics_whitelist_filtered +++ b/metrics/testdata/prometheus_metrics_whitelist_filtered @@ -169,6 +169,15 @@ container_memory_events_high_total{container_env_foo_env="prod",id="testcontaine # HELP container_memory_events_max_total Cumulative count of memory.max limit hit events for the container # TYPE container_memory_events_max_total counter container_memory_events_max_total{container_env_foo_env="prod",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 5 1395066363000 +# HELP container_memory_events_oom_total Cumulative count of memory allocation failure events for the container +# TYPE container_memory_events_oom_total counter +container_memory_events_oom_total{container_env_foo_env="prod",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 3 1395066363000 +# HELP container_memory_events_oom_kill_total Cumulative count of memory OOM kill events for the container +# TYPE container_memory_events_oom_kill_total counter +container_memory_events_oom_kill_total{container_env_foo_env="prod",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 2 1395066363000 +# HELP container_memory_events_oom_group_kill_total Cumulative count of memory OOM group kill events for the container +# TYPE container_memory_events_oom_group_kill_total counter +container_memory_events_oom_group_kill_total{container_env_foo_env="prod",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 1 1395066363000 # HELP container_memory_failcnt Number of memory usage hits limits # TYPE container_memory_failcnt counter container_memory_failcnt{container_env_foo_env="prod",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 0 1395066363000 From 4b8b98fc76d7c40933890f8b3cffe1d154e69c29 Mon Sep 17 00:00:00 2001 From: Nils Eriksson Date: Thu, 11 Jun 2026 22:21:13 +0100 Subject: [PATCH 2/3] Expose cgroup v2 memory.events low counter Surface the low counter from memory.events alongside the OOM counters, as container_memory_events_low_total. Requested in PR feedback. --- container/libcontainer/handler.go | 3 +++ container/libcontainer/handler_test.go | 13 ++++++++----- docs/storage/prometheus.md | 1 + info/v1/container.go | 1 + integration/tests/metrics/prometheus_test.go | 2 ++ metrics/prometheus.go | 7 +++++++ metrics/prometheus_fake.go | 1 + metrics/testdata/prometheus_metrics | 3 +++ .../testdata/prometheus_metrics_whitelist_filtered | 3 +++ 9 files changed, 29 insertions(+), 5 deletions(-) diff --git a/container/libcontainer/handler.go b/container/libcontainer/handler.go index 5e9301e2b0..3304910461 100644 --- a/container/libcontainer/handler.go +++ b/container/libcontainer/handler.go @@ -889,6 +889,9 @@ func setMemoryStats(s *cgroups.Stats, ret *info.ContainerStats) { } func setMemoryEvents(cgroupPath string, ret *info.ContainerStats) { + if val, err := fscommon.GetValueByKey(cgroupPath, "memory.events", "low"); err == nil { + ret.Memory.Events.Low = val + } if val, err := fscommon.GetValueByKey(cgroupPath, "memory.events", "high"); err == nil { ret.Memory.Events.High = val } diff --git a/container/libcontainer/handler_test.go b/container/libcontainer/handler_test.go index 5f1065c467..b62d7e82c2 100644 --- a/container/libcontainer/handler_test.go +++ b/container/libcontainer/handler_test.go @@ -396,6 +396,7 @@ func TestSetMemoryEvents(t *testing.T) { testData := []struct { name string content string + low uint64 high uint64 max uint64 oom uint64 @@ -404,23 +405,23 @@ func TestSetMemoryEvents(t *testing.T) { }{ { "all counters", - "low 0\nhigh 42\nmax 5\noom 3\noom_kill 2\noom_group_kill 1\n", - 42, 5, 3, 2, 1, + "low 7\nhigh 42\nmax 5\noom 3\noom_kill 2\noom_group_kill 1\n", + 7, 42, 5, 3, 2, 1, }, { "zeros", "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0\n", - 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, }, { "high only", "low 0\nhigh 238\nmax 0\noom 0\noom_kill 0\noom_group_kill 0\n", - 238, 0, 0, 0, 0, + 0, 238, 0, 0, 0, 0, }, { "empty file", "", - 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, }, } @@ -433,6 +434,7 @@ func TestSetMemoryEvents(t *testing.T) { var ret info.ContainerStats setMemoryEvents(dir, &ret) + assert.Equal(t, testItem.low, ret.Memory.Events.Low) assert.Equal(t, testItem.high, ret.Memory.Events.High) assert.Equal(t, testItem.max, ret.Memory.Events.Max) assert.Equal(t, testItem.oom, ret.Memory.Events.Oom) @@ -446,6 +448,7 @@ func TestSetMemoryEventsFileNotFound(t *testing.T) { var ret info.ContainerStats setMemoryEvents("/nonexistent/path", &ret) + assert.Equal(t, uint64(0), ret.Memory.Events.Low) assert.Equal(t, uint64(0), ret.Memory.Events.High) assert.Equal(t, uint64(0), ret.Memory.Events.Max) assert.Equal(t, uint64(0), ret.Memory.Events.Oom) diff --git a/docs/storage/prometheus.md b/docs/storage/prometheus.md index 7010e86f29..c05bef10f5 100644 --- a/docs/storage/prometheus.md +++ b/docs/storage/prometheus.md @@ -56,6 +56,7 @@ Metric name | Type | Description | Unit (where applicable) | option parameter | `container_memory_bandwidth_bytes` | Gauge | Total memory bandwidth usage statistics for container counted with RDT Memory Bandwidth Monitoring (MBM). | bytes | resctrl | `container_memory_bandwidth_local_bytes` | Gauge | Local memory bandwidth usage statistics for container counted with RDT Memory Bandwidth Monitoring (MBM). | bytes | resctrl | `container_memory_cache` | Gauge | Total page cache memory | bytes | memory | +`container_memory_events_low_total` | Counter | Cumulative count of memory.low breach events for the container | | memory | `container_memory_events_oom_group_kill_total` | Counter | Cumulative count of memory OOM group kill events for the container | | memory | `container_memory_events_oom_kill_total` | Counter | Cumulative count of memory OOM kill events for the container | | memory | `container_memory_events_oom_total` | Counter | Cumulative count of memory allocation failure events for the container | | memory | diff --git a/info/v1/container.go b/info/v1/container.go index fa710cad2f..e6f332a207 100644 --- a/info/v1/container.go +++ b/info/v1/container.go @@ -491,6 +491,7 @@ type MemoryStats struct { } type MemoryEvents struct { + Low uint64 `json:"low"` High uint64 `json:"high"` Max uint64 `json:"max"` Oom uint64 `json:"oom"` diff --git a/integration/tests/metrics/prometheus_test.go b/integration/tests/metrics/prometheus_test.go index 0297b642bc..4f4d2523f9 100644 --- a/integration/tests/metrics/prometheus_test.go +++ b/integration/tests/metrics/prometheus_test.go @@ -117,6 +117,7 @@ func TestCoreMemoryMetricsExist(t *testing.T) { "container_memory_working_set_bytes", "container_memory_cache", "container_memory_rss", + "container_memory_events_low_total", "container_memory_events_high_total", "container_memory_events_max_total", "container_memory_events_oom_total", @@ -290,6 +291,7 @@ func TestMetricsHaveCorrectTypes(t *testing.T) { "container_cpu_system_seconds_total", "container_network_receive_bytes_total", "container_network_transmit_bytes_total", + "container_memory_events_low_total", "container_memory_events_high_total", "container_memory_events_max_total", "container_memory_events_oom_total", diff --git a/metrics/prometheus.go b/metrics/prometheus.go index 02526fdf07..6399362e31 100644 --- a/metrics/prometheus.go +++ b/metrics/prometheus.go @@ -518,6 +518,13 @@ func NewPrometheusCollector(i infoProvider, f ContainerLabelsFunc, includedMetri }, } }, + }, { + name: "container_memory_events_low_total", + help: "Cumulative count of memory.low breach events for the container", + valueType: prometheus.CounterValue, + getValues: func(s *info.ContainerStats) metricValues { + return metricValues{{value: float64(s.Memory.Events.Low), timestamp: s.Timestamp}} + }, }, { name: "container_memory_events_high_total", help: "Cumulative count of memory.high throttle events for the container", diff --git a/metrics/prometheus_fake.go b/metrics/prometheus_fake.go index 6dcda802a5..3ed8c3e59e 100644 --- a/metrics/prometheus_fake.go +++ b/metrics/prometheus_fake.go @@ -396,6 +396,7 @@ func (p testSubcontainersInfoProvider) GetRequestedContainersInfo(string, v2.Req }, }, Events: info.MemoryEvents{ + Low: 4, High: 42, Max: 5, Oom: 3, diff --git a/metrics/testdata/prometheus_metrics b/metrics/testdata/prometheus_metrics index fe0efaf95a..57970ce492 100644 --- a/metrics/testdata/prometheus_metrics +++ b/metrics/testdata/prometheus_metrics @@ -163,6 +163,9 @@ container_last_seen{container_env_foo_env="prod",container_label_foo_label="bar" # HELP container_memory_cache Number of bytes of page cache memory. # TYPE container_memory_cache gauge container_memory_cache{container_env_foo_env="prod",container_label_foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 14 1395066363000 +# HELP container_memory_events_low_total Cumulative count of memory.low breach events for the container +# TYPE container_memory_events_low_total counter +container_memory_events_low_total{container_env_foo_env="prod",container_label_foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 4 1395066363000 # HELP container_memory_events_high_total Cumulative count of memory.high throttle events for the container # TYPE container_memory_events_high_total counter container_memory_events_high_total{container_env_foo_env="prod",container_label_foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 42 1395066363000 diff --git a/metrics/testdata/prometheus_metrics_whitelist_filtered b/metrics/testdata/prometheus_metrics_whitelist_filtered index b0064aa9c6..9eb33dd122 100644 --- a/metrics/testdata/prometheus_metrics_whitelist_filtered +++ b/metrics/testdata/prometheus_metrics_whitelist_filtered @@ -163,6 +163,9 @@ container_last_seen{container_env_foo_env="prod",id="testcontainer",image="test" # HELP container_memory_cache Number of bytes of page cache memory. # TYPE container_memory_cache gauge container_memory_cache{container_env_foo_env="prod",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 14 1395066363000 +# HELP container_memory_events_low_total Cumulative count of memory.low breach events for the container +# TYPE container_memory_events_low_total counter +container_memory_events_low_total{container_env_foo_env="prod",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 4 1395066363000 # HELP container_memory_events_high_total Cumulative count of memory.high throttle events for the container # TYPE container_memory_events_high_total counter container_memory_events_high_total{container_env_foo_env="prod",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 42 1395066363000 From 994d9a5077cb912a91d60bea87cd4d001e13d632 Mon Sep 17 00:00:00 2001 From: Nils Eriksson Date: Fri, 12 Jun 2026 09:37:43 +0100 Subject: [PATCH 3/3] Clarify help text for low and oom event metrics Align metric descriptions with kernel cgroup v2 semantics: - low counts reclaim despite usage being under the memory.low boundary, so "reclaim events" instead of "breach events". - oom counts entering the OOM state, not page allocation faults; the previous wording was nearly identical to the existing container_memory_failures_total help text. --- docs/storage/prometheus.md | 4 ++-- metrics/prometheus.go | 4 ++-- metrics/testdata/prometheus_metrics | 4 ++-- metrics/testdata/prometheus_metrics_whitelist_filtered | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/storage/prometheus.md b/docs/storage/prometheus.md index c05bef10f5..feb567dcae 100644 --- a/docs/storage/prometheus.md +++ b/docs/storage/prometheus.md @@ -56,10 +56,10 @@ Metric name | Type | Description | Unit (where applicable) | option parameter | `container_memory_bandwidth_bytes` | Gauge | Total memory bandwidth usage statistics for container counted with RDT Memory Bandwidth Monitoring (MBM). | bytes | resctrl | `container_memory_bandwidth_local_bytes` | Gauge | Local memory bandwidth usage statistics for container counted with RDT Memory Bandwidth Monitoring (MBM). | bytes | resctrl | `container_memory_cache` | Gauge | Total page cache memory | bytes | memory | -`container_memory_events_low_total` | Counter | Cumulative count of memory.low breach events for the container | | memory | +`container_memory_events_low_total` | Counter | Cumulative count of memory.low reclaim events for the container | | memory | `container_memory_events_oom_group_kill_total` | Counter | Cumulative count of memory OOM group kill events for the container | | memory | `container_memory_events_oom_kill_total` | Counter | Cumulative count of memory OOM kill events for the container | | memory | -`container_memory_events_oom_total` | Counter | Cumulative count of memory allocation failure events for the container | | memory | +`container_memory_events_oom_total` | Counter | Cumulative count of memory OOM events for the container | | memory | `container_memory_failcnt` | Counter | Number of memory usage hits limits | | memory | `container_memory_failures_total` | Counter | Cumulative count of memory allocation failures | | memory | `container_memory_file_dirty_bytes` | Gauge | File cache that has been modified but not yet written back to disk (cgroup v2) | bytes | memory | diff --git a/metrics/prometheus.go b/metrics/prometheus.go index 6399362e31..f152815fc2 100644 --- a/metrics/prometheus.go +++ b/metrics/prometheus.go @@ -520,7 +520,7 @@ func NewPrometheusCollector(i infoProvider, f ContainerLabelsFunc, includedMetri }, }, { name: "container_memory_events_low_total", - help: "Cumulative count of memory.low breach events for the container", + help: "Cumulative count of memory.low reclaim events for the container", valueType: prometheus.CounterValue, getValues: func(s *info.ContainerStats) metricValues { return metricValues{{value: float64(s.Memory.Events.Low), timestamp: s.Timestamp}} @@ -541,7 +541,7 @@ func NewPrometheusCollector(i infoProvider, f ContainerLabelsFunc, includedMetri }, }, { name: "container_memory_events_oom_total", - help: "Cumulative count of memory allocation failure events for the container", + help: "Cumulative count of memory OOM events for the container", valueType: prometheus.CounterValue, getValues: func(s *info.ContainerStats) metricValues { return metricValues{{value: float64(s.Memory.Events.Oom), timestamp: s.Timestamp}} diff --git a/metrics/testdata/prometheus_metrics b/metrics/testdata/prometheus_metrics index 57970ce492..bbd8debc13 100644 --- a/metrics/testdata/prometheus_metrics +++ b/metrics/testdata/prometheus_metrics @@ -163,7 +163,7 @@ container_last_seen{container_env_foo_env="prod",container_label_foo_label="bar" # HELP container_memory_cache Number of bytes of page cache memory. # TYPE container_memory_cache gauge container_memory_cache{container_env_foo_env="prod",container_label_foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 14 1395066363000 -# HELP container_memory_events_low_total Cumulative count of memory.low breach events for the container +# HELP container_memory_events_low_total Cumulative count of memory.low reclaim events for the container # TYPE container_memory_events_low_total counter container_memory_events_low_total{container_env_foo_env="prod",container_label_foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 4 1395066363000 # HELP container_memory_events_high_total Cumulative count of memory.high throttle events for the container @@ -172,7 +172,7 @@ container_memory_events_high_total{container_env_foo_env="prod",container_label_ # HELP container_memory_events_max_total Cumulative count of memory.max limit hit events for the container # TYPE container_memory_events_max_total counter container_memory_events_max_total{container_env_foo_env="prod",container_label_foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 5 1395066363000 -# HELP container_memory_events_oom_total Cumulative count of memory allocation failure events for the container +# HELP container_memory_events_oom_total Cumulative count of memory OOM events for the container # TYPE container_memory_events_oom_total counter container_memory_events_oom_total{container_env_foo_env="prod",container_label_foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 3 1395066363000 # HELP container_memory_events_oom_kill_total Cumulative count of memory OOM kill events for the container diff --git a/metrics/testdata/prometheus_metrics_whitelist_filtered b/metrics/testdata/prometheus_metrics_whitelist_filtered index 9eb33dd122..5253568f16 100644 --- a/metrics/testdata/prometheus_metrics_whitelist_filtered +++ b/metrics/testdata/prometheus_metrics_whitelist_filtered @@ -163,7 +163,7 @@ container_last_seen{container_env_foo_env="prod",id="testcontainer",image="test" # HELP container_memory_cache Number of bytes of page cache memory. # TYPE container_memory_cache gauge container_memory_cache{container_env_foo_env="prod",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 14 1395066363000 -# HELP container_memory_events_low_total Cumulative count of memory.low breach events for the container +# HELP container_memory_events_low_total Cumulative count of memory.low reclaim events for the container # TYPE container_memory_events_low_total counter container_memory_events_low_total{container_env_foo_env="prod",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 4 1395066363000 # HELP container_memory_events_high_total Cumulative count of memory.high throttle events for the container @@ -172,7 +172,7 @@ container_memory_events_high_total{container_env_foo_env="prod",id="testcontaine # HELP container_memory_events_max_total Cumulative count of memory.max limit hit events for the container # TYPE container_memory_events_max_total counter container_memory_events_max_total{container_env_foo_env="prod",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 5 1395066363000 -# HELP container_memory_events_oom_total Cumulative count of memory allocation failure events for the container +# HELP container_memory_events_oom_total Cumulative count of memory OOM events for the container # TYPE container_memory_events_oom_total counter container_memory_events_oom_total{container_env_foo_env="prod",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 3 1395066363000 # HELP container_memory_events_oom_kill_total Cumulative count of memory OOM kill events for the container