diff --git a/docs/storage/prometheus.md b/docs/storage/prometheus.md index f7a84ca91f..feb567dcae 100644 --- a/docs/storage/prometheus.md +++ b/docs/storage/prometheus.md @@ -56,6 +56,10 @@ Metric name | Type | Description | Unit (where applicable) | option parameter | `container_memory_bandwidth_bytes` | Gauge | Total memory bandwidth usage statistics for container counted with RDT Memory Bandwidth Monitoring (MBM). | bytes | resctrl | `container_memory_bandwidth_local_bytes` | Gauge | Local memory bandwidth usage statistics for container counted with RDT Memory Bandwidth Monitoring (MBM). | bytes | resctrl | `container_memory_cache` | Gauge | Total page cache memory | bytes | memory | +`container_memory_events_low_total` | Counter | Cumulative count of memory.low reclaim events for the container | | memory | +`container_memory_events_oom_group_kill_total` | Counter | Cumulative count of memory OOM group kill events for the container | | memory | +`container_memory_events_oom_kill_total` | Counter | Cumulative count of memory OOM kill events for the container | | memory | +`container_memory_events_oom_total` | Counter | Cumulative count of memory OOM events for the container | | memory | `container_memory_failcnt` | Counter | Number of memory usage hits limits | | memory | `container_memory_failures_total` | Counter | Cumulative count of memory allocation failures | | memory | `container_memory_file_dirty_bytes` | Gauge | File cache that has been modified but not yet written back to disk (cgroup v2) | bytes | memory | diff --git a/integration/tests/metrics/prometheus_test.go b/integration/tests/metrics/prometheus_test.go index 56cf4d1777..4f4d2523f9 100644 --- a/integration/tests/metrics/prometheus_test.go +++ b/integration/tests/metrics/prometheus_test.go @@ -117,8 +117,12 @@ func TestCoreMemoryMetricsExist(t *testing.T) { "container_memory_working_set_bytes", "container_memory_cache", "container_memory_rss", + "container_memory_events_low_total", "container_memory_events_high_total", "container_memory_events_max_total", + "container_memory_events_oom_total", + "container_memory_events_oom_kill_total", + "container_memory_events_oom_group_kill_total", } for _, name := range memoryMetrics { @@ -287,8 +291,12 @@ func TestMetricsHaveCorrectTypes(t *testing.T) { "container_cpu_system_seconds_total", "container_network_receive_bytes_total", "container_network_transmit_bytes_total", + "container_memory_events_low_total", "container_memory_events_high_total", "container_memory_events_max_total", + "container_memory_events_oom_total", + "container_memory_events_oom_kill_total", + "container_memory_events_oom_group_kill_total", } for _, name := range counterMetrics { if mf, ok := families[name]; ok { diff --git a/lib/container/libcontainer/handler.go b/lib/container/libcontainer/handler.go index a47f6d9f9c..7831a6f567 100644 --- a/lib/container/libcontainer/handler.go +++ b/lib/container/libcontainer/handler.go @@ -903,12 +903,24 @@ func setMemoryEvents(cgroupPath string, ret *info.ContainerStats) { if ret.Memory == nil { ret.Memory = &info.MemoryStats{} } + if val, err := fscommon.GetValueByKey(cgroupPath, "memory.events", "low"); err == nil { + ret.Memory.Events.Low = val + } if val, err := fscommon.GetValueByKey(cgroupPath, "memory.events", "high"); err == nil { ret.Memory.Events.High = val } if val, err := fscommon.GetValueByKey(cgroupPath, "memory.events", "max"); err == nil { ret.Memory.Events.Max = val } + if val, err := fscommon.GetValueByKey(cgroupPath, "memory.events", "oom"); err == nil { + ret.Memory.Events.Oom = val + } + if val, err := fscommon.GetValueByKey(cgroupPath, "memory.events", "oom_kill"); err == nil { + ret.Memory.Events.OomKill = val + } + if val, err := fscommon.GetValueByKey(cgroupPath, "memory.events", "oom_group_kill"); err == nil { + ret.Memory.Events.OomGroupKill = val + } } func setCPUSetStats(s *cgroups.Stats, ret *info.ContainerStats) { diff --git a/lib/container/libcontainer/handler_test.go b/lib/container/libcontainer/handler_test.go index 602bf25526..70d99fc29f 100644 --- a/lib/container/libcontainer/handler_test.go +++ b/lib/container/libcontainer/handler_test.go @@ -394,30 +394,34 @@ func TestSetMemoryEvents(t *testing.T) { defer func() { cgroups.TestMode = false }() testData := []struct { - name string - content string - high uint64 - max uint64 + name string + content string + low uint64 + high uint64 + max uint64 + oom uint64 + oomKill uint64 + oomGroupKill uint64 }{ { "all counters", - "low 0\nhigh 42\nmax 5\noom 1\noom_kill 0\noom_group_kill 0\n", - 42, 5, + "low 7\nhigh 42\nmax 5\noom 3\noom_kill 2\noom_group_kill 1\n", + 7, 42, 5, 3, 2, 1, }, { "zeros", "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0\n", - 0, 0, + 0, 0, 0, 0, 0, 0, }, { "high only", "low 0\nhigh 238\nmax 0\noom 0\noom_kill 0\noom_group_kill 0\n", - 238, 0, + 0, 238, 0, 0, 0, 0, }, { "empty file", "", - 0, 0, + 0, 0, 0, 0, 0, 0, }, } @@ -430,8 +434,12 @@ func TestSetMemoryEvents(t *testing.T) { var ret info.ContainerStats setMemoryEvents(dir, &ret) + assert.Equal(t, testItem.low, ret.Memory.Events.Low) assert.Equal(t, testItem.high, ret.Memory.Events.High) assert.Equal(t, testItem.max, ret.Memory.Events.Max) + assert.Equal(t, testItem.oom, ret.Memory.Events.Oom) + assert.Equal(t, testItem.oomKill, ret.Memory.Events.OomKill) + assert.Equal(t, testItem.oomGroupKill, ret.Memory.Events.OomGroupKill) }) } } @@ -440,6 +448,10 @@ func TestSetMemoryEventsFileNotFound(t *testing.T) { var ret info.ContainerStats setMemoryEvents("/nonexistent/path", &ret) + assert.Equal(t, uint64(0), ret.Memory.Events.Low) assert.Equal(t, uint64(0), ret.Memory.Events.High) assert.Equal(t, uint64(0), ret.Memory.Events.Max) + assert.Equal(t, uint64(0), ret.Memory.Events.Oom) + assert.Equal(t, uint64(0), ret.Memory.Events.OomKill) + assert.Equal(t, uint64(0), ret.Memory.Events.OomGroupKill) } diff --git a/lib/metrics/prometheus.go b/lib/metrics/prometheus.go index 2e2bbaac80..f1551ff583 100644 --- a/lib/metrics/prometheus.go +++ b/lib/metrics/prometheus.go @@ -592,6 +592,13 @@ func NewPrometheusCollector(i infoProvider, f ContainerLabelsFunc, includedMetri }, } }, + }, { + name: "container_memory_events_low_total", + help: "Cumulative count of memory.low reclaim events for the container", + valueType: prometheus.CounterValue, + getValues: func(s *info.ContainerStats) metricValues { + return metricValues{{value: float64(s.Memory.Events.Low), timestamp: s.Timestamp}} + }, }, { name: "container_memory_events_high_total", help: "Cumulative count of memory.high throttle events for the container", @@ -612,6 +619,27 @@ func NewPrometheusCollector(i infoProvider, f ContainerLabelsFunc, includedMetri } return metricValues{{value: float64(s.Memory.Events.Max), timestamp: s.Timestamp}} }, + }, { + name: "container_memory_events_oom_total", + help: "Cumulative count of memory OOM events for the container", + valueType: prometheus.CounterValue, + getValues: func(s *info.ContainerStats) metricValues { + return metricValues{{value: float64(s.Memory.Events.Oom), timestamp: s.Timestamp}} + }, + }, { + name: "container_memory_events_oom_kill_total", + help: "Cumulative count of memory OOM kill events for the container", + valueType: prometheus.CounterValue, + getValues: func(s *info.ContainerStats) metricValues { + return metricValues{{value: float64(s.Memory.Events.OomKill), timestamp: s.Timestamp}} + }, + }, { + name: "container_memory_events_oom_group_kill_total", + help: "Cumulative count of memory OOM group kill events for the container", + valueType: prometheus.CounterValue, + getValues: func(s *info.ContainerStats) metricValues { + return metricValues{{value: float64(s.Memory.Events.OomGroupKill), timestamp: s.Timestamp}} + }, }, { name: "container_memory_file_dirty_bytes", help: "Number of bytes of file cache that has been modified but not yet written back to disk.", diff --git a/lib/metrics/prometheus_fake_test.go b/lib/metrics/prometheus_fake_test.go index c5140f4ff3..090cd32020 100644 --- a/lib/metrics/prometheus_fake_test.go +++ b/lib/metrics/prometheus_fake_test.go @@ -395,8 +395,12 @@ func (p testSubcontainersInfoProvider) GetRequestedContainersInfo(string, info.R }, }, Events: info.MemoryEvents{ - High: 42, - Max: 5, + Low: 4, + High: 42, + Max: 5, + Oom: 3, + OomKill: 2, + OomGroupKill: 1, }, }, Hugetlb: map[string]info.HugetlbStats{ diff --git a/lib/metrics/testdata/prometheus_metrics b/lib/metrics/testdata/prometheus_metrics index 01f29ee0d2..bbd8debc13 100644 --- a/lib/metrics/testdata/prometheus_metrics +++ b/lib/metrics/testdata/prometheus_metrics @@ -163,12 +163,24 @@ container_last_seen{container_env_foo_env="prod",container_label_foo_label="bar" # HELP container_memory_cache Number of bytes of page cache memory. # TYPE container_memory_cache gauge container_memory_cache{container_env_foo_env="prod",container_label_foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 14 1395066363000 +# HELP container_memory_events_low_total Cumulative count of memory.low reclaim events for the container +# TYPE container_memory_events_low_total counter +container_memory_events_low_total{container_env_foo_env="prod",container_label_foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 4 1395066363000 # HELP container_memory_events_high_total Cumulative count of memory.high throttle events for the container # TYPE container_memory_events_high_total counter container_memory_events_high_total{container_env_foo_env="prod",container_label_foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 42 1395066363000 # HELP container_memory_events_max_total Cumulative count of memory.max limit hit events for the container # TYPE container_memory_events_max_total counter container_memory_events_max_total{container_env_foo_env="prod",container_label_foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 5 1395066363000 +# HELP container_memory_events_oom_total Cumulative count of memory OOM events for the container +# TYPE container_memory_events_oom_total counter +container_memory_events_oom_total{container_env_foo_env="prod",container_label_foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 3 1395066363000 +# HELP container_memory_events_oom_kill_total Cumulative count of memory OOM kill events for the container +# TYPE container_memory_events_oom_kill_total counter +container_memory_events_oom_kill_total{container_env_foo_env="prod",container_label_foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 2 1395066363000 +# HELP container_memory_events_oom_group_kill_total Cumulative count of memory OOM group kill events for the container +# TYPE container_memory_events_oom_group_kill_total counter +container_memory_events_oom_group_kill_total{container_env_foo_env="prod",container_label_foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 1 1395066363000 # HELP container_memory_failcnt Number of memory usage hits limits # TYPE container_memory_failcnt counter container_memory_failcnt{container_env_foo_env="prod",container_label_foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 0 1395066363000 diff --git a/lib/metrics/testdata/prometheus_metrics_whitelist_filtered b/lib/metrics/testdata/prometheus_metrics_whitelist_filtered index 9abb88d2d7..5253568f16 100644 --- a/lib/metrics/testdata/prometheus_metrics_whitelist_filtered +++ b/lib/metrics/testdata/prometheus_metrics_whitelist_filtered @@ -163,12 +163,24 @@ container_last_seen{container_env_foo_env="prod",id="testcontainer",image="test" # HELP container_memory_cache Number of bytes of page cache memory. # TYPE container_memory_cache gauge container_memory_cache{container_env_foo_env="prod",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 14 1395066363000 +# HELP container_memory_events_low_total Cumulative count of memory.low reclaim events for the container +# TYPE container_memory_events_low_total counter +container_memory_events_low_total{container_env_foo_env="prod",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 4 1395066363000 # HELP container_memory_events_high_total Cumulative count of memory.high throttle events for the container # TYPE container_memory_events_high_total counter container_memory_events_high_total{container_env_foo_env="prod",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 42 1395066363000 # HELP container_memory_events_max_total Cumulative count of memory.max limit hit events for the container # TYPE container_memory_events_max_total counter container_memory_events_max_total{container_env_foo_env="prod",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 5 1395066363000 +# HELP container_memory_events_oom_total Cumulative count of memory OOM events for the container +# TYPE container_memory_events_oom_total counter +container_memory_events_oom_total{container_env_foo_env="prod",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 3 1395066363000 +# HELP container_memory_events_oom_kill_total Cumulative count of memory OOM kill events for the container +# TYPE container_memory_events_oom_kill_total counter +container_memory_events_oom_kill_total{container_env_foo_env="prod",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 2 1395066363000 +# HELP container_memory_events_oom_group_kill_total Cumulative count of memory OOM group kill events for the container +# TYPE container_memory_events_oom_group_kill_total counter +container_memory_events_oom_group_kill_total{container_env_foo_env="prod",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 1 1395066363000 # HELP container_memory_failcnt Number of memory usage hits limits # TYPE container_memory_failcnt counter container_memory_failcnt{container_env_foo_env="prod",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 0 1395066363000 diff --git a/lib/model/container.go b/lib/model/container.go index b4beb2ba6a..46a6c716c8 100644 --- a/lib/model/container.go +++ b/lib/model/container.go @@ -493,8 +493,12 @@ type MemoryStats struct { } type MemoryEvents struct { - High uint64 `json:"high"` - Max uint64 `json:"max"` + Low uint64 `json:"low"` + High uint64 `json:"high"` + Max uint64 `json:"max"` + Oom uint64 `json:"oom"` + OomKill uint64 `json:"oom_kill"` + OomGroupKill uint64 `json:"oom_group_kill"` } type CPUSetStats struct {