Expose remaining cgroup v2 memory.events counters (oom, oom_kill, oom_group_kill, low)

docs/storage/prometheus.md

@@ -56,6 +56,10 @@ Metric name | Type | Description | Unit (where applicable) | option parameter |
 `container_memory_bandwidth_bytes` | Gauge | Total memory bandwidth usage statistics for container counted with RDT Memory Bandwidth Monitoring (MBM). | bytes | resctrl |
 `container_memory_bandwidth_local_bytes` | Gauge | Local memory bandwidth usage statistics for container counted with RDT Memory Bandwidth Monitoring (MBM). | bytes | resctrl |
 `container_memory_cache` | Gauge | Total page cache memory | bytes | memory |
+`container_memory_events_low_total` | Counter | Cumulative count of memory.low reclaim events for the container | | memory |
+`container_memory_events_oom_group_kill_total` | Counter | Cumulative count of memory OOM group kill events for the container | | memory |
+`container_memory_events_oom_kill_total` | Counter | Cumulative count of memory OOM kill events for the container | | memory |
+`container_memory_events_oom_total` | Counter | Cumulative count of memory OOM events for the container | | memory |
 `container_memory_failcnt` | Counter | Number of memory usage hits limits | | memory |
 `container_memory_failures_total` | Counter | Cumulative count of memory allocation failures | | memory |
 `container_memory_file_dirty_bytes` | Gauge | File cache that has been modified but not yet written back to disk (cgroup v2) | bytes | memory |

integration/tests/metrics/prometheus_test.go

@@ -117,8 +117,12 @@ func TestCoreMemoryMetricsExist(t *testing.T) {
 		"container_memory_working_set_bytes",
 		"container_memory_cache",
 		"container_memory_rss",
+		"container_memory_events_low_total",
 		"container_memory_events_high_total",
 		"container_memory_events_max_total",
+		"container_memory_events_oom_total",
+		"container_memory_events_oom_kill_total",
+		"container_memory_events_oom_group_kill_total",
 	}
 
 	for _, name := range memoryMetrics {
@@ -287,8 +291,12 @@ func TestMetricsHaveCorrectTypes(t *testing.T) {
 		"container_cpu_system_seconds_total",
 		"container_network_receive_bytes_total",
 		"container_network_transmit_bytes_total",
+		"container_memory_events_low_total",
 		"container_memory_events_high_total",
 		"container_memory_events_max_total",
+		"container_memory_events_oom_total",
+		"container_memory_events_oom_kill_total",
+		"container_memory_events_oom_group_kill_total",
 	}
 	for _, name := range counterMetrics {
 		if mf, ok := families[name]; ok {

lib/container/libcontainer/handler.go

@@ -903,12 +903,24 @@ func setMemoryEvents(cgroupPath string, ret *info.ContainerStats) {
 	if ret.Memory == nil {
 		ret.Memory = &info.MemoryStats{}
 	}
+	if val, err := fscommon.GetValueByKey(cgroupPath, "memory.events", "low"); err == nil {
+		ret.Memory.Events.Low = val
+	}
 	if val, err := fscommon.GetValueByKey(cgroupPath, "memory.events", "high"); err == nil {
 		ret.Memory.Events.High = val
 	}
 	if val, err := fscommon.GetValueByKey(cgroupPath, "memory.events", "max"); err == nil {
 		ret.Memory.Events.Max = val
 	}
+	if val, err := fscommon.GetValueByKey(cgroupPath, "memory.events", "oom"); err == nil {
+		ret.Memory.Events.Oom = val
+	}
+	if val, err := fscommon.GetValueByKey(cgroupPath, "memory.events", "oom_kill"); err == nil {
+		ret.Memory.Events.OomKill = val
+	}
+	if val, err := fscommon.GetValueByKey(cgroupPath, "memory.events", "oom_group_kill"); err == nil {
+		ret.Memory.Events.OomGroupKill = val
+	}
 }
 
 func setCPUSetStats(s *cgroups.Stats, ret *info.ContainerStats) {

lib/container/libcontainer/handler_test.go

@@ -394,30 +394,34 @@ func TestSetMemoryEvents(t *testing.T) {
 	defer func() { cgroups.TestMode = false }()
 
 	testData := []struct {
-		name    string
-		content string
-		high    uint64
-		max     uint64
+		name         string
+		content      string
+		low          uint64
+		high         uint64
+		max          uint64
+		oom          uint64
+		oomKill      uint64
+		oomGroupKill uint64
 	}{
 		{
 			"all counters",
-			"low 0\nhigh 42\nmax 5\noom 1\noom_kill 0\noom_group_kill 0\n",
-			42, 5,
+			"low 7\nhigh 42\nmax 5\noom 3\noom_kill 2\noom_group_kill 1\n",
+			7, 42, 5, 3, 2, 1,
 		},
 		{
 			"zeros",
 			"low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0\n",
-			0, 0,
+			0, 0, 0, 0, 0, 0,
 		},
 		{
 			"high only",
 			"low 0\nhigh 238\nmax 0\noom 0\noom_kill 0\noom_group_kill 0\n",
-			238, 0,
+			0, 238, 0, 0, 0, 0,
 		},
 		{
 			"empty file",
 			"",
-			0, 0,
+			0, 0, 0, 0, 0, 0,
 		},
 	}
 
@@ -430,8 +434,12 @@ func TestSetMemoryEvents(t *testing.T) {
 			var ret info.ContainerStats
 			setMemoryEvents(dir, &ret)
 
+			assert.Equal(t, testItem.low, ret.Memory.Events.Low)
 			assert.Equal(t, testItem.high, ret.Memory.Events.High)
 			assert.Equal(t, testItem.max, ret.Memory.Events.Max)
+			assert.Equal(t, testItem.oom, ret.Memory.Events.Oom)
+			assert.Equal(t, testItem.oomKill, ret.Memory.Events.OomKill)
+			assert.Equal(t, testItem.oomGroupKill, ret.Memory.Events.OomGroupKill)
 		})
 	}
 }
@@ -440,6 +448,10 @@ func TestSetMemoryEventsFileNotFound(t *testing.T) {
 	var ret info.ContainerStats
 	setMemoryEvents("/nonexistent/path", &ret)
 
+	assert.Equal(t, uint64(0), ret.Memory.Events.Low)
 	assert.Equal(t, uint64(0), ret.Memory.Events.High)
 	assert.Equal(t, uint64(0), ret.Memory.Events.Max)
+	assert.Equal(t, uint64(0), ret.Memory.Events.Oom)
+	assert.Equal(t, uint64(0), ret.Memory.Events.OomKill)
+	assert.Equal(t, uint64(0), ret.Memory.Events.OomGroupKill)
 }

lib/metrics/prometheus.go

@@ -592,6 +592,13 @@ func NewPrometheusCollector(i infoProvider, f ContainerLabelsFunc, includedMetri
 						},
 					}
 				},
+			}, {
+				name:      "container_memory_events_low_total",
+				help:      "Cumulative count of memory.low reclaim events for the container",
+				valueType: prometheus.CounterValue,
+				getValues: func(s *info.ContainerStats) metricValues {
+					return metricValues{{value: float64(s.Memory.Events.Low), timestamp: s.Timestamp}}
+				},
 			}, {
 				name:      "container_memory_events_high_total",
 				help:      "Cumulative count of memory.high throttle events for the container",
@@ -612,6 +619,27 @@ func NewPrometheusCollector(i infoProvider, f ContainerLabelsFunc, includedMetri
 					}
 					return metricValues{{value: float64(s.Memory.Events.Max), timestamp: s.Timestamp}}
 				},
+			}, {
+				name:      "container_memory_events_oom_total",
+				help:      "Cumulative count of memory OOM events for the container",
+				valueType: prometheus.CounterValue,
+				getValues: func(s *info.ContainerStats) metricValues {
+					return metricValues{{value: float64(s.Memory.Events.Oom), timestamp: s.Timestamp}}
+				},
+			}, {
+				name:      "container_memory_events_oom_kill_total",
+				help:      "Cumulative count of memory OOM kill events for the container",
+				valueType: prometheus.CounterValue,
+				getValues: func(s *info.ContainerStats) metricValues {
+					return metricValues{{value: float64(s.Memory.Events.OomKill), timestamp: s.Timestamp}}
+				},
+			}, {
+				name:      "container_memory_events_oom_group_kill_total",
+				help:      "Cumulative count of memory OOM group kill events for the container",
+				valueType: prometheus.CounterValue,
+				getValues: func(s *info.ContainerStats) metricValues {
+					return metricValues{{value: float64(s.Memory.Events.OomGroupKill), timestamp: s.Timestamp}}
+				},
 			}, {
 				name:      "container_memory_file_dirty_bytes",
 				help:      "Number of bytes of file cache that has been modified but not yet written back to disk.",

lib/metrics/prometheus_fake_test.go

@@ -395,8 +395,12 @@ func (p testSubcontainersInfoProvider) GetRequestedContainersInfo(string, info.R
 							},
 						},
 						Events: info.MemoryEvents{
-							High: 42,
-							Max:  5,
+							Low:          4,
+							High:         42,
+							Max:          5,
+							Oom:          3,
+							OomKill:      2,
+							OomGroupKill: 1,
 						},
 					},
 					Hugetlb: map[string]info.HugetlbStats{

lib/metrics/testdata/prometheus_metrics

@@ -163,12 +163,24 @@ container_last_seen{container_env_foo_env="prod",container_label_foo_label="bar"
 # HELP container_memory_cache Number of bytes of page cache memory.
 # TYPE container_memory_cache gauge
 container_memory_cache{container_env_foo_env="prod",container_label_foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 14 1395066363000
+# HELP container_memory_events_low_total Cumulative count of memory.low reclaim events for the container
+# TYPE container_memory_events_low_total counter
+container_memory_events_low_total{container_env_foo_env="prod",container_label_foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 4 1395066363000
 # HELP container_memory_events_high_total Cumulative count of memory.high throttle events for the container
 # TYPE container_memory_events_high_total counter
 container_memory_events_high_total{container_env_foo_env="prod",container_label_foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 42 1395066363000
 # HELP container_memory_events_max_total Cumulative count of memory.max limit hit events for the container
 # TYPE container_memory_events_max_total counter
 container_memory_events_max_total{container_env_foo_env="prod",container_label_foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 5 1395066363000
+# HELP container_memory_events_oom_total Cumulative count of memory OOM events for the container
+# TYPE container_memory_events_oom_total counter
+container_memory_events_oom_total{container_env_foo_env="prod",container_label_foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 3 1395066363000
+# HELP container_memory_events_oom_kill_total Cumulative count of memory OOM kill events for the container
+# TYPE container_memory_events_oom_kill_total counter
+container_memory_events_oom_kill_total{container_env_foo_env="prod",container_label_foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 2 1395066363000
+# HELP container_memory_events_oom_group_kill_total Cumulative count of memory OOM group kill events for the container
+# TYPE container_memory_events_oom_group_kill_total counter
+container_memory_events_oom_group_kill_total{container_env_foo_env="prod",container_label_foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 1 1395066363000
 # HELP container_memory_failcnt Number of memory usage hits limits
 # TYPE container_memory_failcnt counter
 container_memory_failcnt{container_env_foo_env="prod",container_label_foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 0 1395066363000

lib/metrics/testdata/prometheus_metrics_whitelist_filtered

@@ -163,12 +163,24 @@ container_last_seen{container_env_foo_env="prod",id="testcontainer",image="test"
 # HELP container_memory_cache Number of bytes of page cache memory.
 # TYPE container_memory_cache gauge
 container_memory_cache{container_env_foo_env="prod",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 14 1395066363000
+# HELP container_memory_events_low_total Cumulative count of memory.low reclaim events for the container
+# TYPE container_memory_events_low_total counter
+container_memory_events_low_total{container_env_foo_env="prod",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 4 1395066363000
 # HELP container_memory_events_high_total Cumulative count of memory.high throttle events for the container
 # TYPE container_memory_events_high_total counter
 container_memory_events_high_total{container_env_foo_env="prod",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 42 1395066363000
 # HELP container_memory_events_max_total Cumulative count of memory.max limit hit events for the container
 # TYPE container_memory_events_max_total counter
 container_memory_events_max_total{container_env_foo_env="prod",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 5 1395066363000
+# HELP container_memory_events_oom_total Cumulative count of memory OOM events for the container
+# TYPE container_memory_events_oom_total counter
+container_memory_events_oom_total{container_env_foo_env="prod",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 3 1395066363000
+# HELP container_memory_events_oom_kill_total Cumulative count of memory OOM kill events for the container
+# TYPE container_memory_events_oom_kill_total counter
+container_memory_events_oom_kill_total{container_env_foo_env="prod",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 2 1395066363000
+# HELP container_memory_events_oom_group_kill_total Cumulative count of memory OOM group kill events for the container
+# TYPE container_memory_events_oom_group_kill_total counter
+container_memory_events_oom_group_kill_total{container_env_foo_env="prod",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 1 1395066363000
 # HELP container_memory_failcnt Number of memory usage hits limits
 # TYPE container_memory_failcnt counter
 container_memory_failcnt{container_env_foo_env="prod",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 0 1395066363000

lib/model/container.go

@@ -493,8 +493,12 @@ type MemoryStats struct {
 }
 
 type MemoryEvents struct {
-	High uint64 `json:"high"`
-	Max  uint64 `json:"max"`
+	Low          uint64 `json:"low"`
+	High         uint64 `json:"high"`
+	Max          uint64 `json:"max"`
+	Oom          uint64 `json:"oom"`
+	OomKill      uint64 `json:"oom_kill"`
+	OomGroupKill uint64 `json:"oom_group_kill"`
 }
 
 type CPUSetStats struct {