helm: derive memcached's -m flag from resources overrides to prevent OOM kills

operations/helm/charts/mimir-distributed/CHANGELOG.md

@@ -38,6 +38,7 @@ Entries should include a reference to the Pull Request that introduced the chang
 * [ENHANCEMENT] Upgrade rollout-operator chart to [0.38.1](https://github.com/grafana/helm-charts/blob/main/charts/rollout-operator/README.md#upgrade-of-grafana-rollout-operator--v0380). Note required actions for upgrading the rollout-operator chart. #16129
 * [ENHANCEMENT] Ruler: make the memory ballast size configurable via `ruler.memBallastSizeBytes` and allow disabling it by setting the value to `0`. The default stays at 1GiB. A ballast larger than the ruler's `GOMEMLIMIT` causes permanent GC thrashing, so deployments setting a lower `GOMEMLIMIT` need to lower or disable the ballast. #16159
 * [BUGFIX]: Fix bug in `ScaledObject` templates when using `kedaAutoscaling.fallback` #15793
+* [BUGFIX] `chunks-cache`, `index-cache`, `metadata-cache`, `results-cache`: derive memcached's own memory ceiling (`-m` flag) from a user-provided `resources` override, instead of always using `allocatedMemory`. Previously, overriding `resources` to a memory limit lower than `allocatedMemory` implied could cause memcached to be OOM killed. #16348
 
 
 ## 6.1.0

operations/helm/charts/mimir-distributed/ci/offline/test-memcached-resources-override-values.yaml

@@ -0,0 +1,31 @@
+# Test that a user-provided .resources override for a memcached-backed cache is reflected in
+# memcached's own memory ceiling (the "-m" flag), instead of always using .allocatedMemory.
+# Without this, overriding resources to a value lower than .allocatedMemory implies can cause the
+# memcached process to be OOM killed by the container runtime.
+# ref grafana/mimir#9737
+kubeVersionOverride: "1.32"
+
+chunks-cache:
+  enabled: true
+  allocatedMemory: 8192 # unused: resources.limits.memory below takes precedence
+  resources:
+    limits:
+      memory: 2000Mi # -m should be floor(2000 * 10 / 12) = 1666, not the allocatedMemory default
+    requests:
+      cpu: 100m
+      memory: 1500Mi
+
+index-cache:
+  enabled: true
+  allocatedMemory: 2048 # unused: resources.requests.memory below takes precedence (no limits set)
+  resources:
+    requests:
+      cpu: 100m
+      memory: 1200Mi # -m should be floor(1200 * 10 / 12) = 1000
+
+metadata-cache:
+  enabled: true
+  allocatedMemory: 512 # -m should fall back to this: the override below has no memory field at all
+  resources:
+    requests:
+      cpu: 100m

operations/helm/charts/mimir-distributed/templates/memcached/_memcached-statefulset.tpl

@@ -96,6 +96,23 @@ spec:
           image: {{ .repository }}:{{ .tag }}
           imagePullPolicy: {{ .pullPolicy }}
           {{- end }}
+          {{- /* $allocatedMemory (MiB) becomes the "-m" flag below; derive it from a .resources override if set, so it doesn't exceed the container's memory limit. */}}
+          {{- $allocatedMemory := .allocatedMemory }}
+          {{- if .resources }}
+            {{- /* dig one level at a time since .resources.limits/.requests may be explicitly null, which a multi-key dig can't traverse. */}}
+            {{- $limits := dig "limits" nil .resources }}
+            {{- $requests := dig "requests" nil .resources }}
+            {{- $limitsMem := "" }}
+            {{- if $limits }}{{- $limitsMem = dig "memory" "" $limits }}{{- end }}
+            {{- $requestsMem := "" }}
+            {{- if $requests }}{{- $requestsMem = dig "memory" "" $requests }}{{- end }}
+            {{- $memValue := $limitsMem | default $requestsMem }}
+            {{- if $memValue }}
+              {{- $memBytes := include "mimir.siToBytes" (dict "value" $memValue) | int64 }}
+              {{- $memMi := div $memBytes 1048576 }}
+              {{- $allocatedMemory = div (mul $memMi 10) 12 }}
+            {{- end }}
+          {{- end }}
           resources:
           {{- if .resources }}
             {{- toYaml .resources | nindent 12 }}
@@ -112,7 +129,7 @@ spec:
             - containerPort: {{ .port }}
               name: client
           args:
-            - -m {{ .allocatedMemory }}
+            - -m {{ $allocatedMemory }}
             - --extended=modern{{ with .extraExtendedOptions }},{{ . }}{{ end }}
             - -I {{ .maxItemMemory }}m
             - -c {{ .connectionLimit }}

operations/helm/charts/mimir-distributed/values.yaml

@@ -2737,6 +2737,8 @@ chunks-cache:
 
   # -- Resource requests and limits for the chunks-cache
   # By default a safe memory limit will be requested based on allocatedMemory value (floor (* 1.2 allocatedMemory)).
+  # If overridden here, memcached's own memory ceiling (-m) is instead derived from the memory limit set below
+  # (falling back to the memory request if no limit is set), so the two stay consistent.
   resources: null
 
   # -- Service annotations and labels
@@ -2845,6 +2847,8 @@ index-cache:
 
   # -- Resource requests and limits for the index-cache
   # By default a safe memory limit will be requested based on allocatedMemory value (floor (* 1.2 allocatedMemory)).
+  # If overridden here, memcached's own memory ceiling (-m) is instead derived from the memory limit set below
+  # (falling back to the memory request if no limit is set), so the two stay consistent.
   resources: null
 
   # -- Service annotations and labels
@@ -2953,6 +2957,8 @@ metadata-cache:
 
   # -- Resource requests and limits for the metadata-cache
   # By default a safe memory limit will be requested based on allocatedMemory value (floor (* 1.2 allocatedMemory)).
+  # If overridden here, memcached's own memory ceiling (-m) is instead derived from the memory limit set below
+  # (falling back to the memory request if no limit is set), so the two stay consistent.
   resources: null
 
   # -- Service annotations and labels
@@ -3061,6 +3067,8 @@ results-cache:
 
   # -- Resource requests and limits for the results-cache
   # By default a safe memory limit will be requested based on allocatedMemory value (floor (* 1.2 allocatedMemory)).
+  # If overridden here, memcached's own memory ceiling (-m) is instead derived from the memory limit set below
+  # (falling back to the memory request if no limit is set), so the two stay consistent.
   resources: null
 
   # -- Service annotations and labels