diff --git a/container/containerd/handler.go b/container/containerd/handler.go index e91e894dc6..f8f92247fc 100644 --- a/container/containerd/handler.go +++ b/container/containerd/handler.go @@ -28,6 +28,7 @@ import ( "github.com/containerd/errdefs" "github.com/opencontainers/cgroups" specs "github.com/opencontainers/runtime-spec/specs-go" + "k8s.io/klog/v2" "github.com/google/cadvisor/container" "github.com/google/cadvisor/container/common" @@ -55,6 +56,10 @@ type containerdContainerHandler struct { libcontainerHandler *containerlibcontainer.Handler client ContainerdClient + // sandbox is non-nil when this container is managed by a sandboxed runtime + // (e.g. gVisor/runsc) whose processes run outside the host cgroup, so the + // host cgroup is empty and stats are instead overlaid from the runtime. + sandbox sandboxStatsProvider } var _ container.ContainerHandler = &containerdContainerHandler{} @@ -149,6 +154,7 @@ func newContainerdContainerHandler( reference: containerReference, libcontainerHandler: libcontainerHandler, client: client, + sandbox: sandboxProviderFor(cntr.Runtime.Name), } if !cntr.CreatedAt.IsZero() && !cntr.CreatedAt.Before(time.Unix(0, 0)) { handler.creationTime = cntr.CreatedAt @@ -217,6 +223,16 @@ func (h *containerdContainerHandler) GetStats() (*info.ContainerStats, error) { return stats, err } + // Sandboxed runtimes (e.g. gVisor) run the workload outside the host + // cgroup, so the stats above are zero. Overlay the real per-container + // values reported by the runtime. Best-effort: any failure leaves the + // base stats untouched so a transient error never fails a scrape. + if h.sandbox != nil { + if err := h.sandbox.overlay(context.Background(), h.reference.Id, stats, h.includedMetrics); err != nil { + klog.V(4).Infof("containerd: %s stats unavailable for %q, keeping cgroup-derived values: %v", h.sandbox.name(), h.reference.Name, err) + } + } + // Get filesystem stats. err = h.getFsStats(stats) return stats, err diff --git a/container/containerd/runsc.go b/container/containerd/runsc.go new file mode 100644 index 0000000000..71f3bc1cae --- /dev/null +++ b/container/containerd/runsc.go @@ -0,0 +1,187 @@ +// Copyright 2026 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build linux + +// runscStatsProvider is the sandboxStatsProvider for gVisor (runsc). It sources +// per-container stats from `runsc events --stats `, the same interface +// containerd's CRI stats path uses for the kubelet /stats/summary endpoint. +// Unlike the generic cgroup metrics proto, runsc events also reports +// per-interface network counters from the sandbox's netstack (which never +// appear in the host /proc or cgroup), so runsc gets a dedicated provider. +// +// See gVisor google/gvisor#13067 (problem) and #13070 (host-side compat dirs +// that make these containers discoverable by cAdvisor in the first place). +package containerd + +import ( + "context" + "encoding/json" + "flag" + "fmt" + "os/exec" + "time" + + "github.com/google/cadvisor/container" + info "github.com/google/cadvisor/info/v1" +) + +var ( + argRunscBinary = flag.String("runsc", "runsc", "path to the runsc binary, used to collect per-container stats for gVisor containers") + argRunscRoot = flag.String("runsc_root", "/run/containerd/runsc/k8s.io", "runsc root directory (containerd's runsc state dir for the configured namespace), used to collect per-container stats for gVisor containers") +) + +// runscStatsTimeout bounds a single `runsc events --stats` invocation. +const runscStatsTimeout = 5 * time.Second + +type runscStatsProvider struct{} + +func (runscStatsProvider) name() string { return "runsc" } + +func (runscStatsProvider) overlay(ctx context.Context, id string, stats *info.ContainerStats, includedMetrics container.MetricSet) error { + ev, err := runscContainerStats(ctx, id) + if err != nil { + return err + } + applyRunscStats(stats, ev, includedMetrics) + return nil +} + +// runscEvent mirrors the JSON emitted by `runsc events --stats ` (gVisor's +// runsc/boot.Event). Only the fields consumed here are declared. +type runscEvent struct { + Data runscEventData `json:"data"` +} + +type runscEventData struct { + CPU runscCPU `json:"cpu"` + Memory runscMemory `json:"memory"` + Pids runscPids `json:"pids"` + NetworkInterfaces []runscNetworkInterface `json:"network_interfaces"` +} + +type runscCPU struct { + Usage runscCPUUsage `json:"usage"` +} + +type runscCPUUsage struct { + // Nanoseconds. + Kernel uint64 `json:"kernel"` + User uint64 `json:"user"` + Total uint64 `json:"total"` +} + +type runscMemory struct { + Cache uint64 `json:"cache"` + Usage runscMemoryEntry `json:"usage"` +} + +type runscMemoryEntry struct { + Usage uint64 `json:"usage"` + Max uint64 `json:"max"` +} + +type runscPids struct { + Current uint64 `json:"current"` +} + +// runscNetworkInterface mirrors runsc/boot.NetworkInterface, which has no JSON +// tags -- so the wire keys are the exported Go field names. +type runscNetworkInterface struct { + Name string + RxBytes uint64 + RxPackets uint64 + RxErrors uint64 + RxDropped uint64 + TxBytes uint64 + TxPackets uint64 + TxErrors uint64 + TxDropped uint64 +} + +// runscContainerStats runs `runsc --root events --stats ` once and +// parses its JSON output. +func runscContainerStats(ctx context.Context, id string) (*runscEvent, error) { + ctx, cancel := context.WithTimeout(ctx, runscStatsTimeout) + defer cancel() + + cmd := exec.CommandContext(ctx, *argRunscBinary, "--root", *argRunscRoot, "events", "--stats", id) + out, err := cmd.Output() + if err != nil { + return nil, fmt.Errorf("running %q events --stats %s (root %q): %w", *argRunscBinary, id, *argRunscRoot, err) + } + var ev runscEvent + if err := json.Unmarshal(out, &ev); err != nil { + return nil, fmt.Errorf("parsing runsc events output for %s: %w", id, err) + } + return &ev, nil +} + +// applyRunscStats overwrites the cgroup-derived (zero) values in stats with the +// per-container values gVisor reports, honoring includedMetrics. Only the +// metrics gVisor reliably provides are overlaid; the rest are left as the +// libcontainer base so consumers that read them degrade gracefully. +func applyRunscStats(stats *info.ContainerStats, ev *runscEvent, includedMetrics container.MetricSet) { + d := ev.Data + + if includedMetrics.Has(container.CpuUsageMetrics) { + stats.Cpu.Usage.Total = d.CPU.Usage.Total + stats.Cpu.Usage.User = d.CPU.Usage.User + stats.Cpu.Usage.System = d.CPU.Usage.Kernel + } + + if includedMetrics.Has(container.MemoryUsageMetrics) { + stats.Memory.Usage = d.Memory.Usage.Usage + stats.Memory.MaxUsage = d.Memory.Usage.Max + stats.Memory.Cache = d.Memory.Cache + // gVisor does not break out inactive_file, so approximate the + // working set as usage minus page cache (clamped at >= 0). + if d.Memory.Usage.Usage > d.Memory.Cache { + stats.Memory.WorkingSet = d.Memory.Usage.Usage - d.Memory.Cache + } else { + stats.Memory.WorkingSet = d.Memory.Usage.Usage + } + } + + if includedMetrics.Has(container.ProcessMetrics) { + stats.Processes.ProcessCount = d.Pids.Current + } + + if includedMetrics.Has(container.NetworkUsageMetrics) && len(d.NetworkInterfaces) > 0 { + ifaces := make([]info.InterfaceStats, 0, len(d.NetworkInterfaces)) + for _, ni := range d.NetworkInterfaces { + ifaces = append(ifaces, info.InterfaceStats{ + Name: ni.Name, + RxBytes: ni.RxBytes, + RxPackets: ni.RxPackets, + RxErrors: ni.RxErrors, + RxDropped: ni.RxDropped, + TxBytes: ni.TxBytes, + TxPackets: ni.TxPackets, + TxErrors: ni.TxErrors, + TxDropped: ni.TxDropped, + }) + } + stats.Network.Interfaces = ifaces + // cAdvisor's inline InterfaceStats reports the primary interface; + // prefer the first non-loopback one. + stats.Network.InterfaceStats = ifaces[0] + for _, is := range ifaces { + if is.Name != "lo" { + stats.Network.InterfaceStats = is + break + } + } + } +} diff --git a/container/containerd/sandbox.go b/container/containerd/sandbox.go new file mode 100644 index 0000000000..e51129923a --- /dev/null +++ b/container/containerd/sandbox.go @@ -0,0 +1,84 @@ +// Copyright 2026 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build linux + +// Per-container stats for sandboxed runtimes. +// +// Some container runtimes run the workload's processes inside a sandbox -- a +// user-space kernel (gVisor/runsc) or a VM (Kata, Firecracker) -- rather than +// directly on the host. For those, the per-container host cgroup that cAdvisor +// reads is empty, so libcontainer's GetStats returns zero CPU/memory/network/ +// pids. The runtime, however, holds the real per-container accounting and +// exposes it out of band (the same source containerd's CRI stats path consumes +// for the kubelet /stats/summary endpoint). +// +// A sandboxStatsProvider knows how to fetch those values for one such runtime +// and overlay them onto the (zero) libcontainer base. Detection is by the +// containerd runtime name; runc and other host-cgroup runtimes have no provider +// and keep the standard libcontainer path unchanged. +package containerd + +import ( + "context" + "strings" + + "github.com/google/cadvisor/container" + info "github.com/google/cadvisor/info/v1" +) + +// sandboxStatsProvider supplies per-container stats for a sandboxed runtime +// whose processes do not live in the host cgroup. +type sandboxStatsProvider interface { + // name identifies the provider, for logging. + name() string + + // overlay fetches per-container stats for id and overlays them onto stats, + // honoring includedMetrics. It should leave fields it cannot populate as + // the libcontainer base. A returned error signals the caller to keep the + // base stats (best-effort: a transient failure must not fail a scrape). + overlay(ctx context.Context, id string, stats *info.ContainerStats, includedMetrics container.MetricSet) error +} + +// sandboxRuntime binds a containerd runtime-name prefix to its stats provider. +type sandboxRuntime struct { + // prefix is matched against containerd's container Runtime.Name, e.g. + // "io.containerd.runsc" matches "io.containerd.runsc.v1". + prefix string + provider sandboxStatsProvider +} + +// sandboxRuntimes is the registry of runtimes whose host cgroup is empty and +// whose stats must come from the runtime instead. To support a new runtime +// (e.g. Kata or Firecracker), add an entry here with a provider -- the handler +// wiring is runtime-agnostic. A generic provider backed by containerd's task +// Metrics API (which every shim implements, returning the standard cgroup +// metrics proto) could serve any sandboxed runtime for CPU/memory/pids; runsc +// uses a dedicated provider because it also reports per-interface network +// stats, which the cgroup metrics proto does not carry. +var sandboxRuntimes = []sandboxRuntime{ + {prefix: "io.containerd.runsc", provider: runscStatsProvider{}}, +} + +// sandboxProviderFor returns the stats provider for a containerd runtime name, +// or nil if the runtime's processes live in the host cgroup (e.g. runc) and the +// standard libcontainer cgroup read is authoritative. +func sandboxProviderFor(runtimeName string) sandboxStatsProvider { + for _, r := range sandboxRuntimes { + if strings.HasPrefix(runtimeName, r.prefix) { + return r.provider + } + } + return nil +}