containerd: surface per-container stats for sandboxed runtimes (gVisor/runsc)

container/containerd/handler.go

@@ -28,6 +28,7 @@ import (
 	"github.com/containerd/errdefs"
 	"github.com/opencontainers/cgroups"
 	specs "github.com/opencontainers/runtime-spec/specs-go"
+	"k8s.io/klog/v2"
 
 	"github.com/google/cadvisor/container"
 	"github.com/google/cadvisor/container/common"
@@ -55,6 +56,10 @@ type containerdContainerHandler struct {
 
 	libcontainerHandler *containerlibcontainer.Handler
 	client              ContainerdClient
+	// sandbox is non-nil when this container is managed by a sandboxed runtime
+	// (e.g. gVisor/runsc) whose processes run outside the host cgroup, so the
+	// host cgroup is empty and stats are instead overlaid from the runtime.
+	sandbox sandboxStatsProvider
 }
 
 var _ container.ContainerHandler = &containerdContainerHandler{}
@@ -149,6 +154,7 @@ func newContainerdContainerHandler(
 		reference:           containerReference,
 		libcontainerHandler: libcontainerHandler,
 		client:              client,
+		sandbox:             sandboxProviderFor(cntr.Runtime.Name),
 	}
 	if !cntr.CreatedAt.IsZero() && !cntr.CreatedAt.Before(time.Unix(0, 0)) {
 		handler.creationTime = cntr.CreatedAt
@@ -217,6 +223,16 @@ func (h *containerdContainerHandler) GetStats() (*info.ContainerStats, error) {
 		return stats, err
 	}
 
+	// Sandboxed runtimes (e.g. gVisor) run the workload outside the host
+	// cgroup, so the stats above are zero. Overlay the real per-container
+	// values reported by the runtime. Best-effort: any failure leaves the
+	// base stats untouched so a transient error never fails a scrape.
+	if h.sandbox != nil {
+		if err := h.sandbox.overlay(context.Background(), h.reference.Id, stats, h.includedMetrics); err != nil {
+			klog.V(4).Infof("containerd: %s stats unavailable for %q, keeping cgroup-derived values: %v", h.sandbox.name(), h.reference.Name, err)
+		}
+	}
+
 	// Get filesystem stats.
 	err = h.getFsStats(stats)
 	return stats, err

container/containerd/runsc.go

@@ -0,0 +1,187 @@
+// Copyright 2026 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build linux
+
+// runscStatsProvider is the sandboxStatsProvider for gVisor (runsc). It sources
+// per-container stats from `runsc events --stats <id>`, the same interface
+// containerd's CRI stats path uses for the kubelet /stats/summary endpoint.
+// Unlike the generic cgroup metrics proto, runsc events also reports
+// per-interface network counters from the sandbox's netstack (which never
+// appear in the host /proc or cgroup), so runsc gets a dedicated provider.
+//
+// See gVisor google/gvisor#13067 (problem) and #13070 (host-side compat dirs
+// that make these containers discoverable by cAdvisor in the first place).
+package containerd
+
+import (
+	"context"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"os/exec"
+	"time"
+
+	"github.com/google/cadvisor/container"
+	info "github.com/google/cadvisor/info/v1"
+)
+
+var (
+	argRunscBinary = flag.String("runsc", "runsc", "path to the runsc binary, used to collect per-container stats for gVisor containers")
+	argRunscRoot   = flag.String("runsc_root", "/run/containerd/runsc/k8s.io", "runsc root directory (containerd's runsc state dir for the configured namespace), used to collect per-container stats for gVisor containers")
+)
+
+// runscStatsTimeout bounds a single `runsc events --stats` invocation.
+const runscStatsTimeout = 5 * time.Second
+
+type runscStatsProvider struct{}
+
+func (runscStatsProvider) name() string { return "runsc" }
+
+func (runscStatsProvider) overlay(ctx context.Context, id string, stats *info.ContainerStats, includedMetrics container.MetricSet) error {
+	ev, err := runscContainerStats(ctx, id)
+	if err != nil {
+		return err
+	}
+	applyRunscStats(stats, ev, includedMetrics)
+	return nil
+}
+
+// runscEvent mirrors the JSON emitted by `runsc events --stats <id>` (gVisor's
+// runsc/boot.Event). Only the fields consumed here are declared.
+type runscEvent struct {
+	Data runscEventData `json:"data"`
+}
+
+type runscEventData struct {
+	CPU               runscCPU                `json:"cpu"`
+	Memory            runscMemory             `json:"memory"`
+	Pids              runscPids               `json:"pids"`
+	NetworkInterfaces []runscNetworkInterface `json:"network_interfaces"`
+}
+
+type runscCPU struct {
+	Usage runscCPUUsage `json:"usage"`
+}
+
+type runscCPUUsage struct {
+	// Nanoseconds.
+	Kernel uint64 `json:"kernel"`
+	User   uint64 `json:"user"`
+	Total  uint64 `json:"total"`
+}
+
+type runscMemory struct {
+	Cache uint64           `json:"cache"`
+	Usage runscMemoryEntry `json:"usage"`
+}
+
+type runscMemoryEntry struct {
+	Usage uint64 `json:"usage"`
+	Max   uint64 `json:"max"`
+}
+
+type runscPids struct {
+	Current uint64 `json:"current"`
+}
+
+// runscNetworkInterface mirrors runsc/boot.NetworkInterface, which has no JSON
+// tags -- so the wire keys are the exported Go field names.
+type runscNetworkInterface struct {
+	Name      string
+	RxBytes   uint64
+	RxPackets uint64
+	RxErrors  uint64
+	RxDropped uint64
+	TxBytes   uint64
+	TxPackets uint64
+	TxErrors  uint64
+	TxDropped uint64
+}
+
+// runscContainerStats runs `runsc --root <root> events --stats <id>` once and
+// parses its JSON output.
+func runscContainerStats(ctx context.Context, id string) (*runscEvent, error) {
+	ctx, cancel := context.WithTimeout(ctx, runscStatsTimeout)
+	defer cancel()
+
+	cmd := exec.CommandContext(ctx, *argRunscBinary, "--root", *argRunscRoot, "events", "--stats", id)
+	out, err := cmd.Output()
+	if err != nil {
+		return nil, fmt.Errorf("running %q events --stats %s (root %q): %w", *argRunscBinary, id, *argRunscRoot, err)
+	}
+	var ev runscEvent
+	if err := json.Unmarshal(out, &ev); err != nil {
+		return nil, fmt.Errorf("parsing runsc events output for %s: %w", id, err)
+	}
+	return &ev, nil
+}
+
+// applyRunscStats overwrites the cgroup-derived (zero) values in stats with the
+// per-container values gVisor reports, honoring includedMetrics. Only the
+// metrics gVisor reliably provides are overlaid; the rest are left as the
+// libcontainer base so consumers that read them degrade gracefully.
+func applyRunscStats(stats *info.ContainerStats, ev *runscEvent, includedMetrics container.MetricSet) {
+	d := ev.Data
+
+	if includedMetrics.Has(container.CpuUsageMetrics) {
+		stats.Cpu.Usage.Total = d.CPU.Usage.Total
+		stats.Cpu.Usage.User = d.CPU.Usage.User
+		stats.Cpu.Usage.System = d.CPU.Usage.Kernel
+	}
+
+	if includedMetrics.Has(container.MemoryUsageMetrics) {
+		stats.Memory.Usage = d.Memory.Usage.Usage
+		stats.Memory.MaxUsage = d.Memory.Usage.Max
+		stats.Memory.Cache = d.Memory.Cache
+		// gVisor does not break out inactive_file, so approximate the
+		// working set as usage minus page cache (clamped at >= 0).
+		if d.Memory.Usage.Usage > d.Memory.Cache {
+			stats.Memory.WorkingSet = d.Memory.Usage.Usage - d.Memory.Cache
+		} else {
+			stats.Memory.WorkingSet = d.Memory.Usage.Usage
+		}
+	}
+
+	if includedMetrics.Has(container.ProcessMetrics) {
+		stats.Processes.ProcessCount = d.Pids.Current
+	}
+
+	if includedMetrics.Has(container.NetworkUsageMetrics) && len(d.NetworkInterfaces) > 0 {
+		ifaces := make([]info.InterfaceStats, 0, len(d.NetworkInterfaces))
+		for _, ni := range d.NetworkInterfaces {
+			ifaces = append(ifaces, info.InterfaceStats{
+				Name:      ni.Name,
+				RxBytes:   ni.RxBytes,
+				RxPackets: ni.RxPackets,
+				RxErrors:  ni.RxErrors,
+				RxDropped: ni.RxDropped,
+				TxBytes:   ni.TxBytes,
+				TxPackets: ni.TxPackets,
+				TxErrors:  ni.TxErrors,
+				TxDropped: ni.TxDropped,
+			})
+		}
+		stats.Network.Interfaces = ifaces
+		// cAdvisor's inline InterfaceStats reports the primary interface;
+		// prefer the first non-loopback one.
+		stats.Network.InterfaceStats = ifaces[0]
+		for _, is := range ifaces {
+			if is.Name != "lo" {
+				stats.Network.InterfaceStats = is
+				break
+			}
+		}
+	}
+}

container/containerd/sandbox.go

@@ -0,0 +1,84 @@
+// Copyright 2026 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build linux
+
+// Per-container stats for sandboxed runtimes.
+//
+// Some container runtimes run the workload's processes inside a sandbox -- a
+// user-space kernel (gVisor/runsc) or a VM (Kata, Firecracker) -- rather than
+// directly on the host. For those, the per-container host cgroup that cAdvisor
+// reads is empty, so libcontainer's GetStats returns zero CPU/memory/network/
+// pids. The runtime, however, holds the real per-container accounting and
+// exposes it out of band (the same source containerd's CRI stats path consumes
+// for the kubelet /stats/summary endpoint).
+//
+// A sandboxStatsProvider knows how to fetch those values for one such runtime
+// and overlay them onto the (zero) libcontainer base. Detection is by the
+// containerd runtime name; runc and other host-cgroup runtimes have no provider
+// and keep the standard libcontainer path unchanged.
+package containerd
+
+import (
+	"context"
+	"strings"
+
+	"github.com/google/cadvisor/container"
+	info "github.com/google/cadvisor/info/v1"
+)
+
+// sandboxStatsProvider supplies per-container stats for a sandboxed runtime
+// whose processes do not live in the host cgroup.
+type sandboxStatsProvider interface {
+	// name identifies the provider, for logging.
+	name() string
+
+	// overlay fetches per-container stats for id and overlays them onto stats,
+	// honoring includedMetrics. It should leave fields it cannot populate as
+	// the libcontainer base. A returned error signals the caller to keep the
+	// base stats (best-effort: a transient failure must not fail a scrape).
+	overlay(ctx context.Context, id string, stats *info.ContainerStats, includedMetrics container.MetricSet) error
+}
+
+// sandboxRuntime binds a containerd runtime-name prefix to its stats provider.
+type sandboxRuntime struct {
+	// prefix is matched against containerd's container Runtime.Name, e.g.
+	// "io.containerd.runsc" matches "io.containerd.runsc.v1".
+	prefix   string
+	provider sandboxStatsProvider
+}
+
+// sandboxRuntimes is the registry of runtimes whose host cgroup is empty and
+// whose stats must come from the runtime instead. To support a new runtime
+// (e.g. Kata or Firecracker), add an entry here with a provider -- the handler
+// wiring is runtime-agnostic. A generic provider backed by containerd's task
+// Metrics API (which every shim implements, returning the standard cgroup
+// metrics proto) could serve any sandboxed runtime for CPU/memory/pids; runsc
+// uses a dedicated provider because it also reports per-interface network
+// stats, which the cgroup metrics proto does not carry.
+var sandboxRuntimes = []sandboxRuntime{
+	{prefix: "io.containerd.runsc", provider: runscStatsProvider{}},
+}
+
+// sandboxProviderFor returns the stats provider for a containerd runtime name,
+// or nil if the runtime's processes live in the host cgroup (e.g. runc) and the
+// standard libcontainer cgroup read is authoritative.
+func sandboxProviderFor(runtimeName string) sandboxStatsProvider {
+	for _, r := range sandboxRuntimes {
+		if strings.HasPrefix(runtimeName, r.prefix) {
+			return r.provider
+		}
+	}
+	return nil
+}