Emit Kubernetes events on pods during eviction

common/schedule.go

@@ -19,6 +19,7 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/events"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
@@ -70,6 +71,16 @@ type DrainParameters struct {
 	// on pods afterwards.
 	ForceEviction      bool
 	GracePeriodSeconds *int64
+	// optional recorder to emit events on evicted pods
+	Recorder events.EventRecorder
+	Node     *corev1.Node
+}
+
+func recordDrainEvent(recorder events.EventRecorder, pod *corev1.Pod, node *corev1.Node, reason, note string) {
+	if recorder == nil {
+		return
+	}
+	recorder.Eventf(pod, node, corev1.EventTypeNormal, reason, "Drain", note)
 }
 
 func EnsureDrain(ctx context.Context, node *corev1.Node, log logr.Logger, params DrainParameters) (bool, error) {
@@ -99,6 +110,9 @@ func EnsureDrain(ctx context.Context, node *corev1.Node, log logr.Logger, params
 		}
 		if len(podsToForceDelete) > 0 {
 			log.Info("Force deleting pods that exceeded grace period", "count", len(podsToForceDelete), "node", node.Name)
+			for i := range podsToForceDelete {
+				recordDrainEvent(params.Recorder, &podsToForceDelete[i], node, "ForceDeleting", "Force deleting pod that exceeded grace period during node drain")
+			}
 			gracePeriodZero := int64(0)
 			err = deletePods(ctx, params.Client, podsToForceDelete, &gracePeriodZero)
 			if err != nil {
@@ -112,6 +126,9 @@ func EnsureDrain(ctx context.Context, node *corev1.Node, log logr.Logger, params
 		if err != nil {
 			return false, err
 		}
+		for i := range active {
+			recordDrainEvent(params.Recorder, &active[i], node, "Evicting", "Evicting pod during node drain")
+		}
 		if version == none {
 			log.Info("Going to delete pods from node.", "count", len(active), "node", node.Name)
 			err = deletePods(ctx, params.Client, active, params.GracePeriodSeconds)

esx/runnable.go

@@ -14,6 +14,7 @@ import (
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/events"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	"github.com/sapcc/maintenance-controller/common"
@@ -27,8 +28,9 @@ import (
 
 type Runnable struct {
 	client.Client
-	Log  logr.Logger
-	Conf *rest.Config
+	Log      logr.Logger
+	Conf     *rest.Config
+	Recorder events.EventRecorder
 }
 
 func (r *Runnable) NeedLeaderElection() bool {
@@ -178,6 +180,8 @@ func (r *Runnable) ShutdownNodes(ctx context.Context, conf *Config, esx *Host) e
 				},
 				ForceEviction:      conf.Intervals.PodEviction.Force,
 				GracePeriodSeconds: nodeGracePeriod(node),
+				Recorder:           r.Recorder,
+				Node:               node,
 			},
 		)
 		if err != nil {

kubernikus/node_controller.go

@@ -19,6 +19,7 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/events"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/builder"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -69,9 +70,10 @@ func (r *NodeReconciler) loadConfig() (Config, error) {
 // NodeReconciler reconciles a Node object.
 type NodeReconciler struct {
 	client.Client
-	Conf   *rest.Config
-	Log    logr.Logger
-	Scheme *runtime.Scheme
+	Conf     *rest.Config
+	Log      logr.Logger
+	Scheme   *runtime.Scheme
+	Recorder events.EventRecorder
 }
 
 // Reconcile reconciles the given request.
@@ -119,6 +121,8 @@ func (r *NodeReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.
 					Timeout: conf.Intervals.PodEviction.Timeout,
 				},
 				ForceEviction: conf.Intervals.PodEviction.Force,
+				Recorder:      r.Recorder,
+				Node:          node,
 			},
 		)
 		if err != nil {

main.go

@@ -203,10 +203,11 @@ func setupReconcilers(mgr manager.Manager, cfg *reconcilerConfig) error {
 	if cfg.enableKubernikusMaintenance {
 		setupLog.Info("Kubernikus integration is enabled")
 		if err := (&kubernikus.NodeReconciler{
-			Client: mgr.GetClient(),
-			Log:    ctrl.Log.WithName("controllers").WithName("kubernikus"),
-			Scheme: mgr.GetScheme(),
-			Conf:   mgr.GetConfig(),
+			Client:   mgr.GetClient(),
+			Log:      ctrl.Log.WithName("controllers").WithName("kubernikus"),
+			Scheme:   mgr.GetScheme(),
+			Conf:     mgr.GetConfig(),
+			Recorder: mgr.GetEventRecorder("kubernikus-maintenance"),
 		}).SetupWithManager(mgr); err != nil {
 			return fmt.Errorf("failed to setup kubernikus node reconciler: %w", err)
 		}
@@ -215,9 +216,10 @@ func setupReconcilers(mgr manager.Manager, cfg *reconcilerConfig) error {
 	if cfg.enableESXMaintenance {
 		setupLog.Info("ESX integration is enabled")
 		controller := esx.Runnable{
-			Client: mgr.GetClient(),
-			Conf:   mgr.GetConfig(),
-			Log:    ctrl.Log.WithName("controllers").WithName("esx"),
+			Client:   mgr.GetClient(),
+			Conf:     mgr.GetConfig(),
+			Log:      ctrl.Log.WithName("controllers").WithName("esx"),
+			Recorder: mgr.GetEventRecorder("esx-maintenance"),
 		}
 		if err := mgr.Add(&controller); err != nil {
 			return fmt.Errorf("failed to create ESX reconciler: %w", err)

plugin/impl/eviction.go

@@ -92,6 +92,8 @@ func (e *Eviction) Trigger(params plugin.Parameters) error {
 			Client:        params.Client,
 			Clientset:     params.Clientset,
 			ForceEviction: e.ForceEviction,
+			Recorder:      params.Recorder,
+			Node:          params.Node,
 		})
 		if err != nil {
 			return err