diff --git a/common/schedule.go b/common/schedule.go index 2a2bec24..34e86416 100644 --- a/common/schedule.go +++ b/common/schedule.go @@ -19,6 +19,7 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/events" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -70,6 +71,16 @@ type DrainParameters struct { // on pods afterwards. ForceEviction bool GracePeriodSeconds *int64 + // optional recorder to emit events on evicted pods + Recorder events.EventRecorder + Node *corev1.Node +} + +func recordDrainEvent(recorder events.EventRecorder, pod *corev1.Pod, node *corev1.Node, reason, note string) { + if recorder == nil { + return + } + recorder.Eventf(pod, node, corev1.EventTypeNormal, reason, "Drain", note) } func EnsureDrain(ctx context.Context, node *corev1.Node, log logr.Logger, params DrainParameters) (bool, error) { @@ -99,6 +110,9 @@ func EnsureDrain(ctx context.Context, node *corev1.Node, log logr.Logger, params } if len(podsToForceDelete) > 0 { log.Info("Force deleting pods that exceeded grace period", "count", len(podsToForceDelete), "node", node.Name) + for i := range podsToForceDelete { + recordDrainEvent(params.Recorder, &podsToForceDelete[i], node, "ForceDeleting", "Force deleting pod that exceeded grace period during node drain") + } gracePeriodZero := int64(0) err = deletePods(ctx, params.Client, podsToForceDelete, &gracePeriodZero) if err != nil { @@ -112,6 +126,9 @@ func EnsureDrain(ctx context.Context, node *corev1.Node, log logr.Logger, params if err != nil { return false, err } + for i := range active { + recordDrainEvent(params.Recorder, &active[i], node, "Evicting", "Evicting pod during node drain") + } if version == none { log.Info("Going to delete pods from node.", "count", len(active), "node", node.Name) err = deletePods(ctx, params.Client, active, params.GracePeriodSeconds) diff --git a/esx/runnable.go b/esx/runnable.go index fe49a1b8..92e7ed5d 100644 --- a/esx/runnable.go +++ b/esx/runnable.go @@ -14,6 +14,7 @@ import ( "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" + "k8s.io/client-go/tools/events" "sigs.k8s.io/controller-runtime/pkg/client" "github.com/sapcc/maintenance-controller/common" @@ -27,8 +28,9 @@ import ( type Runnable struct { client.Client - Log logr.Logger - Conf *rest.Config + Log logr.Logger + Conf *rest.Config + Recorder events.EventRecorder } func (r *Runnable) NeedLeaderElection() bool { @@ -178,6 +180,8 @@ func (r *Runnable) ShutdownNodes(ctx context.Context, conf *Config, esx *Host) e }, ForceEviction: conf.Intervals.PodEviction.Force, GracePeriodSeconds: nodeGracePeriod(node), + Recorder: r.Recorder, + Node: node, }, ) if err != nil { diff --git a/kubernikus/node_controller.go b/kubernikus/node_controller.go index b7a4560d..9e36ba9e 100644 --- a/kubernikus/node_controller.go +++ b/kubernikus/node_controller.go @@ -19,6 +19,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" + "k8s.io/client-go/tools/events" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" @@ -69,9 +70,10 @@ func (r *NodeReconciler) loadConfig() (Config, error) { // NodeReconciler reconciles a Node object. type NodeReconciler struct { client.Client - Conf *rest.Config - Log logr.Logger - Scheme *runtime.Scheme + Conf *rest.Config + Log logr.Logger + Scheme *runtime.Scheme + Recorder events.EventRecorder } // Reconcile reconciles the given request. @@ -119,6 +121,8 @@ func (r *NodeReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl. Timeout: conf.Intervals.PodEviction.Timeout, }, ForceEviction: conf.Intervals.PodEviction.Force, + Recorder: r.Recorder, + Node: node, }, ) if err != nil { diff --git a/main.go b/main.go index 12938e46..159f473a 100644 --- a/main.go +++ b/main.go @@ -203,10 +203,11 @@ func setupReconcilers(mgr manager.Manager, cfg *reconcilerConfig) error { if cfg.enableKubernikusMaintenance { setupLog.Info("Kubernikus integration is enabled") if err := (&kubernikus.NodeReconciler{ - Client: mgr.GetClient(), - Log: ctrl.Log.WithName("controllers").WithName("kubernikus"), - Scheme: mgr.GetScheme(), - Conf: mgr.GetConfig(), + Client: mgr.GetClient(), + Log: ctrl.Log.WithName("controllers").WithName("kubernikus"), + Scheme: mgr.GetScheme(), + Conf: mgr.GetConfig(), + Recorder: mgr.GetEventRecorder("kubernikus-maintenance"), }).SetupWithManager(mgr); err != nil { return fmt.Errorf("failed to setup kubernikus node reconciler: %w", err) } @@ -215,9 +216,10 @@ func setupReconcilers(mgr manager.Manager, cfg *reconcilerConfig) error { if cfg.enableESXMaintenance { setupLog.Info("ESX integration is enabled") controller := esx.Runnable{ - Client: mgr.GetClient(), - Conf: mgr.GetConfig(), - Log: ctrl.Log.WithName("controllers").WithName("esx"), + Client: mgr.GetClient(), + Conf: mgr.GetConfig(), + Log: ctrl.Log.WithName("controllers").WithName("esx"), + Recorder: mgr.GetEventRecorder("esx-maintenance"), } if err := mgr.Add(&controller); err != nil { return fmt.Errorf("failed to create ESX reconciler: %w", err) diff --git a/plugin/impl/eviction.go b/plugin/impl/eviction.go index 0e5e9aa3..39f62519 100644 --- a/plugin/impl/eviction.go +++ b/plugin/impl/eviction.go @@ -92,6 +92,8 @@ func (e *Eviction) Trigger(params plugin.Parameters) error { Client: params.Client, Clientset: params.Clientset, ForceEviction: e.ForceEviction, + Recorder: params.Recorder, + Node: params.Node, }) if err != nil { return err