diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 695916a..1c07307 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,2 +1,6 @@ # Default owner for everything in the repo * @densify-dev/cluster-kit + +# Kubex AI charts +/charts/kubex-connector/ @gasarekubex @shsma +/charts/kubex-ai-cdi/ @gasarekubex @shsma diff --git a/charts/container-optimization-data-forwarder/templates/configmap.yaml b/charts/container-optimization-data-forwarder/templates/configmap.yaml index a0f1207..22874b6 100644 --- a/charts/container-optimization-data-forwarder/templates/configmap.yaml +++ b/charts/container-optimization-data-forwarder/templates/configmap.yaml @@ -8,12 +8,15 @@ metadata: chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} release: {{ .Release.Name }} heritage: {{ .Release.Service }} +{{- $densifyUrl := .Values.config.forwarder.densify.url | required ".Values.config.forwarder.densify.url is required." }} +{{- $densifyHost := $densifyUrl.host | required ".Values.config.forwarder.densify.url.host is required." }} data: + kubex_host: {{ $densifyHost | quote }} + kubex_tenant_id: {{ index (splitList "." $densifyHost) 0 | quote }} + kubex_cluster_name: {{ (first .Values.config.clusters).name | quote }} config.yaml : | forwarder: densify: -{{- $densifyUrl := .Values.config.forwarder.densify.url | required ".Values.config.forwarder.densify.url is required." }} -{{- $densifyHost := $densifyUrl.host | required ".Values.config.forwarder.densify.url.host is required." }} url: {{- omit $densifyUrl "UserSecretName" | toYaml | trim | nindent 10 }} {{- if .Values.config.forwarder.densify.endpoint }} endpoint: {{ .Values.config.forwarder.densify.endpoint }} diff --git a/charts/kubex-ai-cdi/Chart.yaml b/charts/kubex-ai-cdi/Chart.yaml index 4c367af..5342320 100644 --- a/charts/kubex-ai-cdi/Chart.yaml +++ b/charts/kubex-ai-cdi/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: kubex-ai-cdi description: In-cluster Kubex cluster data interface service exposed over HTTP type: application -version: 0.1.8 +version: 0.1.9-rc.3 icon: https://kubex.ai/wp-content/uploads/kubex-logo-landscape.svg maintainers: -- email: support@kubex.ai - name: support + - email: support@kubex.ai + name: support diff --git a/charts/kubex-ai-cdi/values.yaml b/charts/kubex-ai-cdi/values.yaml index 14feb49..60eea2a 100644 --- a/charts/kubex-ai-cdi/values.yaml +++ b/charts/kubex-ai-cdi/values.yaml @@ -1,6 +1,6 @@ image: repository: densify/kubex-ai-cdi - tag: 0.1.8 + tag: 0.1.9-rc.3 pullPolicy: IfNotPresent replicaCount: 1 diff --git a/charts/kubex-automation-stack/CHANGELOG.md b/charts/kubex-automation-stack/CHANGELOG.md index 1f2c349..78d3867 100644 --- a/charts/kubex-automation-stack/CHANGELOG.md +++ b/charts/kubex-automation-stack/CHANGELOG.md @@ -2,6 +2,20 @@ All notable changes to the kubex-automation-stack chart will be documented in this file. +## [1.0.19] - 2026-07-03 + +### Added +- Added optional `kubex-connector` and `kubex-ai-cdi` subchart support to the stack +- Added stack-managed CDI service account, ClusterRole, and ClusterRoleBinding templates +- Added forwarder `ConfigMap` keys for shared Kubex host, tenant, and cluster name consumption + +### Changed +- Switched stack-managed connector tunnel wiring to `/tunnel/connect` +- Documented stack-managed connector/CDI configuration and RBAC behavior + +### Fixed +- Disabled the forwarder hook job in the OpenShift overlay so the cron-based collector path is used consistently +- Added OpenShift security context defaults for stack-managed connector and CDI deployments ## [1.0.18] - 2026-07-03 diff --git a/charts/kubex-automation-stack/Chart.lock b/charts/kubex-automation-stack/Chart.lock index 29464f8..874af2c 100644 --- a/charts/kubex-automation-stack/Chart.lock +++ b/charts/kubex-automation-stack/Chart.lock @@ -2,20 +2,26 @@ dependencies: - name: container-optimization-data-forwarder repository: https://densify-dev.github.io/helm-charts version: 4.0.17 +- name: kubex-connector + repository: https://densify-dev.github.io/helm-charts + version: 0.1.8 +- name: kubex-ai-cdi + repository: https://densify-dev.github.io/helm-charts + version: 0.1.8 - name: prometheus repository: https://prometheus-community.github.io/helm-charts - version: 29.14.0 + version: 29.17.0 - name: beyla repository: https://grafana.github.io/helm-charts - version: 1.16.8 + version: 1.16.10 - name: k8s-ephemeral-storage-metrics repository: https://densify-dev.github.io/helm-charts version: 1.21.0 - name: gpu-process-exporter repository: https://densify-dev.github.io/helm-charts - version: 1.0.0 + version: 1.1.0 - name: node-labeler repository: https://densify-dev.github.io/helm-charts version: 0.1.2 -digest: sha256:d4b36dc243779019f4a3277b17c487f1be80c39d5a6ce5a3b40979a42dbc5955 -generated: "2026-07-03T17:48:40.199770372Z" +digest: sha256:462fdc738b2cabfb00741487e2f519838d81ffc3c094d9662eb237b284f0108d +generated: "2026-07-17T16:01:39.083440872-04:00" diff --git a/charts/kubex-automation-stack/Chart.yaml b/charts/kubex-automation-stack/Chart.yaml index d7b3454..0c48ee9 100644 --- a/charts/kubex-automation-stack/Chart.yaml +++ b/charts/kubex-automation-stack/Chart.yaml @@ -1,13 +1,21 @@ apiVersion: v2 description: Kubex Collection Stack name: kubex-automation-stack -version: 1.0.18 +version: 1.0.19 type: application icon: https://kubex.ai/wp-content/uploads/kubex-logo-landscape.svg dependencies: - name: container-optimization-data-forwarder version: "4.*.*" repository: https://densify-dev.github.io/helm-charts + - name: kubex-connector + version: "0.1.*" + repository: https://densify-dev.github.io/helm-charts + condition: kubex-connector.enabled + - name: kubex-ai-cdi + version: "0.1.*" + repository: https://densify-dev.github.io/helm-charts + condition: kubex-ai-cdi.enabled - name: prometheus version: "29.*.*" repository: https://prometheus-community.github.io/helm-charts diff --git a/charts/kubex-automation-stack/README.md b/charts/kubex-automation-stack/README.md index a23df8a..26de803 100644 --- a/charts/kubex-automation-stack/README.md +++ b/charts/kubex-automation-stack/README.md @@ -114,7 +114,17 @@ The following table lists configuration parameters in `values-edit.yaml`. | `gpu-process-exporter.enabled` | | Enable GPU process exporter subchart (default: `true`) | | `beyla.enabled` | | Enable Grafana Beyla for application runtime detection (default: `true`) | | `k8s-ephemeral-storage-metrics.enabled` | | Enable ephemeral storage metrics collection (default: `true`) | -| `node-labeler.enabled` | | Enable optional node-labeler subchart to add Kubex Node Group labels to nodes; useful when nodes lack standard cloud provider pool/group labels (default: `false`) | +| `node-labeler.enabled` | | Enable optional node-labeler subchart to add Kubex Node Group labels to nodes; useful when nodes lack standard cloud provider node pool/group labels (default: `false`) | +| `kubex-connector.enabled` | | Enable optional connector subchart (default: `false`) | +| `kubex-ai-cdi.enabled` | | Enable optional kubex-ai-cdi subchart (default: `false`) | +| `kubex-connector.heartbeatSeconds` | | Connector heartbeat interval in seconds | +| `kubex-connector.requestTimeoutSeconds` | | Connector request timeout in seconds | + +Connector and CDI use the shared Kubex host and cluster entered under `container-optimization-data-forwarder.config.*`. The forwarder publishes those runtime values in its `ConfigMap`, and the connector consumes them through `forwarderConfigMap.name`. Credentials come from `stack.densify` through `densify-api-secret`, which the connector consumes through `forwarderCredentialsSecretRef.name` by default. The stack chart also owns the CDI service account and RBAC by rendering those manifests itself while disabling `kubex-ai-cdi.rbac.enabled` in the subchart. + +For the full stack RBAC shape and defaults, refer to `charts/kubex-automation-stack/values.yaml`. + +For a guided explanation of the stack-managed CDI RBAC model, see [docs/RBAC-Guide.md](docs/RBAC-Guide.md). ## Limitations @@ -140,6 +150,10 @@ This chart consists of the following subcharts: - Additional Kubex-specific grouping labels are needed beyond cloud provider labels - Enhanced node group visibility is desired in OpenShift environments (leverages Machine API) +* [Kubex Connector](../kubex-connector) - Optional in-cluster connector used for the cluster data interface. + +* [kubex-ai-cdi](../kubex-ai-cdi) - Optional in-cluster Kubex cluster data interface service. + ## Documentation * [Kubex](https://docs.kubex.ai) diff --git a/charts/kubex-automation-stack/docs/RBAC-Guide.md b/charts/kubex-automation-stack/docs/RBAC-Guide.md index 305a9b9..5abb46d 100644 --- a/charts/kubex-automation-stack/docs/RBAC-Guide.md +++ b/charts/kubex-automation-stack/docs/RBAC-Guide.md @@ -1,418 +1,75 @@ -# RBAC Permissions Guide +# RBAC Guide -This guide explains the RBAC permissions specific to Kubex data collection components in the kubex-automation-stack chart. +This guide explains the RBAC model used by `kubex-automation-stack` for the stack-managed connector, CDI, and the supporting collection components. ## Overview -The chart includes standard Prometheus and kube-state-metrics components (with their typical RBAC requirements), plus Kubex-specific components that require additional permissions: +When `kubex-connector.enabled=true` and `kubex-ai-cdi.enabled=true`, the stack deploys: -- **Kubex Data Collector** (container-optimization-data-forwarder) - collects and forwards metrics from Prometheus to Kubex -- **Ephemeral Storage Metrics Collector** - reads node and pod storage data -- **Beyla** - Detects container application runtimes (Java, Go, Python, Node.js, etc.) (requires privileged mode) -- **GPU Exporter** - collects GPU utilization metrics (requires GPU device access and privileged mode) -- **Node Labeler** (optional) - writes labels to nodes and reads OpenShift machine resources -- **Node Exporter** - collects hardware and OS metrics from nodes (requires host access) +- the in-cluster connector used for tunnel-based access +- the in-cluster CDI service used to expose cluster data +- stack-managed CDI RBAC resources + +The stack keeps CDI RBAC ownership at the umbrella chart layer and disables subchart RBAC with `kubex-ai-cdi.rbac.enabled=false`. ## Component RBAC Summary | Component | Service Account Name | RBAC Type | Key Permissions | Enabled by Default | Special Privileges | -|-----------|---------------------|-----------|-----------------|-------------------|-------------------| +|-----------|----------------------|-----------|-----------------|-------------------|-------------------| | Kubex Data Collector | `kubex-stack-kubex-forwarder` | ClusterRole | API discovery, token/subject reviews, namespaces (get) | ✅ Yes | None | +| Connector | stack-owned | None | Uses runtime identity from the forwarder ConfigMap and `densify-api-secret` | ✅ Yes when enabled | None | +| CDI | `kubex-ai-cdi-sa` | ClusterRole + ClusterRoleBinding | core resources, workloads, logs, rollout APIs, Kubex CRDs, self-subject reviews | ✅ Yes when enabled | None | | Ephemeral Storage | `k8s-ephemeral-storage-metrics` | ClusterRole | nodes, nodes/proxy, nodes/stats, pods (get/list/watch) | ✅ Yes | None | | Beyla | `kubex-beyla` | ClusterRole | pods, services, nodes (get/list/watch), replicasets (list/watch) | ✅ Yes | privileged, hostPID | | GPU Exporter | `gpu-process-exporter` | ClusterRole | pods (get/list/watch) | ✅ Yes | privileged, hostPID, host mounts, device access | -| Node Labeler | `kubex-node-labeler` | ClusterRole + Role | nodes (get/list/watch/patch/update), events, machines/machinesets (OpenShift) | ❌ No | None | +| Node Labeler | `kubex-node-labeler` | ClusterRole + Role | nodes, events, machines, machinesets, leases | ❌ No | None | | Node Exporter | `kubex-prometheus-node-exporter` | ClusterRole | token/subject reviews (when kube-rbac-proxy enabled) | ✅ Yes | hostNetwork, hostPID, host mounts | -| Prometheus Server | `kubex-prometheus-server` | ClusterRole | nodes, services, endpoints, pods, configmaps, ingresses (get/list/watch) | ✅ Yes | None | -| Kube-State-Metrics | `kubex-kube-state-metrics` | ClusterRole | Various K8s resources based on collectors config | ✅ Yes | None | - -## Kubex-Specific Component Permissions - -### Kubex Data Collector (container-optimization-data-forwarder) - -The data collector queries the Prometheus API for metrics and forwards them to Kubex for rightsizing analysis and recommendations. - -**What it does:** -- Queries Prometheus HTTP API endpoints for time-series metrics data -- Enriches metrics with Kubernetes cluster metadata (cluster ID, namespace details) -- Forwards aggregated data to Kubex for analysis -- Exposes its own metrics endpoint secured with kube-rbac-proxy for access control - -**Why these permissions:** - -The data collector needs minimal permissions to operate securely. It requires API discovery to detect cluster capabilities, authentication/authorization APIs to secure its own metrics endpoint via kube-rbac-proxy, and namespace read access to enrich metrics with cluster context. The collector does not directly access Prometheus data - it queries Prometheus via its HTTP API using service-to-service communication. - -**ClusterRole permissions:** -- **Non-resource URLs:** - - `/api`, `/api/v1` - `get` verb (for Kubernetes API discovery and version detection) - -- **Authentication/Authorization (for kube-rbac-proxy):** - - tokenreviews (authentication.k8s.io) - `create` verb - - subjectaccessreviews (authorization.k8s.io) - `create` verb - - These allow kube-rbac-proxy to validate authentication tokens and check authorization for requests to the collector's metrics endpoint - -- **Cluster metadata:** - - namespaces - `get` verb (to read namespace information and enrich forwarded metrics with context) - -**Note:** The collector does not need permissions to read from Prometheus itself - it accesses Prometheus via the in-cluster service URL using Prometheus's own access controls. - -### Ephemeral Storage Metrics Collector - -Collects ephemeral storage usage metrics for containers, which is critical for Kubex rightsizing recommendations. Enabled by default. - -**ClusterRole permissions:** -- nodes - `get`, `list`, `watch` verbs -- nodes/proxy - `get`, `list`, `watch` verbs -- nodes/stats - `get`, `list`, `watch` verbs (to access kubelet stats API) -- pods - `get`, `list`, `watch` verbs - -### Node Labeler (Optional Component) - -Adds labels to nodes indicating their Kubex Node Group assignment. Disabled by default; enable via `node-labeler.enabled: true`. - -**When to enable:** -- When nodes lack standard cloud provider node pool/group labels (e.g., `label_cloud_google_com_gke_nodepool`, `label_eks_amazonaws_com_nodegroup`, `label_agentpool`, etc.) -- When you need additional Kubex-specific grouping labels on nodes beyond what the cloud provider provides -- For enhanced node group visibility and management in OpenShift environments (leverages Machine API for machine/machineset metadata) - -**ClusterRole permissions (for node management):** -- nodes - `get`, `list`, `watch`, `patch`, `update` verbs -- events - `create`, `patch` verbs - -**ClusterRole permissions (for OpenShift integration):** -- machines (machine.openshift.io) - `get`, `list`, `watch` verbs -- machinesets (machine.openshift.io) - `get`, `list`, `watch` verbs - -**Role permissions (namespace-scoped for leader election):** -- leases (coordination.k8s.io) - `get`, `list`, `watch`, `create`, `update`, `patch`, `delete` verbs -- events - `create`, `patch` verbs - -### OpenShift Monitoring Integration +| Prometheus Server | `kubex-prometheus-server` | ClusterRole | nodes, services, endpoints, pods, configmaps, ingresses, endpointslices | ✅ Yes | None | +| Kube-State-Metrics | `kubex-kube-state-metrics` | ClusterRole | various K8s resources based on collectors config | ✅ Yes | None | -When deploying on OpenShift with `openshift.enabled: true`, the data collector service account is bound to the built-in `cluster-monitoring-view` ClusterRole, granting read access to OpenShift's user workload monitoring APIs. +## Kubex Data Collector -### Node Exporter (prometheus-node-exporter) +The data collector queries Prometheus and forwards metrics to Kubex for rightsizing analysis. -Collects hardware and OS-level metrics from each node for Kubex rightsizing analysis. +It requires: -**ClusterRole permissions (when kube-rbac-proxy is enabled):** -- tokenreviews (authentication.k8s.io) - `create` verb -- subjectaccessreviews (authorization.k8s.io) - `create` verb +- API discovery +- token/subject review APIs for kube-rbac-proxy +- namespace read access for cluster context -**Special privileges:** -- `hostNetwork: true` - access to host network namespace -- `hostPID: true` - access to host PID namespace -- Host path mounts: `/proc`, `/sys`, `/` (all read-only) - for system metrics +## Connector -### Beyla (Container Runtime Detection) +The connector does not create extra RBAC in the stack. It consumes: -Detects container application runtimes using eBPF instrumentation. Enabled by default. +- `kubex_host` +- `kubex_tenant_id` +- `kubex_cluster_name` +- `densify-api-secret` -**Purpose:** -- Identifies application runtimes running in containers (Java, Go, Python, Node.js, Ruby, .NET, etc.) -- Provides runtime visibility for Kubex workload analysis and recommendations +The stack-managed connector runtime wiring is derived from the forwarder ConfigMap. -**ClusterRole permissions:** -- **Core resources:** - - pods - `get`, `list`, `watch` verbs - - services - `get`, `list`, `watch` verbs - - nodes - `get`, `list`, `watch` verbs -- **Application resources:** - - replicasets (apps) - `list`, `watch` verbs +## CDI -**Service Account:** `kubex-beyla` (created when `beyla.serviceAccount.create: true`) +The stack renders CDI RBAC itself and disables subchart RBAC with `kubex-ai-cdi.rbac.enabled=false`. -**Special privileges:** -- `hostPID: true` - access to host PID namespace for process instrumentation -- `privileged: true` - required for eBPF program loading and kernel instrumentation -- Context propagation capability (`NET_ADMIN`) available but disabled by default in this stack +The stack-managed CDI resources are: -### GPU Exporter (gpu-process-exporter) +- `ServiceAccount/kubex-ai-cdi-sa` +- `ClusterRole/kubex-ai-cdi-reader` +- `ClusterRoleBinding/kubex-ai-cdi-reader` -Collects GPU utilization metrics for GPU-enabled workloads. Enabled by default. +## OpenShift Note -**Purpose:** -- Addresses limitations of DCGM Exporter by providing container-level GPU metrics -- Required for Kubex GPU workload rightsizing recommendations -- Supports GPU sharing scenarios (time-slicing, MPS, KAI scheduler) +For OpenShift installs, use `values-openshift.yaml` so the stack-managed connector and CDI receive compatible security context defaults. -**Deployment behavior:** -- Runs only on nodes labeled with `nvidia.com/gpu.present=true` (typically set by the Nvidia GPU Operator) -- Safe to enable in clusters without GPUs - DaemonSet pods will not be scheduled on non-GPU nodes +## Validation -**ClusterRole permissions:** -- pods - `get`, `list`, `watch` verbs - -**Special privileges:** -- `hostPID: true` - access to host PID namespace -- `privileged: true` - runs as root with privileged container -- Host path mounts: `/` and `/proc` (both read-only) - for NVML libraries and process information -- Access to GPU device files on host (`/dev/nvidia*`) - -## Standard Prometheus/KSM Components - -The chart bundles standard Prometheus components with their typical RBAC requirements. - -### Prometheus Server - -Scrapes metrics from Kubernetes resources and other exporters. - -**ClusterRole permissions:** -- **Core resources (for service discovery):** - - nodes, nodes/metrics - `get`, `list`, `watch` verbs - - services - `get`, `list`, `watch` verbs - - endpoints - `get`, `list`, `watch` verbs - - pods - `get`, `list`, `watch` verbs - - configmaps - `get`, `list`, `watch` verbs - - ingresses - `get`, `list`, `watch` verbs -- **Networking resources:** - - ingresses, ingresses/status (networking.k8s.io) - `get`, `list`, `watch` verbs - - endpointslices (discovery.k8s.io) - `get`, `list`, `watch` verbs -- **Non-resource URLs:** - - `/metrics` - `get` verb (for scraping metrics endpoints) - -**Service Account:** `kubex-prometheus-server` - -### Kube-State-Metrics - -Generates metrics about Kubernetes object state. Permissions are dynamically created based on enabled collectors. - -**ClusterRole permissions (based on collectors in this stack):** -- cronjobs (batch) - `list`, `watch` verbs -- daemonsets (apps) - `list`, `watch` verbs -- deployments (apps) - `list`, `watch` verbs -- horizontalpodautoscalers (autoscaling) - `list`, `watch` verbs -- jobs (batch) - `list`, `watch` verbs -- namespaces - `list`, `watch` verbs -- nodes - `list`, `watch` verbs -- poddisruptionbudgets (policy) - `list`, `watch` verbs -- pods - `list`, `watch` verbs -- replicasets (apps) - `list`, `watch` verbs -- replicationcontrollers - `list`, `watch` verbs -- resourcequotas - `list`, `watch` verbs -- statefulsets (apps) - `list`, `watch` verbs - -**Service Account:** `kubex-kube-state-metrics` - -**Note:** The collectors list can be customized via `prometheus.kube-state-metrics.collectors` in values.yaml. Additional collectors will require corresponding RBAC permissions. - -For detailed upstream documentation: -- [Prometheus RBAC](https://github.com/prometheus-community/helm-charts/tree/main/charts/prometheus) -- [Kube-State-Metrics RBAC](https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-state-metrics) - -## Security Considerations - -**Kubex-specific components:** -- Ephemeral storage collector requires read-only access to node stats endpoints and pods -- Data collector has minimal permissions (namespace read, API discovery) -- Node labeler is the only component with write access (node labels only) and is disabled by default - -**Exporter components:** -- Node Exporter requires host access (`hostNetwork`, `hostPID`) but all mounts are read-only -- Beyla requires privileged mode and hostPID for runtime detection - enabled by default -- GPU exporter requires privileged mode, hostPID, and read-only filesystem access for container-level GPU metrics - enabled by default - -**Standard components:** -- All Prometheus/KSM permissions are read-only except authentication/authorization checks -- Secret collection can be disabled via kube-state-metrics configuration if needed - -## Validate RBAC +Useful checks after install: ```bash -# List all service accounts created by the chart -kubectl get serviceaccount -n - -# List all RBAC resources (ClusterRoles, ClusterRoleBindings, etc.) -kubectl get clusterrole,clusterrolebinding | grep -E "(kubex|densify)" - -# Check permissions for data collector service account -kubectl auth can-i --list --as=system:serviceaccount::kubex-stack-kubex-forwarder - -# Check permissions for ephemeral storage metrics service account -kubectl auth can-i --list --as=system:serviceaccount::k8s-ephemeral-storage-metrics - -# Check permissions for Beyla service account -kubectl auth can-i --list --as=system:serviceaccount::kubex-beyla - -# Check permissions for GPU exporter service account -kubectl auth can-i --list --as=system:serviceaccount::gpu-process-exporter - -# Check permissions for node labeler service account (if enabled) -kubectl auth can-i --list --as=system:serviceaccount::kubex-node-labeler - -# Check permissions for node exporter service account -kubectl auth can-i --list --as=system:serviceaccount::kubex-prometheus-node-exporter - -# Check permissions for Prometheus server service account -kubectl auth can-i --list --as=system:serviceaccount::kubex-prometheus-server - -# Check permissions for kube-state-metrics service account -kubectl auth can-i --list --as=system:serviceaccount::kubex-kube-state-metrics -``` - -Replace `` with your deployment namespace (default: `kubex`). - -## Customization - -You can customize Kubex-specific RBAC settings via values: - -```yaml -# Data collector RBAC -container-optimization-data-forwarder: - rbac: - create: true # Create RBAC resources - -# Ephemeral storage metrics RBAC -k8s-ephemeral-storage-metrics: - rbac: - create: true # Create RBAC resources - serviceAccount: - create: true - name: k8s-ephemeral-storage-metrics - -# Node labeler RBAC (optional component, disabled by default) -node-labeler: - enabled: false # Enable if you want nodes labeled with Kubex node groups - rbac: - create: true # Create RBAC resources when enabled - -# Node exporter RBAC (part of Prometheus chart) -prometheus: - prometheus-node-exporter: - rbac: - create: true # Create RBAC resources - kubeRBACProxy: - enabled: true # Enable kube-rbac-proxy for authentication - -# Ephemeral storage metrics RBAC (enabled by default) -k8s-ephemeral-storage-metrics: - enabled: true # Enabled by default for storage metrics - rbac: - create: true - serviceAccount: - create: true - -# Beyla RBAC (enabled by default) -beyla: - enabled: true # Enabled by default for container runtime detection - privileged: true # Required for runtime detection - rbac: - create: true # Create RBAC resources - serviceAccount: - create: true - name: "" # Defaults to kubex-beyla - -# GPU exporter (enabled by default) -gpu-process-exporter: - enabled: true # Enabled by default for GPU workload monitoring - rbac: - create: true - serviceAccount: - create: true - -# Prometheus and KSM RBAC -prometheus: - rbac: - create: true # Create Prometheus server RBAC - kube-state-metrics: - rbac: - create: true # Create KSM RBAC - collectors: # Enabled collectors (determines required permissions) - - cronjobs - - daemonsets - - deployments - - horizontalpodautoscalers - - jobs - - namespaces - - nodes - - poddisruptionbudgets - - pods - - replicasets - - replicationcontrollers - - resourcequotas - - statefulsets - prometheus-node-exporter: - rbac: - create: true # Create node exporter RBAC (for kube-rbac-proxy) - -# OpenShift monitoring integration -openshift: - enabled: false # Set to true for OpenShift clusters -``` - -For the source-of-truth RBAC rules, inspect: -- Kubex components: `templates/` and `charts/*/templates/` -- Standard Prometheus/KSM: see upstream chart documentation - -## Troubleshooting RBAC Issues - -### Common Issues and Solutions - -**1. Pods failing with "Forbidden" errors** -```bash -# Check pod logs for RBAC errors -kubectl logs -n - -# Verify service account exists -kubectl get sa -n - -# Check if ClusterRoleBinding exists -kubectl get clusterrolebinding | grep -``` - -**2. Ephemeral storage metrics not collected** -- Ensure `k8s-ephemeral-storage-metrics.rbac.create: true` -- Verify access to kubelet stats API: `kubectl get --raw /api/v1/nodes//proxy/stats/summary` - -**3. Beyla not detecting application runtimes** -- Verify privileged mode is enabled: `beyla.privileged: true` -- Check hostPID access in pod spec -- Review Beyla logs for permission errors - -**4. Prometheus unable to scrape metrics** -- Verify Prometheus server service account has ClusterRole permissions -- Check NetworkPolicies that may block scraping -- Verify endpoints are discoverable: `kubectl get endpoints -n ` - -**5. Node labeler not updating node labels** (when enabled) -- Ensure `node-labeler.rbac.create: true` -- Verify node `patch` and `update` permissions -- Check node-labeler logs for admission webhook denials - -**6. OpenShift monitoring integration issues** -- Ensure `openshift.enabled: true` -- Verify `cluster-monitoring-view` ClusterRole exists in OpenShift -- Check if user workload monitoring is enabled in OpenShift cluster - -### Security Best Practices - -1. **Least Privilege**: Each component requests only the minimum permissions needed -2. **Read-Only by Default**: Most components have read-only access (except node-labeler) -3. **Namespace Isolation**: Use dedicated namespaces for monitoring components -4. **Service Account per Component**: Each sub-chart uses its own service account -5. **Audit RBAC Changes**: Review permissions before upgrading chart versions -6. **Monitor Privileged Pods**: Beyla and GPU exporter run privileged - ensure node security policies allow this - -### Restricting Permissions - -If your environment requires stricter RBAC: - -```yaml -# Disable specific components -beyla: - enabled: false # Disable if privileged pods not allowed - -gpu-process-exporter: - enabled: false # Disable if no GPU workloads or privileged pods not allowed - -node-labeler: - enabled: false # Already disabled by default (requires write access to nodes) - -# Use existing RBAC resources -container-optimization-data-forwarder: - rbac: - create: false # Use pre-created RBAC - serviceAccount: - create: false - name: existing-service-account +kubectl -n kubex get serviceaccount kubex-ai-cdi-sa +kubectl get clusterrole kubex-ai-cdi-reader +kubectl get clusterrolebinding kubex-ai-cdi-reader +kubectl auth can-i --list --as=system:serviceaccount:kubex:kubex-ai-cdi-sa +kubectl -n kubex get configmap kubex-kubex-stack -o yaml ``` diff --git a/charts/kubex-automation-stack/templates/kubex-ai-cdi-clusterrole.yaml b/charts/kubex-automation-stack/templates/kubex-ai-cdi-clusterrole.yaml new file mode 100644 index 0000000..d27dfd4 --- /dev/null +++ b/charts/kubex-automation-stack/templates/kubex-ai-cdi-clusterrole.yaml @@ -0,0 +1,111 @@ +{{- $rbac := .Values.rbac | default dict -}} +{{- $cdi := $rbac.permissions.cdi | default dict -}} +{{- $rules := $cdi.rules | default dict -}} +{{- $clusterRole := $cdi.clusterRole | default dict -}} +{{- $cdiValues := index .Values "kubex-ai-cdi" | default dict -}} +{{- $cdiRbac := $cdiValues.rbac | default dict -}} +{{- $extraLabels := $cdiValues.extraLabels | default dict -}} +{{- $extraAnnotations := $cdiValues.extraAnnotations | default dict -}} +{{- if and $cdiValues.enabled $rbac.create (not ($cdiRbac.enabled | default false)) (dig "create" false $clusterRole) }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ default "kubex-ai-cdi-reader" (dig "name" "" $clusterRole) }} + labels: + app: kubex-ai-cdi + chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + {{- if $extraLabels }} + {{- toYaml $extraLabels | nindent 4 }} + {{- end }} + {{- if $extraAnnotations }} + annotations: + {{- toYaml $extraAnnotations | nindent 4 }} + {{- end }} +rules: +{{- with $rules.core }} +{{- if .enabled }} +- apiGroups: [""] + resources: + {{- toYaml (.resources | default (list)) | nindent 4 }} + verbs: + {{- toYaml (.verbs | default (list)) | nindent 4 }} +{{- end }} +{{- end }} +{{- with $rules.podLogs }} +{{- if .enabled }} +- apiGroups: [""] + resources: + {{- toYaml (.resources | default (list)) | nindent 4 }} + verbs: + {{- toYaml (.verbs | default (list)) | nindent 4 }} +{{- end }} +{{- end }} +{{- with $rules.workloads }} +{{- if .enabled }} +- apiGroups: ["apps"] + resources: + {{- toYaml (.resources | default (list)) | nindent 4 }} + verbs: + {{- toYaml (.verbs | default (list)) | nindent 4 }} +{{- end }} +{{- end }} +{{- with $rules.rollouts }} +{{- if .enabled }} +- apiGroups: ["argoproj.io"] + resources: + {{- toYaml (.resources | default (list)) | nindent 4 }} + verbs: + {{- toYaml (.verbs | default (list)) | nindent 4 }} +{{- end }} +{{- end }} +{{- $openshiftRule := $rules.openshift | default dict -}} +{{- $openshiftEnabled := dig "enabled" nil $openshiftRule -}} +{{- if eq $openshiftEnabled nil -}} +{{- $openshiftEnabled = .Values.openshift.enabled -}} +{{- end -}} +{{- if $openshiftEnabled }} +- apiGroups: ["machine.openshift.io"] + resources: + {{- toYaml ((dig "resources" (list) $openshiftRule)) | nindent 4 }} + verbs: + {{- toYaml ((dig "verbs" (list) $openshiftRule)) | nindent 4 }} +{{- end }} +{{- with $rules.karpenter }} +{{- if .enabled }} +- apiGroups: ["karpenter.sh"] + resources: + {{- toYaml (.resources | default (list)) | nindent 4 }} + verbs: + {{- toYaml (.verbs | default (list)) | nindent 4 }} +{{- end }} +{{- end }} +{{- with $rules.karpenterAWS }} +{{- if .enabled }} +- apiGroups: ["karpenter.k8s.aws"] + resources: + {{- toYaml (.resources | default (list)) | nindent 4 }} + verbs: + {{- toYaml (.verbs | default (list)) | nindent 4 }} +{{- end }} +{{- end }} +{{- with $rules.kubexCRDs }} +{{- if .enabled }} +- apiGroups: ["rightsizing.kubex.ai"] + resources: + {{- toYaml (.resources | default (list)) | nindent 4 }} + verbs: + {{- toYaml (.verbs | default (list)) | nindent 4 }} +{{- end }} +{{- end }} +{{- with $rules.selfSubjectReview }} +{{- if .enabled }} +- apiGroups: ["authorization.k8s.io"] + resources: + {{- toYaml (.resources | default (list)) | nindent 4 }} + verbs: + {{- toYaml (.verbs | default (list)) | nindent 4 }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/kubex-automation-stack/templates/kubex-ai-cdi-clusterrolebinding.yaml b/charts/kubex-automation-stack/templates/kubex-ai-cdi-clusterrolebinding.yaml new file mode 100644 index 0000000..01ade7f --- /dev/null +++ b/charts/kubex-automation-stack/templates/kubex-ai-cdi-clusterrolebinding.yaml @@ -0,0 +1,35 @@ +{{- $rbac := .Values.rbac | default dict -}} +{{- $cdi := $rbac.permissions.cdi | default dict -}} +{{- $clusterRole := $cdi.clusterRole | default dict -}} +{{- $clusterRoleBinding := $cdi.clusterRoleBinding | default dict -}} +{{- $serviceAccount := $cdi.serviceAccount | default dict -}} +{{- $cdiValues := index .Values "kubex-ai-cdi" | default dict -}} +{{- $cdiRbac := $cdiValues.rbac | default dict -}} +{{- $extraLabels := $cdiValues.extraLabels | default dict -}} +{{- $extraAnnotations := $cdiValues.extraAnnotations | default dict -}} +{{- if and $cdiValues.enabled $rbac.create (not ($cdiRbac.enabled | default false)) (dig "create" false $clusterRoleBinding) }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ default "kubex-ai-cdi-reader" (dig "name" "" $clusterRoleBinding) }} + labels: + app: kubex-ai-cdi + chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + {{- if $extraLabels }} + {{- toYaml $extraLabels | nindent 4 }} + {{- end }} + {{- if $extraAnnotations }} + annotations: + {{- toYaml $extraAnnotations | nindent 4 }} + {{- end }} +subjects: +- kind: ServiceAccount + name: {{ default "kubex-ai-cdi-sa" (dig "name" "" $serviceAccount) }} + namespace: {{ template "common.namespace" . }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ default "kubex-ai-cdi-reader" (dig "name" "" $clusterRole) }} +{{- end }} diff --git a/charts/kubex-automation-stack/templates/kubex-ai-cdi-serviceaccount.yaml b/charts/kubex-automation-stack/templates/kubex-ai-cdi-serviceaccount.yaml new file mode 100644 index 0000000..863a449 --- /dev/null +++ b/charts/kubex-automation-stack/templates/kubex-ai-cdi-serviceaccount.yaml @@ -0,0 +1,26 @@ +{{- $rbac := .Values.rbac | default dict -}} +{{- $cdi := $rbac.permissions.cdi | default dict -}} +{{- $serviceAccount := $cdi.serviceAccount | default dict -}} +{{- $cdiValues := index .Values "kubex-ai-cdi" | default dict -}} +{{- $cdiRbac := $cdiValues.rbac | default dict -}} +{{- $extraLabels := $cdiValues.extraLabels | default dict -}} +{{- $extraAnnotations := $cdiValues.extraAnnotations | default dict -}} +{{- if and $cdiValues.enabled $rbac.create (not ($cdiRbac.enabled | default false)) (dig "create" false $serviceAccount) }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ default "kubex-ai-cdi-sa" (dig "name" "" $serviceAccount) }} + namespace: {{ template "common.namespace" . }} + labels: + app: kubex-ai-cdi + chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + {{- if $extraLabels }} + {{- toYaml $extraLabels | nindent 4 }} + {{- end }} + {{- if $extraAnnotations }} + annotations: + {{- toYaml $extraAnnotations | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/kubex-automation-stack/templates/kubex-connector-configmap.yaml b/charts/kubex-automation-stack/templates/kubex-connector-configmap.yaml new file mode 100644 index 0000000..36c7b20 --- /dev/null +++ b/charts/kubex-automation-stack/templates/kubex-connector-configmap.yaml @@ -0,0 +1,36 @@ +{{- $connectorValues := index .Values "kubex-connector" | default dict -}} +{{- $enabled := $connectorValues.enabled | default false -}} +{{- $forwarder := index .Values "container-optimization-data-forwarder" | default dict -}} +{{- $config := $forwarder.config | default dict -}} +{{- $forwarderConfig := $config.forwarder | default dict -}} +{{- $densify := $forwarderConfig.densify | default dict -}} +{{- $url := $densify.url | default dict -}} +{{- $host := trim (default "" $url.host) -}} +{{- $clusters := $config.clusters | default list -}} +{{- $cluster := dict -}} +{{- if gt (len $clusters) 0 -}} +{{- $cluster = first $clusters -}} +{{- end -}} +{{- if $enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: kubex-connector-runtime + namespace: {{ template "common.namespace" . }} + labels: + app: kubex-connector + chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + {{- with $connectorValues.extraLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with $connectorValues.extraAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +data: + kubex_host: {{ $host | quote }} + kubex_tenant_id: {{ (index (splitList "." $host) 0) | quote }} + kubex_cluster_name: {{ (default "" $cluster.name) | quote }} +{{- end }} diff --git a/charts/kubex-automation-stack/values-edit.yaml b/charts/kubex-automation-stack/values-edit.yaml index 05f14ce..8e70a4e 100644 --- a/charts/kubex-automation-stack/values-edit.yaml +++ b/charts/kubex-automation-stack/values-edit.yaml @@ -17,20 +17,24 @@ container-optimization-data-forwarder: clusters: - name: '' # mandatory - # Optional: Additional labels to add to CronJob and Job resources - #extraLabels: - # my-custom-label: my-value-1 - # Optional: Additional annotations to add to CronJob and Job resources - #extraAnnotations: - # my-custom-annotation: my-value-2 +# Optional: Enable the connector and CDI tunnel path. +# Keep these opt-in until the feature is generally available in production. +# kubex-connector: +# enabled: true +# kubex-ai-cdi: +# enabled: true +# extraLabels: +# my-custom-label: my-value-1 +# extraAnnotations: +# my-custom-annotation: my-value-2 # The following values are optional, if job cleanup is required; # each one of these should be a non-negative integer (0 is allowed) - # cronJob: - # successfulJobsHistoryLimit: - # failedJobsHistoryLimit: - # ttlSecondsAfterFinished: # applies for both the cronjob and the job - # backoffLimit: # applies for both the cronjob and the job, default: 4 +# cronJob: +# successfulJobsHistoryLimit: +# failedJobsHistoryLimit: +# ttlSecondsAfterFinished: # applies for both the cronjob and the job +# backoffLimit: # applies for both the cronjob and the job, default: 4 # The following section is optional, and is required only in the following use-case: # 1. The persistent volume for Prometheus has to use an explicit storage class (not the default) diff --git a/charts/kubex-automation-stack/values-openshift.yaml b/charts/kubex-automation-stack/values-openshift.yaml index edad3b8..70e3740 100644 --- a/charts/kubex-automation-stack/values-openshift.yaml +++ b/charts/kubex-automation-stack/values-openshift.yaml @@ -8,6 +8,7 @@ stack: container-optimization-data-forwarder: assignedUIDs: true job: + enable: false checkPrometheusReady: false config: prometheus: @@ -37,6 +38,22 @@ k8s-ephemeral-storage-metrics: interval: 30s scrapeTimeout: 10s +kubex-connector: + containerSecurityContext: + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault + +kubex-ai-cdi: + containerSecurityContext: + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault + gpu-process-exporter: enabled: false diff --git a/charts/kubex-automation-stack/values.yaml b/charts/kubex-automation-stack/values.yaml index b2d2464..c83b31f 100644 --- a/charts/kubex-automation-stack/values.yaml +++ b/charts/kubex-automation-stack/values.yaml @@ -6,6 +6,76 @@ openshift: enabled: false +rbac: + create: true + permissions: + cdi: + serviceAccount: + create: true + name: kubex-ai-cdi-sa + clusterRole: + create: true + name: kubex-ai-cdi-reader + clusterRoleBinding: + create: true + name: kubex-ai-cdi-reader + role: + create: false + name: kubex-ai-cdi-reader + roleBinding: + create: false + name: kubex-ai-cdi-reader + rules: + core: + enabled: true + resources: ["pods", "services", "configmaps", "namespaces", "nodes", "events"] + verbs: ["get", "list"] + podLogs: + enabled: true + resources: ["pods/log"] + verbs: ["get"] + workloads: + enabled: true + resources: ["deployments", "daemonsets", "statefulsets", "replicasets"] + verbs: ["get", "list"] + # Automatically enable OpenShift permissions when openshift mode is on. + openshift: + enabled: null + resources: ["machines", "machinesets", "machinedeployments", "machinehealthchecks", "controlplanemachinesets"] + verbs: ["get", "list"] + karpenter: + enabled: true + resources: ["nodepools", "nodeclaims"] + verbs: ["get", "list"] + karpenterAWS: + enabled: true + resources: ["ec2nodeclasses"] + verbs: ["get", "list"] + rollouts: + enabled: true + resources: ["rollouts", "analysisruns", "experiments"] + verbs: ["get", "list"] + kubexCRDs: + enabled: true + resources: ["automationstrategies", "clusterautomationstrategies", "clustergpurebalancingpolicies", "clusterproactivepolicies", "clusterrollbackpolicies", "clusterstaticpolicies", "globalconfigurations", "gpuconsolidationpolicies", "gpurebalancingpolicies", "podaffinitypolicies", "policyevaluations", "proactivepolicies", "rollbackpolicies", "staticpolicies"] + verbs: ["get", "list"] + selfSubjectReview: + enabled: true + resources: ["selfsubjectrulesreviews", "selfsubjectaccessreviews"] + verbs: ["create"] + connector: + serviceAccount: + create: false + clusterRole: + create: false + clusterRoleBinding: + create: false + role: + create: false + roleBinding: + create: false + rules: {} + stack: densify: createSecret: true @@ -18,6 +88,26 @@ gpu-process-exporter: node-labeler: enabled: false +kubex-connector: + enabled: false + extraLabels: {} + extraAnnotations: {} + image: + tag: 0.1.9-rc.3 + forwarderConfigMap: + name: kubex-connector-runtime + forwarderCredentialsSecretRef: + name: densify-api-secret + +kubex-ai-cdi: + enabled: false + extraLabels: {} + extraAnnotations: {} + image: + tag: 0.1.9-rc.3 + rbac: + enabled: false + container-optimization-data-forwarder: nameOverride: kubex-stack tolerations: @@ -34,6 +124,7 @@ container-optimization-data-forwarder: densify: url: UserSecretName: densify-api-secret + clusters: [] prometheus: url: host: kubex-prometheus-server diff --git a/charts/kubex-connector/Chart.yaml b/charts/kubex-connector/Chart.yaml index d5577d1..b3af8e5 100644 --- a/charts/kubex-connector/Chart.yaml +++ b/charts/kubex-connector/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: kubex-connector description: Stack-managed in-cluster connector used for the cluster data interface type: application -version: 0.1.8 +version: 0.1.9-rc.3 icon: https://kubex.ai/wp-content/uploads/kubex-logo-landscape.svg maintainers: -- email: support@kubex.ai - name: support + - email: support@kubex.ai + name: support diff --git a/charts/kubex-connector/values.yaml b/charts/kubex-connector/values.yaml index db1972f..90ee54a 100644 --- a/charts/kubex-connector/values.yaml +++ b/charts/kubex-connector/values.yaml @@ -1,6 +1,6 @@ image: repository: densify/kubex-connector - tag: 0.1.8 + tag: 0.1.9-rc.3 pullPolicy: IfNotPresent replicaCount: 1