From d29499789b6e2a391e066f102cc376bd732926a7 Mon Sep 17 00:00:00 2001 From: pebble <166765742+ignorant05@users.noreply.github.com> Date: Sun, 22 Feb 2026 18:50:36 +0100 Subject: [PATCH] feat: deploy to k8s --- .gitignore | 1 + helm/log-processing-system/Chart.lock | 6 + helm/log-processing-system/Chart.yaml | 12 ++ .../log-processing-system/templates/NOTES.txt | 45 ++++++ .../templates/_helpers.tpl | 71 +++++++++ .../templates/configmap.yaml | 10 ++ .../templates/deployment.yaml | 145 ++++++++++++++++++ .../templates/serviceaccount.yaml | 12 ++ helm/log-processing-system/values.yaml | 91 +++++++++++ k8s/confluent-kafka.yaml | 122 +++++++++++++++ 10 files changed, 515 insertions(+) create mode 100644 helm/log-processing-system/Chart.lock create mode 100644 helm/log-processing-system/Chart.yaml create mode 100644 helm/log-processing-system/templates/NOTES.txt create mode 100644 helm/log-processing-system/templates/_helpers.tpl create mode 100644 helm/log-processing-system/templates/configmap.yaml create mode 100644 helm/log-processing-system/templates/deployment.yaml create mode 100644 helm/log-processing-system/templates/serviceaccount.yaml create mode 100644 helm/log-processing-system/values.yaml create mode 100644 k8s/confluent-kafka.yaml diff --git a/.gitignore b/.gitignore index 667aaef..87e2edb 100644 --- a/.gitignore +++ b/.gitignore @@ -31,3 +31,4 @@ build/ ### VS Code ### .vscode/ +helm/log-processing-system/charts/ diff --git a/helm/log-processing-system/Chart.lock b/helm/log-processing-system/Chart.lock new file mode 100644 index 0000000..4e13d31 --- /dev/null +++ b/helm/log-processing-system/Chart.lock @@ -0,0 +1,6 @@ +dependencies: +- name: kafka + repository: https://charts.bitnami.com/bitnami + version: 26.11.4 +digest: sha256:87111fdf138f01f0002710f56323e9b1e0d9af86a2e93cdefae413b18a914ac3 +generated: "2026-02-21T20:34:21.108607762+01:00" diff --git a/helm/log-processing-system/Chart.yaml b/helm/log-processing-system/Chart.yaml new file mode 100644 index 0000000..a06c79e --- /dev/null +++ b/helm/log-processing-system/Chart.yaml @@ -0,0 +1,12 @@ +apiVersion: v2 +name: log-processing-system +description: Kafka log processing CLI tool +type: application +version: 0.1.0 +appVersion: "0.1.0" + +dependencies: + - name: kafka + version: "26.x.x" + repository: "https://charts.bitnami.com/bitnami" + condition: kafka.enabled diff --git a/helm/log-processing-system/templates/NOTES.txt b/helm/log-processing-system/templates/NOTES.txt new file mode 100644 index 0000000..eb20479 --- /dev/null +++ b/helm/log-processing-system/templates/NOTES.txt @@ -0,0 +1,45 @@ +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + log-processing-system {{ .Chart.AppVersion }} deployed successfully +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +Kafka bootstrap servers: + {{ include "log-processing-system.kafkaBootstrapServers" . }} + +Topics pre-created: + {{- range .Values.topics }} + • {{ .name }} ({{ .partitions }} partitions, replication factor {{ .replicationFactor }}) + {{- end }} + +── Run commands ──────────────────────────────────────── + +Exec into the CLI pod: + + kubectl exec -it deploy/{{ include "log-processing-system.fullname" . }} -- \ + java -jar /app/klog.jar health + +Check health: + + kubectl exec deploy/{{ include "log-processing-system.fullname" . }} -- \ + java -jar /app/klog.jar health \ + -b {{ include "log-processing-system.kafkaBootstrapServers" . }} \ + -t {{ (index .Values.topics 0).name }} \ + --json + +Generate test logs: + + kubectl exec deploy/{{ include "log-processing-system.fullname" . }} -- \ + java -jar /app/klog.jar generate \ + -b {{ include "log-processing-system.kafkaBootstrapServers" . }} \ + -t {{ (index .Values.topics 0).name }} \ + -c 100 -i 500 + +View metrics: + + kubectl exec deploy/{{ include "log-processing-system.fullname" . }} -- \ + java -jar /app/klog.jar metrics + +── Logs ──────────────────────────────────────────────── + + kubectl logs deploy/{{ include "log-processing-system.fullname" . }} + kubectl logs deploy/{{ include "log-processing-system.fullname" . }} -c wait-for-kafka + kubectl logs deploy/{{ include "log-processing-system.fullname" . }} -c create-topics diff --git a/helm/log-processing-system/templates/_helpers.tpl b/helm/log-processing-system/templates/_helpers.tpl new file mode 100644 index 0000000..0cb8b1e --- /dev/null +++ b/helm/log-processing-system/templates/_helpers.tpl @@ -0,0 +1,71 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "log-processing-system.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +*/}} +{{- define "log-processing-system.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Chart label — name + version. +*/}} +{{- define "log-processing-system.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels applied to every resource. +*/}} +{{- define "log-processing-system.labels" -}} +helm.sh/chart: {{ include "log-processing-system.chart" . }} +{{ include "log-processing-system.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels (used in matchLabels and pod template labels). +*/}} +{{- define "log-processing-system.selectorLabels" -}} +app.kubernetes.io/name: {{ include "log-processing-system.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Service account name +*/}} +{{- define "log-processing-system.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "log-processing-system.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} + +{{/* +Kafka bootstrap servers +*/}} +{{- define "log-processing-system.kafkaBootstrapServers" -}} +{{- if .Values.kafka.enabled }} +{{- printf "%s-kafka:9092" .Release.Name }} +{{- else }} +{{- .Values.externalKafka.bootstrapServers }} +{{- end }} +{{- end }} diff --git a/helm/log-processing-system/templates/configmap.yaml b/helm/log-processing-system/templates/configmap.yaml new file mode 100644 index 0000000..cacb9ce --- /dev/null +++ b/helm/log-processing-system/templates/configmap.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "log-processing-system.fullname" . }}-config + labels: + {{- include "log-processing-system.labels" . | nindent 4 }} +data: + KAFKA_BOOTSTRAP_SERVERS: {{ include "log-processing-system.kafkaBootstrapServers" . | quote }} + KAFKA_HEALTH_TIMEOUT_SECONDS: {{ .Values.healthCheck.timeoutSeconds | quote }} + KAFKA_TOPICS: {{ range $i, $t := .Values.topics }}{{ if $i }},{{ end }}{{ $t.name }}{{ end }} diff --git a/helm/log-processing-system/templates/deployment.yaml b/helm/log-processing-system/templates/deployment.yaml new file mode 100644 index 0000000..f73622e --- /dev/null +++ b/helm/log-processing-system/templates/deployment.yaml @@ -0,0 +1,145 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "log-processing-system.fullname" . }} + labels: + {{- include "log-processing-system.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + {{- include "log-processing-system.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "log-processing-system.selectorLabels" . | nindent 8 }} + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ include "log-processing-system.serviceAccountName" . }} + + {{- with .Values.podSecurityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} + + initContainers: + # 1. Wait for Kafka to be healthy before doing anything else + - name: wait-for-kafka + image: {{ .Values.image.repository }}:{{ .Values.image.tag }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: + - sh + - -c + - | + echo "Waiting for Kafka to be healthy..." + until java -jar /app/log-processing-system.jar health \ + -b "$KAFKA_BOOTSTRAP_SERVERS" \ + -t "$KAFKA_TOPICS" \ + --timeout "$KAFKA_HEALTH_TIMEOUT_SECONDS" > /dev/null 2>&1; do + echo " Kafka not ready — retrying in 5s..." + sleep 5 + done + echo "Kafka is ready." + envFrom: + - configMapRef: + name: {{ include "log-processing-system.fullname" . }}-config + {{- with .Values.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + + # 2. Create required topics + - name: create-topics + image: {{ .Values.image.repository }}:{{ .Values.image.tag }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: + - sh + - -c + - | + {{- range .Values.topics }} + echo "Creating topic '{{ .name }}'..." + java -jar /app/log-processing-system.jar topic create \ + -b "$KAFKA_BOOTSTRAP_SERVERS" \ + -n "{{ .name }}" \ + -p {{ .partitions }} || echo "Topic '{{ .name }}' may already exist — continuing." + {{- end }} + echo "Topic initialisation complete." + envFrom: + - configMapRef: + name: {{ include "log-processing-system.fullname" . }}-config + {{- with .Values.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + containers: + - name: klog + image: {{ .Values.image.repository }}:{{ .Values.image.tag }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: {{ toJson .Values.command }} + envFrom: + - configMapRef: + name: {{ include "log-processing-system.fullname" . }}-config + + {{- with .Values.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + + {{- with .Values.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + + livenessProbe: + exec: + command: + - java + - -jar + - /app/log-processing-system.jar + - health + - -b + - $(KAFKA_BOOTSTRAP_SERVERS) + - -t + - $(KAFKA_TOPICS) + - --timeout + - $(KAFKA_HEALTH_TIMEOUT_SECONDS) + initialDelaySeconds: 15 + periodSeconds: 30 + failureThreshold: 3 + timeoutSeconds: 10 + + readinessProbe: + exec: + command: + - java + - -jar + - /app/log-processing-system.jar + - health + - -b + - $(KAFKA_BOOTSTRAP_SERVERS) + - -t + - $(KAFKA_TOPICS) + - --timeout + - $(KAFKA_HEALTH_TIMEOUT_SECONDS) + initialDelaySeconds: 10 + periodSeconds: 15 + failureThreshold: 2 + timeoutSeconds: 10 + + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/helm/log-processing-system/templates/serviceaccount.yaml b/helm/log-processing-system/templates/serviceaccount.yaml new file mode 100644 index 0000000..18adf31 --- /dev/null +++ b/helm/log-processing-system/templates/serviceaccount.yaml @@ -0,0 +1,12 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "log-processing-system.serviceAccountName" . }} + labels: + {{- include "log-processing-system.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/helm/log-processing-system/values.yaml b/helm/log-processing-system/values.yaml new file mode 100644 index 0000000..e902003 --- /dev/null +++ b/helm/log-processing-system/values.yaml @@ -0,0 +1,91 @@ +image: + repository: ghcr.io/ignorant05/log-processing-system + tag: "latest" + pullPolicy: IfNotPresent + +# Command the CLI container runs on startup. +# Override to run a specific command, e.g. ["generate", "-t", "logs", "-c", "1000"] +# Use ["sleep", "infinity"] to keep the pod alive for exec-based usage. +command: ["sleep", "infinity"] + +# Number of CLI pod replicas +replicaCount: 1 + +resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 500m + memory: 512Mi + +kafka: + image: + registry: docker.io + repository: confluentinc/cp-kafka + tag: "7.4.0" + pullPolicy: Never + + # Set to false if you are connecting to an external Kafka cluster + enabled: true + + # Bitnami Kafka chart overrides + # Full values reference: https://github.com/bitnami/charts/tree/main/bitnami/kafka + replicaCount: 1 + + listeners: + client: + protocol: PLAINTEXT + + # Disable auth for local/dev deployments + auth: + clientProtocol: plaintext + interBrokerProtocol: plaintext + + zookeeper: + enabled: false # Bitnami Kafka 26+ uses KRaft by default + + persistence: + enabled: true + size: 8Gi + + resources: + requests: + cpu: 250m + memory: 512Mi + limits: + cpu: 1000m + memory: 1Gi + +externalKafka: + bootstrapServers: "my-external-kafka:9092" + +topics: + - name: logs + partitions: 4 + replicationFactor: 1 + +healthCheck: + timeoutSeconds: 5 + +serviceAccount: + create: true + name: "" + annotations: {} + +podAnnotations: {} + +podSecurityContext: + runAsNonRoot: true + runAsUser: 1000 + fsGroup: 1000 + +securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + +nodeSelector: {} +tolerations: [] +affinity: {} diff --git a/k8s/confluent-kafka.yaml b/k8s/confluent-kafka.yaml new file mode 100644 index 0000000..52453c3 --- /dev/null +++ b/k8s/confluent-kafka.yaml @@ -0,0 +1,122 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: klog + +--- +apiVersion: v1 +kind: Service +metadata: + name: zookeeper + namespace: klog +spec: + ports: + - port: 2181 + name: client + selector: + app: zookeeper + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: zookeeper + namespace: klog +spec: + replicas: 1 + selector: + matchLabels: + app: zookeeper + template: + metadata: + labels: + app: zookeeper + spec: + containers: + - name: zookeeper + image: confluentinc/cp-zookeeper:7.4.0 + imagePullPolicy: Never + ports: + - containerPort: 2181 + env: + - name: ZOOKEEPER_CLIENT_PORT + value: "2181" + - name: ZOOKEEPER_TICK_TIME + value: "2000" + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 500m + memory: 512Mi + readinessProbe: + tcpSocket: + port: 2181 + initialDelaySeconds: 10 + periodSeconds: 5 + +--- +apiVersion: v1 +kind: Service +metadata: + name: confluent-kafka + namespace: klog +spec: + ports: + - port: 9092 + name: plaintext + selector: + app: confluent-kafka + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: confluent-kafka + namespace: klog +spec: + replicas: 1 + selector: + matchLabels: + app: confluent-kafka + template: + metadata: + labels: + app: confluent-kafka + spec: + containers: + - name: kafka + image: confluentinc/cp-kafka:7.4.0 + imagePullPolicy: Never + ports: + - containerPort: 9092 + env: + - name: KAFKA_BROKER_ID + value: "1" + - name: KAFKA_ZOOKEEPER_CONNECT + value: "zookeeper:2181" + - name: KAFKA_ADVERTISED_LISTENERS + value: "PLAINTEXT://confluent-kafka:9092" + - name: KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR + value: "1" + - name: KAFKA_AUTO_CREATE_TOPICS_ENABLE + value: "true" + resources: + requests: + cpu: 250m + memory: 512Mi + limits: + cpu: 1000m + memory: 1Gi + readinessProbe: + tcpSocket: + port: 9092 + initialDelaySeconds: 20 + periodSeconds: 10 + livenessProbe: + tcpSocket: + port: 9092 + initialDelaySeconds: 30 + periodSeconds: 15