From 6aca7cafe492a4a90e75fa0d94d5042e42d77a11 Mon Sep 17 00:00:00 2001 From: Amir Rajabi Date: Sun, 29 Jun 2025 17:13:47 +0330 Subject: [PATCH 1/2] Migrate Kafka to KRaft mode and update image version Removed Zookeeper service and configuration from docker-compose files, updated Kafka services to use KRaft (self-managed metadata quorum), and set required environment variables for KRaft mode. Also upgraded the Kafka Docker image from version 7.1.1 to 8.0.0. --- docker-compose.local.yml | 2 -- docker-compose.yml | 47 +++++++++++++++++----------------- docker-images/kafka/Dockerfile | 2 +- 3 files changed, 25 insertions(+), 26 deletions(-) diff --git a/docker-compose.local.yml b/docker-compose.local.yml index 451d279fe..177eebbce 100644 --- a/docker-compose.local.yml +++ b/docker-compose.local.yml @@ -1,7 +1,5 @@ version: '3.8' services: - zookeeper: - user: "root" kafka-1: user: "root" kafka-2: diff --git a/docker-compose.yml b/docker-compose.yml index fdfadc3a0..069c940fb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -14,27 +14,15 @@ x-postgres-db: &postgres-db version: '3.8' services: - zookeeper: - image: confluentinc/cp-zookeeper:7.1.1 - hostname: zookeeper - volumes: - - zookeeper-data:/var/lib/zookeeper/data - - zookeeper-log:/var/lib/zookeeper/log - environment: - - ALLOW_ANONYMOUS_LOGIN=yes - - ZOOKEEPER_CLIENT_PORT=2181 - networks: - - default - deploy: - restart_policy: - condition: on-failure kafka-1: image: ghcr.io/opexdev/kafka hostname: kafka-1 volumes: - kafka-1:/var/lib/kafka/data environment: - - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 + - KAFKA_NODE_ID=1 + - KAFKA_PROCESS_ROLES=broker,controller + - CLUSTER_ID=c29f3d6e-47e1-4d92-b6d8-72b09f2eae23 - ALLOW_PLAINTEXT_LISTENER=yes - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=CLIENT:PLAINTEXT,EXTERNAL:PLAINTEXT - KAFKA_LISTENERS=CLIENT://kafka-1:29092,EXTERNAL://kafka-1:9092 @@ -42,8 +30,11 @@ services: - KAFKA_INTER_BROKER_LISTENER_NAME=CLIENT - KAFKA_UNCLEAN_LEADER_ELECTION_ENABLE=false - KAFKA_OPTS=-javaagent:/opt/prometheus/jmx-exporter.jar=1234:/opt/prometheus/kafka-jmx-exporter.yml - depends_on: - - zookeeper + - KAFKA_CONTROLLER_QUORUM_VOTERS=1@kafka-1:29093,2@kafka-2:29093,3@kafka-3:29093 + - KAFKA_CONTROLLER_LISTENER_NAMES=CONTROLLER + - KAFKA_LISTENERS=CLIENT://kafka-1:29092,EXTERNAL://kafka-1:9092,CONTROLLER://kafka-1:29093 + - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=CLIENT:PLAINTEXT,EXTERNAL:PLAINTEXT,CONTROLLER:PLAINTEXT + networks: - default deploy: @@ -55,7 +46,9 @@ services: volumes: - kafka-2:/var/lib/kafka/data environment: - - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 + - KAFKA_NODE_ID=2 + - KAFKA_PROCESS_ROLES=broker,controller + - CLUSTER_ID=c29f3d6e-47e1-4d92-b6d8-72b09f2eae23 - ALLOW_PLAINTEXT_LISTENER=yes - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=CLIENT:PLAINTEXT,EXTERNAL:PLAINTEXT - KAFKA_LISTENERS=CLIENT://kafka-2:29092,EXTERNAL://kafka-2:9092 @@ -63,8 +56,11 @@ services: - KAFKA_INTER_BROKER_LISTENER_NAME=CLIENT - KAFKA_UNCLEAN_LEADER_ELECTION_ENABLE=false - KAFKA_OPTS=-javaagent:/opt/prometheus/jmx-exporter.jar=1234:/opt/prometheus/kafka-jmx-exporter.yml - depends_on: - - zookeeper + - KAFKA_CONTROLLER_QUORUM_VOTERS=1@kafka-1:29093,2@kafka-2:29093,3@kafka-3:29093 + - KAFKA_CONTROLLER_LISTENER_NAMES=CONTROLLER + - KAFKA_LISTENERS=CLIENT://kafka-2:29092,EXTERNAL://kafka-2:9092,CONTROLLER://kafka-2:29093 + - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=CLIENT:PLAINTEXT,EXTERNAL:PLAINTEXT,CONTROLLER:PLAINTEXT + networks: - default deploy: @@ -76,7 +72,9 @@ services: volumes: - kafka-3:/var/lib/kafka/data environment: - - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 + - KAFKA_NODE_ID=3 + - KAFKA_PROCESS_ROLES=broker,controller + - CLUSTER_ID=c29f3d6e-47e1-4d92-b6d8-72b09f2eae23 - ALLOW_PLAINTEXT_LISTENER=yes - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=CLIENT:PLAINTEXT,EXTERNAL:PLAINTEXT - KAFKA_LISTENERS=CLIENT://kafka-3:29092,EXTERNAL://kafka-3:9092 @@ -84,8 +82,11 @@ services: - KAFKA_INTER_BROKER_LISTENER_NAME=CLIENT - KAFKA_UNCLEAN_LEADER_ELECTION_ENABLE=false - KAFKA_OPTS=-javaagent:/opt/prometheus/jmx-exporter.jar=1234:/opt/prometheus/kafka-jmx-exporter.yml - depends_on: - - zookeeper + - KAFKA_CONTROLLER_QUORUM_VOTERS=1@kafka-1:29093,2@kafka-2:29093,3@kafka-3:29093 + - KAFKA_CONTROLLER_LISTENER_NAMES=CONTROLLER + - KAFKA_LISTENERS=CLIENT://kafka-3:29092,EXTERNAL://kafka-3:9092,CONTROLLER://kafka-3:29093 + - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=CLIENT:PLAINTEXT,EXTERNAL:PLAINTEXT,CONTROLLER:PLAINTEXT + networks: - default deploy: diff --git a/docker-images/kafka/Dockerfile b/docker-images/kafka/Dockerfile index e3c2623ba..43f699594 100644 --- a/docker-images/kafka/Dockerfile +++ b/docker-images/kafka/Dockerfile @@ -1,4 +1,4 @@ -FROM confluentinc/cp-kafka:7.1.1 +FROM confluentinc/cp-kafka:8.0.0 USER root RUN mkdir /opt/prometheus RUN chmod +rx /opt/prometheus From 4c0bea77f60f623f4569fb275b5853c3afc95044 Mon Sep 17 00:00:00 2001 From: Amir Rajabi Date: Mon, 30 Jun 2025 13:19:49 +0330 Subject: [PATCH 2/2] Add cluster_id to env file --- docker-compose.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 069c940fb..ff9e633a8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -22,7 +22,7 @@ services: environment: - KAFKA_NODE_ID=1 - KAFKA_PROCESS_ROLES=broker,controller - - CLUSTER_ID=c29f3d6e-47e1-4d92-b6d8-72b09f2eae23 + - CLUSTER_ID=${KAFKA_CLUSTER_ID} - ALLOW_PLAINTEXT_LISTENER=yes - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=CLIENT:PLAINTEXT,EXTERNAL:PLAINTEXT - KAFKA_LISTENERS=CLIENT://kafka-1:29092,EXTERNAL://kafka-1:9092 @@ -48,7 +48,7 @@ services: environment: - KAFKA_NODE_ID=2 - KAFKA_PROCESS_ROLES=broker,controller - - CLUSTER_ID=c29f3d6e-47e1-4d92-b6d8-72b09f2eae23 + - CLUSTER_ID=${KAFKA_CLUSTER_ID} - ALLOW_PLAINTEXT_LISTENER=yes - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=CLIENT:PLAINTEXT,EXTERNAL:PLAINTEXT - KAFKA_LISTENERS=CLIENT://kafka-2:29092,EXTERNAL://kafka-2:9092 @@ -74,7 +74,7 @@ services: environment: - KAFKA_NODE_ID=3 - KAFKA_PROCESS_ROLES=broker,controller - - CLUSTER_ID=c29f3d6e-47e1-4d92-b6d8-72b09f2eae23 + - CLUSTER_ID=${KAFKA_CLUSTER_ID} - ALLOW_PLAINTEXT_LISTENER=yes - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=CLIENT:PLAINTEXT,EXTERNAL:PLAINTEXT - KAFKA_LISTENERS=CLIENT://kafka-3:29092,EXTERNAL://kafka-3:9092