diff --git a/docker-compose.local.yml b/docker-compose.local.yml index 451d279fe..177eebbce 100644 --- a/docker-compose.local.yml +++ b/docker-compose.local.yml @@ -1,7 +1,5 @@ version: '3.8' services: - zookeeper: - user: "root" kafka-1: user: "root" kafka-2: diff --git a/docker-compose.yml b/docker-compose.yml index fdfadc3a0..ff9e633a8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -14,27 +14,15 @@ x-postgres-db: &postgres-db version: '3.8' services: - zookeeper: - image: confluentinc/cp-zookeeper:7.1.1 - hostname: zookeeper - volumes: - - zookeeper-data:/var/lib/zookeeper/data - - zookeeper-log:/var/lib/zookeeper/log - environment: - - ALLOW_ANONYMOUS_LOGIN=yes - - ZOOKEEPER_CLIENT_PORT=2181 - networks: - - default - deploy: - restart_policy: - condition: on-failure kafka-1: image: ghcr.io/opexdev/kafka hostname: kafka-1 volumes: - kafka-1:/var/lib/kafka/data environment: - - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 + - KAFKA_NODE_ID=1 + - KAFKA_PROCESS_ROLES=broker,controller + - CLUSTER_ID=${KAFKA_CLUSTER_ID} - ALLOW_PLAINTEXT_LISTENER=yes - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=CLIENT:PLAINTEXT,EXTERNAL:PLAINTEXT - KAFKA_LISTENERS=CLIENT://kafka-1:29092,EXTERNAL://kafka-1:9092 @@ -42,8 +30,11 @@ services: - KAFKA_INTER_BROKER_LISTENER_NAME=CLIENT - KAFKA_UNCLEAN_LEADER_ELECTION_ENABLE=false - KAFKA_OPTS=-javaagent:/opt/prometheus/jmx-exporter.jar=1234:/opt/prometheus/kafka-jmx-exporter.yml - depends_on: - - zookeeper + - KAFKA_CONTROLLER_QUORUM_VOTERS=1@kafka-1:29093,2@kafka-2:29093,3@kafka-3:29093 + - KAFKA_CONTROLLER_LISTENER_NAMES=CONTROLLER + - KAFKA_LISTENERS=CLIENT://kafka-1:29092,EXTERNAL://kafka-1:9092,CONTROLLER://kafka-1:29093 + - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=CLIENT:PLAINTEXT,EXTERNAL:PLAINTEXT,CONTROLLER:PLAINTEXT + networks: - default deploy: @@ -55,7 +46,9 @@ services: volumes: - kafka-2:/var/lib/kafka/data environment: - - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 + - KAFKA_NODE_ID=2 + - KAFKA_PROCESS_ROLES=broker,controller + - CLUSTER_ID=${KAFKA_CLUSTER_ID} - ALLOW_PLAINTEXT_LISTENER=yes - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=CLIENT:PLAINTEXT,EXTERNAL:PLAINTEXT - KAFKA_LISTENERS=CLIENT://kafka-2:29092,EXTERNAL://kafka-2:9092 @@ -63,8 +56,11 @@ services: - KAFKA_INTER_BROKER_LISTENER_NAME=CLIENT - KAFKA_UNCLEAN_LEADER_ELECTION_ENABLE=false - KAFKA_OPTS=-javaagent:/opt/prometheus/jmx-exporter.jar=1234:/opt/prometheus/kafka-jmx-exporter.yml - depends_on: - - zookeeper + - KAFKA_CONTROLLER_QUORUM_VOTERS=1@kafka-1:29093,2@kafka-2:29093,3@kafka-3:29093 + - KAFKA_CONTROLLER_LISTENER_NAMES=CONTROLLER + - KAFKA_LISTENERS=CLIENT://kafka-2:29092,EXTERNAL://kafka-2:9092,CONTROLLER://kafka-2:29093 + - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=CLIENT:PLAINTEXT,EXTERNAL:PLAINTEXT,CONTROLLER:PLAINTEXT + networks: - default deploy: @@ -76,7 +72,9 @@ services: volumes: - kafka-3:/var/lib/kafka/data environment: - - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 + - KAFKA_NODE_ID=3 + - KAFKA_PROCESS_ROLES=broker,controller + - CLUSTER_ID=${KAFKA_CLUSTER_ID} - ALLOW_PLAINTEXT_LISTENER=yes - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=CLIENT:PLAINTEXT,EXTERNAL:PLAINTEXT - KAFKA_LISTENERS=CLIENT://kafka-3:29092,EXTERNAL://kafka-3:9092 @@ -84,8 +82,11 @@ services: - KAFKA_INTER_BROKER_LISTENER_NAME=CLIENT - KAFKA_UNCLEAN_LEADER_ELECTION_ENABLE=false - KAFKA_OPTS=-javaagent:/opt/prometheus/jmx-exporter.jar=1234:/opt/prometheus/kafka-jmx-exporter.yml - depends_on: - - zookeeper + - KAFKA_CONTROLLER_QUORUM_VOTERS=1@kafka-1:29093,2@kafka-2:29093,3@kafka-3:29093 + - KAFKA_CONTROLLER_LISTENER_NAMES=CONTROLLER + - KAFKA_LISTENERS=CLIENT://kafka-3:29092,EXTERNAL://kafka-3:9092,CONTROLLER://kafka-3:29093 + - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=CLIENT:PLAINTEXT,EXTERNAL:PLAINTEXT,CONTROLLER:PLAINTEXT + networks: - default deploy: diff --git a/docker-images/kafka/Dockerfile b/docker-images/kafka/Dockerfile index e3c2623ba..43f699594 100644 --- a/docker-images/kafka/Dockerfile +++ b/docker-images/kafka/Dockerfile @@ -1,4 +1,4 @@ -FROM confluentinc/cp-kafka:7.1.1 +FROM confluentinc/cp-kafka:8.0.0 USER root RUN mkdir /opt/prometheus RUN chmod +rx /opt/prometheus