7

docker-compose多服务器部署kafka集群 - 杭州红烧肉

 1 year ago
source link: https://www.cnblogs.com/chinalxx/p/17496277.html
Go to the source link to view the article. You can view the picture content, updated content and better typesetting reading experience. If the link is broken, please click the button below to view the snapshot at that time.
neoserver,ios ssh client
  • Kafka 是一个开源的分布式事件流平台，依赖Zookeeper或者KRaft，本文基于Zookeeper。

服务器IP配置

本文使用三个服务器来做集群搭建，IP如下：

nodeName IP
node1 10.10.210.96
node2 10.10.210.97
node3 10.10.210.98

部署zookeeper

  • 工作目录为/home/zookeeper

node1配置

- zookeeper
  - config
    - zoo.cfg
  - docker-compose.yml

zoo.cfg

dataDir=/data
dataLogDir=/datalog
tickTime=2000
initLimit=5
syncLimit=2
clientPort:2181
server.1=127.0.0.1:2888:3888
server.2=10.10.210.97:2888:3888
server.3=10.10.210.98:2888:3888

docker-compose.yml

version: '3'
services:
  zookeeper:
    image: zookeeper:3.7.0
    restart: always
    hostname: zookeeper-node-1
    container_name: zookeeper
    ports:
    - 2181:2181
    - 2888:2888
    - 3888:3888
    - 8080:8080
    volumes:
    - ./data:/data
    - ./datalog:/datalog
    - ./config/zoo.cfg:/conf/zoo.cfg
    environment:
      ZOO_MY_ID: 1

node2配置

- zookeeper
  - config
    - zoo.cfg
  - docker-compose.yml

zoo.cfg

dataDir=/data
dataLogDir=/datalog
tickTime=2000
initLimit=5
syncLimit=2
clientPort:2181
server.1=10.10.210.96:2888:3888
server.2=127.0.0.1:2888:3888
server.3=10.10.210.98:2888:3888

docker-compose.yml

version: '3'
services:
  zookeeper:
    image: zookeeper:3.7.0
    restart: always
    hostname: zookeeper-node-2
    container_name: zookeeper
    ports:
    - 2181:2181
    - 2888:2888
    - 3888:3888
    - 8080:8080
    volumes:
    - ./data:/data
    - ./datalog:/datalog
    - ./config/zoo.cfg:/conf/zoo.cfg
    environment:
      ZOO_MY_ID: 2

node3配置

- zookeeper
  - config
    - zoo.cfg
  - docker-compose.yml

zoo.cfg

dataDir=/data
dataLogDir=/datalog
tickTime=2000
initLimit=5
syncLimit=2
clientPort:2181
server.1=10.10.210.96:2888:3888
server.2=10.10.210.97:2888:3888
server.3=127.0.0.1:2888:3888

docker-compose.yml

version: '3'
services:
  zookeeper:
    image: zookeeper:3.7.0
    restart: always
    hostname: zookeeper-node-3
    container_name: zookeeper
    ports:
    - 2181:2181
    - 2888:2888
    - 3888:3888
    - 8080:8080
    volumes:
    - ./data:/data
    - ./datalog:/datalog
    - ./config/zoo.cfg:/conf/zoo.cfg
    environment:
      ZOO_MY_ID: 3
  • 在对应服务器的/home/zookeeper执行 docker-compose up -d 启动三个Zookeeper服务,通过docker-compose logs -f观察启动日志
  • ZOO_MY_ID 对应zookeeper的id,多台服务器需设置不同,对应zoo.cfg的server.1,其中.1 就是对应的ZOO_MY_ID
  • zoo.cfg配置信息具体可参考 Zookeeper部署和管理指南

部署kafka

  • 工作目录为/home/kafka

node1配置

- kafka
  - docker-compose.yml
  - config/server.properties

docker-compose.yml

version: '3'
services:
  kafka:
    image: bitnami/kafka:3.0.0
    restart: always
    hostname: kafka-node-1
    container_name: kafka
    ports:
    - 9092:9092
    - 9999:9999
    volumes:
    - ./logs:/opt/bitnami/kafka/logs
    - ./data:/bitnami/kafka/data
    - ./config/server.properties:/opt/bitnami/kafka/config/server.properties

server.properties

broker.id=1
listeners=PLAINTEXT://:9092
advertised.listeners=PLAINTEXT://10.10.210.96:9092
num.network.threads=3
num.io.threads=8
socket.send.buffer.bytes=102400
socket.receive.buffer.bytes=102400
socket.request.max.bytes=104857600
log.dirs=/bitnami/kafka/data
num.partitions=1
num.recovery.threads.per.data.dir=1
offsets.topic.replication.factor=1
transaction.state.log.replication.factor=1
transaction.state.log.min.isr=1
log.retention.hours=168
log.segment.bytes=1073741824
log.retention.check.interval.ms=300000
zookeeper.connect=10.10.210.96:2181,10.10.210.97:2181,10.10.210.98:2181
zookeeper.connection.timeout.ms=18000
group.initial.rebalance.delay.ms=0
auto.create.topics.enable=true
max.partition.fetch.bytes=1048576
max.request.size=1048576
sasl.enabled.mechanisms=PLAIN,SCRAM-SHA-256,SCRAM-SHA-512
sasl.mechanism.inter.broker.protocol=

node2配置

- kafka
  - docker-compose.yml
  - config/server.properties

docker-compose.yml

version: '3'
services:
  kafka:
    image: bitnami/kafka:3.0.0
    restart: always
    hostname: kafka-node-2
    container_name: kafka
    ports:
    - 9092:9092
    - 9999:9999
    volumes:
    - ./logs:/opt/bitnami/kafka/logs
    - ./data:/bitnami/kafka/data
    - ./config/server.properties:/opt/bitnami/kafka/config/server.properties

server.properties

broker.id=2
listeners=PLAINTEXT://:9092
advertised.listeners=PLAINTEXT://10.10.210.97:9092
num.network.threads=3
num.io.threads=8
socket.send.buffer.bytes=102400
socket.receive.buffer.bytes=102400
socket.request.max.bytes=104857600
log.dirs=/bitnami/kafka/data
num.partitions=1
num.recovery.threads.per.data.dir=1
offsets.topic.replication.factor=1
transaction.state.log.replication.factor=1
transaction.state.log.min.isr=1
log.retention.hours=168
log.segment.bytes=1073741824
log.retention.check.interval.ms=300000
zookeeper.connect=10.10.210.96:2181,10.10.210.97:2181,10.10.210.98:2181
zookeeper.connection.timeout.ms=18000
group.initial.rebalance.delay.ms=0
auto.create.topics.enable=true
max.partition.fetch.bytes=1048576
max.request.size=1048576
sasl.enabled.mechanisms=PLAIN,SCRAM-SHA-256,SCRAM-SHA-512
sasl.mechanism.inter.broker.protocol=

node3配置

- kafka
  - docker-compose.yml
  - config/server.properties

docker-compose.yml

version: '3'
services:
  kafka:
    image: bitnami/kafka:3.0.0
    restart: always
    hostname: kafka-node-3
    container_name: kafka
    ports:
    - 9092:9092
    - 9999:9999
    volumes:
    - ./logs:/opt/bitnami/kafka/logs
    - ./data:/bitnami/kafka/data
    - ./config/server.properties:/opt/bitnami/kafka/config/server.properties

server.properties

broker.id=3
listeners=PLAINTEXT://:9092
advertised.listeners=PLAINTEXT://10.10.210.98:9092
num.network.threads=3
num.io.threads=8
socket.send.buffer.bytes=102400
socket.receive.buffer.bytes=102400
socket.request.max.bytes=104857600
log.dirs=/bitnami/kafka/data
num.partitions=1
num.recovery.threads.per.data.dir=1
offsets.topic.replication.factor=1
transaction.state.log.replication.factor=1
transaction.state.log.min.isr=1
log.retention.hours=168
log.segment.bytes=1073741824
log.retention.check.interval.ms=300000
zookeeper.connect=10.10.210.96:2181,10.10.210.97:2181,10.10.210.98:2181
zookeeper.connection.timeout.ms=18000
group.initial.rebalance.delay.ms=0
auto.create.topics.enable=true
max.partition.fetch.bytes=1048576
max.request.size=1048576
sasl.enabled.mechanisms=PLAIN,SCRAM-SHA-256,SCRAM-SHA-512
sasl.mechanism.inter.broker.protocol=
  • 在对应服务器的/home/kafka执行 docker-compose up -d 启动三个Kafka服务,通过docker-compose logs -f观察启动日志
  • server.properties配置信息具体可参考 Kafka Broker Configs

kafka测试使用

  • 通过offset explorer测试连接kafka是否可用。
  • 如果想要简单配置的情况下，可以通过environment的方式启动kafka，参考如下：

docker-compose.yml

version: '3'
services:
  kafka:
    image: bitnami/kafka:3.0.0
    restart: always
    hostname: kafka-node
    container_name: kafka
    ports:
    - 9092:9092
    - 9999:9999
    environment:
      - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://10.10.210.96:9092
      - KAFKA_ADVERTISED_HOST_NAME=10.10.210.96
      - KAFKA_ADVERTISED_PORT=9092
      - KAFKA_ZOOKEEPER_CONNECT=10.10.210.96:2181,10.10.210.97:2181,10.10.210.98:2181
      - ALLOW_PLAINTEXT_LISTENER=yes
      - KAFKA_CFG_LISTENERS=PLAINTEXT://:9092
      - JMX_PORT=9999 
    volumes:
    - ./logs:/opt/bitnami/kafka/logs
    - ./data:/bitnami/kafka/data

report

Recommend

About Joyk


Aggregate valuable and interesting links.
Joyk means Joy of geeK