Openeuler2203系统zookeeper+kafka集群部署
一，具体环境
1，云主机
前三台用来安装zookeeper-3.9.1与kafka_2.13-3.6.1,Java 采用JAVA-11-openjdk版本
192.168.0.11 ecs-0001
192.168.0.12 ecs-0002
192.168.0.13 ecs-0003
192.168.0.14 ecs-0004
注意: kafka新版本可以使用kraft配置文件，不再需要zookeeper。
2，目录信息
/data/kafka 用来存放kafka文件
/data/zookeeper 用来存放zookeeper文件
/data/bin 用来存放启动脚本
二，部署步骤
1，主机环境配置
# 主机免密
ssh-keygen
for i in {11..14};do ssh-copy-id 192.168.0.${i};done
# 安装java-11
for i in {11..14};do ssh 192.168.0.${i} "yum install -y java-11";done
#配置hosts
for i in {1..4};do ssh 192.168.0.1${i} "sed -i '3d' /etc/hosts";done
for i in {1..4};do ssh 192.168.0.1${i} "echo "192.168.0.11 ecs-0001" >> /etc/hosts";done
for i in {1..4};do ssh 192.168.0.1${i} "echo "192.168.0.12 ecs-0002" >> /etc/hosts";done
for i in {1..4};do ssh 192.168.0.1${i} "echo "192.168.0.13 ecs-0003" >> /etc/hosts";done
for i in {1..4};do ssh 192.168.0.1${i} "echo "192.168.0.14 ecs-0004" >> /etc/hosts";done
2，zookeeper集群安装
安装步骤
# 创建目录与下载
for i in {1..4};do ssh 192.168.0.1${i} "mkdir -p /data/";done
cd /data/
wget https://mirrors.tuna.tsinghua.edu.cn/apache/kafka/3.6.1/kafka_2.13-3.6.1.tgz
wget https://mirrors.tuna.tsinghua.edu.cn/apache/zookeeper/zookeeper-3.9.1/apache-zookeeper-3.9.1-bin.tar.gz
# 解压文件
tar -zxvf apache-zookeeper-3.9.1-bin.tar.gz -C /data/
tar -zxvf kafka_2.13-3.6.1.tgz -C /data/
mv apache-zookeeper-3.9.1-bin zookeeper
mv kafka_2.13-3.6.1 kafka
#根据后面的内容将ecs-0001主机的zookkeeper/config/zoocfg配置文件内容修改下
#然后打包zookeeper目录
cd /data/
zip -r zookeeper.zip zookeeper
# 将打包文件传到其它主机上并解压
for i in {2..3};do scp /data/zookeeper.zip 192.168.0.1${i}:/data/;done
for i in {11..13};do ssh 192.168.0.${i} "cd /data && unzip zookeeper.zip";done
# 修改zookeeper的id
for i in {1..3};do ssh 192.168.0.1${i} " echo ${i} > /data/zookeeper/zkdata/myid";done
以下内容是zoo.cfg配置
dataDir=/data/zookeeper/zkdata
dataLogDir=/data/zookeeper/logs
server.1=ecs-0001:3188:3288
server.2=ecs-0002:3188:3288
server.3=ecs-0003:3188:3288
以下内容是批量启动zookeeper的脚本，放到 /data/bin/zookeeper.sh中
#!/bin/bash
ZK_HOME=/data/zookeeper
case $1 in
"start") {
for i in ecs-0001 ecs-0002 ecs-0003; do
echo ---------- zookeeper $i 启动 ------------
ssh $i "/data/zookeeper/bin/zkServer.sh start"
done
} ;;
"stop") {
for i in ecs-0001 ecs-0002 ecs-0003; do
echo ---------- zookeeper $i 停止 ------------
ssh $i "/data/zookeeper/bin/zkServer.sh stop"
done
} ;;
"status") {
for i in ecs-0001 ecs-0002 ecs-0003; do
echo ---------- zookeeper $i 状态 ------------
ssh $i "/data/zookeeper/bin/zkServer.sh status"
done
} ;;
esac
3，启动zookeeper
#启动zookeeper服务
[root@ecs-0001 bin]# ./zookeeper.sh start
---------- zookeeper ecs-0001 启动 ------------
Authorized users only. All activities may be monitored and reported.
/usr/bin/java
ZooKeeper JMX enabled by default
Using config: /data/zookeeper/bin/../conf/zoo.cfg
Starting zookeeper ... STARTED
---------- zookeeper ecs-0002 启动 ------------
/usr/bin/java
ZooKeeper JMX enabled by default
Using config: /data/zookeeper/bin/../conf/zoo.cfg
Starting zookeeper ... STARTED
---------- zookeeper ecs-0003 启动 ------------
/usr/bin/java
ZooKeeper JMX enabled by default
Using config: /data/zookeeper/bin/../conf/zoo.cfg
Starting zookeeper ... STARTED
#检测zookeeper服务状态,可以看到ecs-0002已经是Leader
[root@ecs-0001 bin]# ./zookeeper.sh status
---------- zookeeper ecs-0001 状态 ------------
/usr/bin/java
ZooKeeper JMX enabled by default
Using config: /data/zookeeper/bin/../conf/zoo.cfg
Client port found: 2181. Client address: localhost. Client SSL: false.
Mode: follower
---------- zookeeper ecs-0002 状态 ------------
/usr/bin/java
ZooKeeper JMX enabled by default
Using config: /data/zookeeper/bin/../conf/zoo.cfg
Client port found: 2181. Client address: localhost. Client SSL: false.
Mode: leader
---------- zookeeper ecs-0003 状态 ------------
/usr/bin/java
ZooKeeper JMX enabled by default
Using config: /data/zookeeper/bin/../conf/zoo.cfg
Client port found: 2181. Client address: localhost. Client SSL: false.
Mode: follower
4，kafka安装步骤
安装流程
#根据后面的内容将ecs-0001主机的kafka/config/kraft/server.properties配置文件内容修改下
#然后打包kafka目录
cd /data/
zip -r kafka.zip kafka
# 将打包文件传到其它主机上并解压
for i in {2..3};do scp /data/kafka.zip 192.168.0.1${i}:/data/;done
for i in {12..13};do ssh 192.168.0.${i} "cd /data && unzip kafka.zip";done
# 注意另外两台主机都要修改配置文件
kafka配置文件修改内容，每台主机都要修改
node.id=1
controller.quorum.voters=1@ecs-0001:9093,2@ecs-0002:9093,3@ecs-0003:9093
listeners=PLAINTEXT://ecs-0001:9092,CONTROLLER://ecs-0001:9093
advertised.listeners=PLAINTEXT://ecs-0001:9092
log.dirs=/data/kafka/logs
num.partitions=10
kafka集群初始化
[root@ecs-0001 bin]# /data/kafka/bin/kafka-storage.sh random-uuid
xJevzljvR76d1nnM8IANMQ
[root@ecs-0001 bin]#
for i in {1..3};do ssh 192.168.0.1${i} "/data/kafka/bin/kafka-storage.sh format -t xJevzljvR76d1nnM8IANMQ -c /data/kafka/config/kraft/server.properties";done
Formatting /data/kafka/logs with metadata.version 3.6-IV2.
Formatting /data/kafka/logs with metadata.version 3.6-IV2.
Formatting /data/kafka/logs with metadata.version 3.6-IV2.
[root@ecs-0001 bin]# cat /data/kafka/logs/meta.properties
#
#Wed Jan 31 14:55:41 CST 2024
node.id=1
version=1
cluster.id=xJevzljvR76d1nnM8IANMQ
以下内容是批量启动kafka的脚本，放到 /data/bin/kafka.sh中
#!/bin/bash
#KAFKA_HOME
export KAFKA_HOME=/data/kafka/
export PATH=$PATH:$KAFKA_HOME/bin
case $1 in
"start") {
for i in ecs-0001 ecs-0002 ecs-0003; do
echo ---------- kafka $i 启动 ------------
ssh $i "export KAFKA_HOME=/data/kafka/ && export PATH=$PATH:$KAFKA_HOME/bin && /data/kafka/bin/kafka-server-start.sh /data/kafka/config/server.properties"
done
} ;;
"stop") {
for i in ecs-0001 ecs-0002 ecs-0003; do
echo ---------- kafka $i 停止 ------------
ssh $i "export KAFKA_HOME=/data/kafka/ && export PATH=$PATH:$KAFKA_HOME/bin && /data/kafka/bin/kafka-server-stop.sh stop"
done
} ;;
esac
5，kafka功能测试
# 创建topic测试：
[root@kafka10]# ./bin/kafka-topics.sh --create --bootstrap-server ecs-0001:9092 --replication-factor 1 --partitions 1 --topic tp1
Created topic tp1.
# 查看所有的topic
[root@kafka1]# ./bin/kafka-topics.sh --list --bootstrap-server ecs-0003:9092
tp1
# 模拟一个数据
[root@kafka1]# ./bin/kafka-console-producer.sh --bootstrap-server ecs-0001:9092 --topic tp1
>测试数据
>ceshi1
# 起一个消费者：
[root@kafka1]# ./bin/kafka-console-consumer.sh --bootstrap-server ecs-0002:9092 --topic tp1
测试数据
ceshi1
# 到这里kafka集群已经ok
————————————————
目录 返回
首页