Kafka 集群安装

1. 安装Zookeeper

   1.1 下载地址：https://downloads.apache.org/zookeeper/

   wget https://downloads.apache.org/zookeeper/stable/apache-zookeeper-3.6.3-bin.tar.gz
   

   1.2 解压安装

   tar -xzvf apache-zookeeper-3.6.3-bin.tar.gz
   mv apache-zookeeper-3.6.3-bin zookeeper
   

   1.3 添加到环境变量

   vim /etc/profile
   # 添加如下内容
   export ZOOKEEPER_HOME=/opt/zookeeper
   export PATH=$PATH:$ZOOKEEPER_HOME/bin
   

   1.4 修改zookeeper配置文件

   cp /opt/zookeeper/conf/zoo_sample.cfg /opt/zookeeper/conf/zoo.cfg
   vim /opt/zookeeper/conf/zoo.cfg
   

   修改如下内容：

    # 数据存放路径
   dataDir=/opt/zookeeper/data
   
   # 末尾追加：
   server.1=node2:2888:3888
   server.2=node3:2888:3888
   server.3=node4:2888:3888
   

   1.5 创建节点ID

   mkdir /opt/zookeeper/data
   echo "1" > /opt/zookeeper/data/myid
   

   1.6 启动zookeeper

   zkServer.sh start
   # 使用jps 检测是否启动成功， QuorumPeerMain
   

   1.7 开机自启动

   cat > /etc/systemd/system/zookeeper.service << EOF
   [Unit]
   Description=zookeeper
   After=syslog.target network.target
   
   [Service]
   Type=forking
   # 指定zookeeper 日志文件路径，也可以在zkServer.sh 中定义
   Environment=ZOO_LOG_DIR=/opt/zookeeper/logs
   # 指定JDK路径，也可以在zkServer.sh 中定义
   Environment=JAVA_HOME=/opt/jdk
   ExecStart=/opt/zookeeper/bin/zkServer.sh start
   ExecStop=/opt/zookeeper/bin/zkServer.sh stop
   Restart=always
   User=root
   Group=root
   
   [Install]
   WantedBy=multi-user.target
   
   EOF
   

   重新加载服务

   systemctl daemon-reload
   

   开机自启动

   systemctl enable zookeeper
   

   查看zookeeper状态

   systemctl status zookeeper
   

   1.8 zookeeper 查看器

   https://issues.apache.org/jira/secure/attachment/12436620/ZooInspector.zip

2. 安装Kafka

   2.1 下载地址：https://downloads.apache.org/kafka/

   wget https://downloads.apache.org/kafka/2.7.0/kafka_2.13-2.7.0.tgz
   

   2.2 安装

   tar -xzvf kafka_2.13-2.7.0.tgz
   mv kafka_2.13-2.7.0 kafka
   

   2.3 添加到环境变量中

   vim /etc/profile
   
   JAVA_HOME=/opt/jdk
   ZOOKEEPER_HOME=/opt/zookeeper
   KAFKA_HOME=/opt/kafka
   PATH=$PATH:$ZOOKEEPER_HOME/bin:$JAVA_HOME/bin:$KAFKA_HOME/bin
   export JAVA_HOME ZOOKEEPER_HOME PATH
   
   source /etc/profile
   

   2.4 修改配置文件

   cp /opt/kafka/config/server.properties /opt/kafka/config/server.properties.backup
   
   cd /opt/kafka/config
   
   vim /opt/kafka/config/server.properties
   

   修改如下信息

   #broker 的全局唯一编号，不能重复
   broker.id=0
   #删除 topic 功能使能
   delete.topic.enable=true
   #kafka 运行日志存放的路径
   log.dirs=/opt/module/kafka/logs
   #配置连接 Zookeeper 集群地址
   zookeeper.connect=hadoop102:2181,hadoop103:2181,hadoop104:2181
   
   # -------------不需要修改------------------
   #处理网络请求的线程数量
   num.network.threads=3
   #用来处理磁盘 IO 的现成数量
   num.io.threads=8
   #发送套接字的缓冲区大小
   socket.send.buffer.bytes=102400
   #接收套接字的缓冲区大小
   socket.receive.buffer.bytes=102400
   #请求套接字的缓冲区大小
   socket.request.max.bytes=104857600
   #topic 在当前 broker 上的分区个数
   num.partitions=1
   #用来恢复和清理 data 下数据的线程数量
   num.recovery.threads.per.data.dir=1
   #segment 文件保留的最长时间，超时将被删除
   log.retention.hours=168
   

   2.5 启动集群

   启动集群

   cd /opt/kafka
   # 启动
   bin/kafka-server-start.sh -daemon config/server.properties
   
   # 关闭
   bin/kafka-server-stop.sh stop
   

   kafka 群起脚本（部分）

   for i in hadoop102 hadoop103 hadoop104
   do
   echo "========== $i ==========" 
   ssh $i '/opt/module/kafka/bin/kafka-server-start.sh -daemon 
   /opt/module/kafka/config/server.properties'
   done
   

   2.6 kafka 开机自启动

   cat > /etc/systemd/system/kafka.service << EOF
   [Unit]
   Description=kafka
   After=syslog.target network.target zookeeper.service
   
   [Service]
   Type=simple
   # 指定JDK路径，也可以在zkServer.sh 中定义
   Environment=JAVA_HOME=/opt/jdk
   ExecStart=/opt/kafka/bin/kafka-server-start.sh /opt/kafka/config/server.properties
   ExecStop=/opt/kafka/bin/kafka-server-stop.sh stop
   Restart=always
   User=root
   Group=root
   
   [Install]
   WantedBy=multi-user.target
   
   EOF
   

   刷新配置文件

   systemctl daemon-reload
   
   # 开机启动
   systemctl enable kafka
   
   # 启动服务
   systemctl start kafka
   # 关闭服务
   systemctl stop kafka
   # 查看状态
   systemctl status kafka