[大数据]Kafka(三) 集群环境搭建

Kafka 集群环境搭建

最好参考之前的单机版本一起学习

准备三台虚拟机

1. 环境准备

先把单机版本的虚拟机克隆出三台 环境基本都是有的，下面就是修改软件里面的配置； 修改IP

第一台

[root@localhost package]# java -version
java version "1.8.0_51"
Java(TM) SE Runtime Environment (build 1.8.0_51-b16)
Java HotSpot(TM) 64-Bit Server VM (build 25.51-b03, mixed mode)
[root@localhost package]# ifconfig
ens33: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1500
        inet 192.168.141.131  netmask 255.255.255.0  broadcast 192.168.141.255
        inet6 fe80::9be5:3c62:732d:29cd  prefixlen 64  scopeid 0x20<link>
        inet6 fe80::e063:9e44:5283:b012  prefixlen 64  scopeid 0x20<link>
        inet6 fe80::e126:19f1:2090:3682  prefixlen 64  scopeid 0x20<link>
        ether 00:0c:29:91:5a:19  txqueuelen 1000  (Ethernet)
        RX packets 653  bytes 143724 (140.3 KiB)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 208  bytes 21087 (20.5 KiB)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

lo: flags=73<UP,LOOPBACK,RUNNING>  mtu 65536
        inet 127.0.0.1  netmask 255.0.0.0
        inet6 ::1  prefixlen 128  scopeid 0x10<host>
        loop  txqueuelen 1000  (Local Loopback)
        RX packets 68  bytes 5920 (5.7 KiB)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 68  bytes 5920 (5.7 KiB)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

第二台

[root@localhost ~]# java -version
java version "1.8.0_51"
Java(TM) SE Runtime Environment (build 1.8.0_51-b16)
Java HotSpot(TM) 64-Bit Server VM (build 25.51-b03, mixed mode)
[root@localhost ~]# ifconfig
ens33: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1500
        inet 192.168.141.132  netmask 255.255.255.0  broadcast 192.168.141.255
        inet6 fe80::9be5:3c62:732d:29cd  prefixlen 64  scopeid 0x20<link>
        inet6 fe80::e126:19f1:2090:3682  prefixlen 64  scopeid 0x20<link>
        ether 00:0c:29:89:8d:46  txqueuelen 1000  (Ethernet)
        RX packets 933  bytes 226961 (221.6 KiB)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 188  bytes 18301 (17.8 KiB)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

lo: flags=73<UP,LOOPBACK,RUNNING>  mtu 65536
        inet 127.0.0.1  netmask 255.0.0.0
        inet6 ::1  prefixlen 128  scopeid 0x10<host>
        loop  txqueuelen 1000  (Local Loopback)
        RX packets 68  bytes 5920 (5.7 KiB)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 68  bytes 5920 (5.7 KiB)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

第三台

[root@localhost ~]# 
[root@localhost ~]# java -version
java version "1.8.0_51"
Java(TM) SE Runtime Environment (build 1.8.0_51-b16)
Java HotSpot(TM) 64-Bit Server VM (build 25.51-b03, mixed mode)
[root@localhost ~]# ifconfig
ens33: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1500
        inet 192.168.141.133  netmask 255.255.255.0  broadcast 192.168.141.255
        inet6 fe80::9be5:3c62:732d:29cd  prefixlen 64  scopeid 0x20<link>
        ether 00:0c:29:57:64:6d  txqueuelen 1000  (Ethernet)
        RX packets 1854  bytes 400234 (390.8 KiB)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 501  bytes 80498 (78.6 KiB)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

lo: flags=73<UP,LOOPBACK,RUNNING>  mtu 65536
        inet 127.0.0.1  netmask 255.0.0.0
        inet6 ::1  prefixlen 128  scopeid 0x10<host>
        loop  txqueuelen 1000  (Local Loopback)
        RX packets 68  bytes 5920 (5.7 KiB)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 68  bytes 5920 (5.7 KiB)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

2. 配置hosts 文件

三台使用一个配置文件

127.0.0.1   localhost localhost.localdomain localhost4 localhost4.localdomain4
::1         localhost localhost.localdomain localhost6 localhost6.localdomain6
192.168.141.131 CentOSA
192.168.141.132 CentOSB
192.168.141.133 CentOSC

##复制到其他两台
scp /etc/hosts  CentOSB:/etc/
scp /etc/hosts  CentOSC:/etc/

3.关闭防火墙

[root@localhost package]# systemctl stop firewalld
[root@localhost package]# systemctl disable firewalld

4.时钟同步

时钟的头部服务器地址:net[1-7].aliyun.com

[root@localhost ~]# yum install ntp -y
[root@localhost package]# ntpdate ntp1.aliyun.com
22 Sep 23:09:25 ntpdate[1975]: adjust time server 120.25.115.20 offset 0.000890 sec
[root@localhost package]# clock -w

5.安装zookeeper

修改zoo.cfg文件

##和单机版本一样  需要手动创建这个文件夹
dataDir=/package/zkdata
.......
# 集群配置  集群的每个电脑都这样配置
server.1=CentOSA:2888:3888
server.2=CentOSB:2888:3888
server.3=CentOSC:2888:3888

###
[root@CentOSA conf]# scp zoo.cfg CentOSB:/package/zookeeper/conf/
root@centosb's password: 
zoo.cfg                                                        100% 1019     1.1MB/s   00:00    
[root@CentOSA conf]# scp zoo.cfg CentOSC:/package/zookeeper/conf/
root@centosc's password: 
zoo.cfg  

根据zoo.cfg 在dataDir 对应的文件夹中创建文件 保存id

##第一台电脑
[root@CentOSA zkdata]# echo 1 > /package/zkdata/myid
##第二台电脑
[root@CentOSB conf]# echo 2 > /package/zkdata/myid
##第三台电脑
[root@CentOSC ~]# echo 3 > /package/zkdata/myid

6.启动zookeeper

第一台

[root@CentOSA zkdata]# /package/zookeeper/bin/zkServer.sh  start /package/zookeeper/conf/zoo.cfg JMX enabled by default
Using config: /package/zookeeper/conf/zoo.cfg
Starting zookeeper ... STARTED
##集群其他还没有开
[root@CentOSA zkdata]# /package/zookeeper/bin/zkServer.sh  status /package/zookeeper/conf/zoo.cfg 
JMX enabled by default
Using config: /package/zookeeper/conf/zoo.cfg
Error contacting service. It is probably not running.
##集群其他打开
[root@CentOSA zkdata]# /package/zookeeper/bin/zkServer.sh  status /package/zookeeper/conf/zoo.cfg 
JMX enabled by default
Using config: /package/zookeeper/conf/zoo.cfg
Mode: follower

第二台

[root@CentOSB conf]# /package/zookeeper/bin/zkServer.sh  start /package/zookeeper/conf/zoo.cfg
JMX enabled by default
Using config: /package/zookeeper/conf/zoo.cfg
Starting zookeeper ... STARTED
[root@CentOSB conf]# /package/zookeeper/bin/zkServer.sh  status /package/zookeeper/conf/zoo.cfg
JMX enabled by default
Using config: /package/zookeeper/conf/zoo.cfg
Mode: leader

第三台

[root@CentOSC ~]# /package/zookeeper/bin/zkServer.sh  start /package/zookeeper/conf/zoo.cfg
JMX enabled by default
Using config: /package/zookeeper/conf/zoo.cfg
Starting zookeeper ... STARTED
[root@CentOSC ~]# /package/zookeeper/bin/zkServer.sh  status /package/zookeeper/conf/zoo.cfg
JMX enabled by default
Using config: /package/zookeeper/conf/zoo.cfg
Mode: follower

7.安装Kafka

第一台

# see kafka.server.KafkaConfig for additional details and defaults  不需要修改  因为是第一个
broker.id=0

##修改配置文件 server.properties  CentOSC 是本机
listeners=PLAINTEXT://CentOSA:9092
## 同单机版
log.dirs=/package/kfkdata
############################# Zookeeper #############################
zookeeper.connect=CentOSA:2181,CentOSB:2181,CentOSC:2181

第二台

# see kafka.server.KafkaConfig for additional details and defaults  不需要修改  因为是第一个
broker.id=1

##修改配置文件 server.properties  CentOSC 是本机
listeners=PLAINTEXT://CentOSB:9092
## 同单机版
log.dirs=/package/kfkdata
############################# Zookeeper #############################
zookeeper.connect=CentOSA:2181,CentOSB:2181,CentOSC:2181

第三台

# see kafka.server.KafkaConfig for additional details and defaults  不需要修改  因为是第一个
broker.id=2

##修改配置文件 server.properties  CentOSC 是本机
listeners=PLAINTEXT://CentOSC:9092
## 同单机版
log.dirs=/package/kfkdata
############################# Zookeeper #############################
zookeeper.connect=CentOSA:2181,CentOSB:2181,CentOSC:2181

8.启动Kafka

./bin/kafka-server-start.sh -daemon config/server.properties

注意：中间由于把kafka单机的包直接拷贝过来，里面有原来单机版本跑过的数据，导致起不来

Kafka：Configured broker.id 2 doesn't match stored broker.id 0 in meta.properties.

解决方案 修改 server.properties 中的log.dirs文件夹下面的文件：meta.properties 中的 broker.id 的值为 server.properties 里面配置的broker.id 一样就可以了