[大数据]Sentry之kafka与zk集群部署

一 zk集群部署

#部署zookeeper：
#下载安装包。
wget https://downloads.apache.org/zookeeper/zookeeper-3.6.3/apache-zookeeper-3.6.3-bin.tar.gz
#解压到data目录：
tar zxvf apache-zookeeper-3.6.3-bin.tar.gz
#查看配置文件：
[root@kafka-dohko-b9tip8 conf]# cat zoo.cfg
tickTime=2000
initLimit=10
syncLimit=5
dataDir=/data/apache-zookeeper-3.6.3-bin/data
clientPort=2181
dataLogDir=/data/apache-zookeeper-3.6.3-bin/logs
server.1=ip01:2888:3888
server.2=ip02:2888:3888
server.3=ip03:2888:3888
#每个节点不同的myid，手动生成。
[root@kafka-dohko-b9tip8 data]# pwd
/data/apache-zookeeper-3.6.3-bin/data
[root@kafka-dohko-b9tip8 data]# ls
myid  version-2  zookeeper_server.pid
[root@kafka-dohko-b9tip8 data]# cat myid
1
 
====================================
#复制第一个节点去其他几点的机器上：
#然后修改myid。
[root@kafka-dohko-5lzlg7 conf]# cat zoo.cfg
tickTime=2000
initLimit=10
syncLimit=5
dataDir=/data/apache-zookeeper-3.6.3-bin/data
clientPort=2181
dataLogDir=/data/apache-zookeeper-3.6.3-bin/logs
server.1=ip01:2888:3888
server.2=ip02:2888:3888
server.3=ip03:2888:3888
[root@kafka-dohko-5lzlg7 conf]# cat ../data/myid
2
 
================================================
[root@kafka-dohko-9s17pk apache-zookeeper-3.6.3-bin]# pwd
/data/apache-zookeeper-3.6.3-bin
[root@kafka-dohko-9s17pk apache-zookeeper-3.6.3-bin]# cat data/myid
3
[root@kafka-dohko-9s17pk apache-zookeeper-3.6.3-bin]# cat conf/zoo
zoo.cfg         zoo_sample.cfg  
[root@kafka-dohko-9s17pk apache-zookeeper-3.6.3-bin]# cat conf/zoo.cfg
tickTime=2000
initLimit=10
syncLimit=5
dataDir=/data/apache-zookeeper-3.6.3-bin/data
clientPort=2181
dataLogDir=/data/apache-zookeeper-3.6.3-bin/logs
server.1=ip01:2888:3888
server.2=ip02:2888:3888
server.3=ip03:2888:3888
 
 
 
===========================================
 启动的前提是你的机器上有java环境。
[root@kafka-dohko-9s17pk apache-zookeeper-3.6.3-bin]# java -version
java version "1.8.0_311"
Java(TM) SE Runtime Environment (build 1.8.0_311-b11)
Java HotSpot(TM) 64-Bit Server VM (build 25.311-b11, mixed mode)
 
 
=======================启动命令-=======================
三个节点一次启动，组成集群：
./zkServer.sh  start
常看节点状态
./zkServer.sh  status
 
[root@kafka-dohko-b9tip8 bin]# ./zkServer.sh  --help
ZooKeeper JMX enabled by default
Using config: /data/apache-zookeeper-3.6.3-bin/bin/../conf/zoo.cfg
Usage: ./zkServer.sh [--config <conf-dir>] {start|start-foreground|stop|version|restart|status|print-cmd}

二 kafka集群部署

#部署kafka集群：
wget http://archive.apache.org/dist/kafka/2.8.1/kafka_2.13-2.8.1.tgz
#解压：
tar zxvf kafka_2.13-2.8.1.tgz
#配置文件：
[root@kafka-dohko-b9tip8 config]# cat server.properties
broker.id=0
delete.topic.enable=true
listeners=PLAINTEXT://ip01:9092
advertised.listeners=PLAINTEXT://ip01:9092
num.network.threads=8
num.io.threads=8
socket.send.buffer.bytes=1024000
socket.receive.buffer.bytes=1024000
socket.request.max.bytes=1048576000
log.dirs=/data1/kafka_2.13-2.8.1/logs
num.partitions=1
num.recovery.threads.per.data.dir=1
offsets.topic.replication.factor=1
transaction.state.log.replication.factor=1
transaction.state.log.min.isr=1
log.retention.hours=168
log.segment.bytes=1073741824
log.retention.check.interval.ms=300000
zookeeper.connect=ip01:2181,ip02:2181,ip03:2181
zookeeper.connection.timeout.ms=6000
group.initial.rebalance.delay.ms=0
unclean.leader.election.enable = false
auto.leader.rebalance.enable = false

#注意：
#3个节点配置不同的broker.id和listeners、advertised.listeners即可。
 
#启动命令：
./bin/kafka-server-start.sh -daemon ./config/server.properties

三 kafka配置文件特殊配置

broker.id=0
delete.topic.enable=true
listeners=PLAINTEXT://172.20.184.232:9092
advertised.listeners=PLAINTEXT://172.20.184.232:9092
num.network.threads=8
num.io.threads=8
socket.send.buffer.bytes=1024000
socket.receive.buffer.bytes=1024000
socket.request.max.bytes=1048576000
log.dirs=/data1/kafka_2.13-2.8.1/logs
num.partitions=1
num.recovery.threads.per.data.dir=1
offsets.topic.replication.factor=1
transaction.state.log.replication.factor=1
transaction.state.log.min.isr=1
log.retention.hours=168
log.segment.bytes=1073741824
log.retention.check.interval.ms=300000
zookeeper.connect=ip01:2181,ip02:2181,ip03:2181
zookeeper.connection.timeout.ms=6000
group.initial.rebalance.delay.ms=0
unclean.leader.election.enable = false
auto.leader.rebalance.enable = false
message.max.bytes=2147483647
socket.request.max.bytes=2147483647 
replica.fetch.max.bytes=2147483647
fetch.message.max.bytes=2147483647
og.segment.bytes=2147483647  

如果是sentry使用的kafka 这边需要做一些参数的调优

message.max.bytes=2147483647

socket.request.max.bytes=2147483647

replica.fetch.max.bytes=2147483647

fetch.message.max.bytes=2147483647

og.segment.bytes=2147483647