[大数据]kafka常用指令测试总结

1.各个节点启动zookeeper ：./bin/zkServer.sh start
master和从节点都要启动；

[root@slave2 zookeeper-3.4.11]# ./bin/zkServer.sh start
ZooKeeper JMX enabled by default
Using config: /usr/local/src/zookeeper-3.4.11/bin/../conf/zoo.cfg
Starting zookeeper ... STARTED
[root@slave2 zookeeper-3.4.11]# 

2.启动kafka:
./bin/kafka-server-start.sh config/server.properties

[root@master kafka_2.11-0.10.2.1]# 
[root@master kafka_2.11-0.10.2.1]# ./bin/kafka-server-start.sh config/server.properties 
[2020-06-23 21:51:19,880] INFO KafkaConfig values: 
	advertised.host.name = null
	advertised.listeners = null
	advertised.port = null

3.查看主题：

root@master kafka_2.11-0.10.2.1]# 
[root@master kafka_2.11-0.10.2.1]# ./bin/kafka-topics.sh --list --zookeeper localhost:2181
__consumer_offsets
test
[root@master kafka_2.11-0.10.2.1]# 

4.查看主题详情：

[root@master kafka_2.11-0.10.2.1]# ./bin/kafka-topics.sh --describe --zookeeper localhost:2181 --topic test
Topic:test	PartitionCount:1	ReplicationFactor:1	Configs:
	Topic: test	Partition: 0	Leader: 0	Replicas: 0	Isr: 0
[root@master kafka_2.11-0.10.2.1]# 
[root@master kafka_2.11-0.10.2.1]# 

5.启动消息生产者：

./bin/kafka-console-producer.sh --broker-list master:9092 --topic test
[root@master kafka_2.11-0.10.2.1]# 
[root@master kafka_2.11-0.10.2.1]# 
[root@master kafka_2.11-0.10.2.1]# 
[root@master kafka_2.11-0.10.2.1]# ./bin/kafka-console-producer.sh --broker-list master:9092 --topic test

6.启动消费者消费消息：

./bin/kafka-console-consumer.sh --zookeeper master:2181 --topic test 
[root@master kafka_2.11-0.10.2.1]# 
[root@master kafka_2.11-0.10.2.1]# 
[root@master kafka_2.11-0.10.2.1]# ./bin/kafka-console-consumer.sh --zookeeper master:2181 --topic test 
Using the ConsoleConsumer with old consumer is deprecated and will be removed in a future major release. Consider using the new consumer by passing [bootstrap-server] instead of [zookeeper].

7.查看topic的情况：

[root@master kafka_2.11-0.10.2.1]# 
[root@master kafka_2.11-0.10.2.1]# ./bin/kafka-topics.sh --describe --zookeeper localhost:2181 --topic test
Topic:test	PartitionCount:1	ReplicationFactor:1	Configs:
	Topic: test	Partition: 0	Leader: 0	Replicas: 0	Isr: 0
[root@master kafka_2.11-0.10.2.1]# 

8. 修改消费者以群组消费：

[root@master kafka_2.11-0.10.2.1]# 
[root@master kafka_2.11-0.10.2.1]# 
[root@master kafka_2.11-0.10.2.1]# ./bin/kafka-console-consumer.sh --zookeeper master:2181 --topic test --consumer-property group.id=group_test
Using the ConsoleConsumer with old consumer is deprecated and will be removed in a future major release. Consider using the new consumer by passing [bootstrap-server] instead of [zookeeper].

9.访问zk;

root@master zookeeper-3.4.11]# ./bin/zkCli.sh 
Connecting to localhost:2181
2020-06-22 23:32:32,005 [myid:] - INFO  [main:Environment@100] - Client environment:zookeeper.version=3.4.11-37e277162d567b55a07d1755f0b31c32e93c01a0, built on 11/01/2017 18:06 GMT
2020-06-22 23:32:32,008 [myid:] - INFO  [main:Environment@100] - Client environment:host.name=master
2020-06-22 23:32:32,008 [myid:] - INFO  [main:Environment@100] - Client environment:java.version=1.8.0_221
2020-06-22 23:32:32,010 [myid:] - INFO  [main:Environment@100] - Client environment:java.vendor=Oracle Corporation
2020-06-22 23:32:32,011 [myid:] - INFO  [main:Environment@100] - Client environment:java.home=/usr/local/src/jdk1.8.0_221/jre
2020-06-22 23:32:32,011 [myid:] - INFO  [main:Environment@100] - Client environment:java.class.path=/usr/local/src/zookeeper-3.4.11/bin/../build/classes:/usr/local/src/zookeeper-3.4.11/bin/../build/lib/*.jar:/usr/local/src/zookeeper-3.4.11/bin/../lib/slf4j-log4j12-1.6.1.jar:/usr/local/src/zookeeper-3.4.11/bin/../lib/slf4j-api-1.6.1.jar:/usr/local/src/zookeeper-3.4.11/bin/../lib/netty-3.10.5.Final.jar:/usr/local/src/zookeeper-3.4.11/bin/../lib/log4j-1.2.16.jar:/usr/local/src/zookeeper-3.4.11/bin/../lib/jline-0.9.94.jar:/usr/local/src/zookeeper-3.4.11/bin/../lib/audience-annotations-0.5.0.jar:/usr/local/src/zookeeper-3.4.11/bin/../zookeeper-3.4.11.jar:/usr/local/src/zookeeper-3.4.11/bin/../src/java/lib/*.jar:/usr/local/src/zookeeper-3.4.11/bin/../conf:.::/usr/local/src/jdk1.8.0_221/lib
2020-06-22 23:32:32,011 [myid:] - INFO  [main:Environment@100] - Client environment:java.library.path=/usr/java/packages/lib/amd64:/usr/lib64:/lib64:/lib:/usr/lib
2020-06-22 23:32:32,011 [myid:] - INFO  [main:Environment@100] - Client environment:java.io.tmpdir=/tmp
2020-06-22 23:32:32,011 [myid:] - INFO  [main:Environment@100] - Client environment:java.compiler=<NA>
2020-06-22 23:32:32,011 [myid:] - INFO  [main:Environment@100] - Client environment:os.name=Linux
2020-06-22 23:32:32,011 [myid:] - INFO  [main:Environment@100] - Client environment:os.arch=amd64
2020-06-22 23:32:32,011 [myid:] - INFO  [main:Environment@100] - Client environment:os.version=3.10.0-1062.el7.x86_64
2020-06-22 23:32:32,011 [myid:] - INFO  [main:Environment@100] - Client environment:user.name=root
2020-06-22 23:32:32,012 [myid:] - INFO  [main:Environment@100] - Client environment:user.home=/root
2020-06-22 23:32:32,012 [myid:] - INFO  [main:Environment@100] - Client environment:user.dir=/usr/local/src/zookeeper-3.4.11
2020-06-22 23:32:32,013 [myid:] - INFO  [main:ZooKeeper@441] - Initiating client connection, connectString=localhost:2181 sessionTimeout=30000 watcher=org.apache.zookeeper.ZooKeeperMain$MyWatcher@68de145
Welcome to ZooKeeper!
JLine support is enabled
2020-06-22 23:32:32,093 [myid:] - INFO  [main-SendThread(localhost:2181):ClientCnxn$SendThread@1035] - Opening socket connection to server localhost/0:0:0:0:0:0:0:1:2181. Will not attempt to authenticate using SASL (unknown error)
2020-06-22 23:32:32,193 [myid:] - INFO  [main-SendThread(localhost:2181):ClientCnxn$SendThread@877] - Socket connection established to localhost/0:0:0:0:0:0:0:1:2181, initiating session
2020-06-22 23:32:32,206 [myid:] - INFO  [main-SendThread(localhost:2181):ClientCnxn$SendThread@1302] - Session establishment complete on server localhost/0:0:0:0:0:0:0:1:2181, sessionid = 0x100010afa8f000e, negotiated timeout = 30000

WATCHER::

WatchedEvent state:SyncConnected type:None path:null
[zk: localhost:2181(CONNECTED) 0] ls
[zk: localhost:2181(CONNECTED) 1] 
[zk: localhost:2181(CONNECTED) 1] 

10.在zk下操作consumer：

[root@master zookeeper-3.4.11]# ./bin/zkCli.sh 
Connecting to localhost:2181
2020-06-22 23:32:32,005 [myid:] - INFO  [main:Environment@100] - Client environment:zookeeper.version=3.4.11-37e277162d567b55a07d1755f0b31c32e93c01a0, built on 11/01/2017 18:06 GMT
2020-06-22 23:32:32,008 [myid:] - INFO  [main:Environment@100] - Client environment:host.name=master
2020-06-22 23:32:32,008 [myid:] - INFO  [main:Environment@100] - Client environment:java.version=1.8.0_221
2020-06-22 23:32:32,010 [myid:] - INFO  [main:Environment@100] - Client environment:java.vendor=Oracle Corporation
2020-06-22 23:32:32,011 [myid:] - INFO  [main:Environment@100] - Client environment:java.home=/usr/local/src/jdk1.8.0_221/jre
2020-06-22 23:32:32,011 [myid:] - INFO  [main:Environment@100] - Client environment:java.class.path=/usr/local/src/zookeeper-3.4.11/bin/../build/classes:/usr/local/src/zookeeper-3.4.11/bin/../build/lib/*.jar:/usr/local/src/zookeeper-3.4.11/bin/../lib/slf4j-log4j12-1.6.1.jar:/usr/local/src/zookeeper-3.4.11/bin/../lib/slf4j-api-1.6.1.jar:/usr/local/src/zookeeper-3.4.11/bin/../lib/netty-3.10.5.Final.jar:/usr/local/src/zookeeper-3.4.11/bin/../lib/log4j-1.2.16.jar:/usr/local/src/zookeeper-3.4.11/bin/../lib/jline-0.9.94.jar:/usr/local/src/zookeeper-3.4.11/bin/../lib/audience-annotations-0.5.0.jar:/usr/local/src/zookeeper-3.4.11/bin/../zookeeper-3.4.11.jar:/usr/local/src/zookeeper-3.4.11/bin/../src/java/lib/*.jar:/usr/local/src/zookeeper-3.4.11/bin/../conf:.::/usr/local/src/jdk1.8.0_221/lib
2020-06-22 23:32:32,011 [myid:] - INFO  [main:Environment@100] - Client environment:java.library.path=/usr/java/packages/lib/amd64:/usr/lib64:/lib64:/lib:/usr/lib
2020-06-22 23:32:32,011 [myid:] - INFO  [main:Environment@100] - Client environment:java.io.tmpdir=/tmp
2020-06-22 23:32:32,011 [myid:] - INFO  [main:Environment@100] - Client environment:java.compiler=<NA>
2020-06-22 23:32:32,011 [myid:] - INFO  [main:Environment@100] - Client environment:os.name=Linux
2020-06-22 23:32:32,011 [myid:] - INFO  [main:Environment@100] - Client environment:os.arch=amd64
2020-06-22 23:32:32,011 [myid:] - INFO  [main:Environment@100] - Client environment:os.version=3.10.0-1062.el7.x86_64
2020-06-22 23:32:32,011 [myid:] - INFO  [main:Environment@100] - Client environment:user.name=root
2020-06-22 23:32:32,012 [myid:] - INFO  [main:Environment@100] - Client environment:user.home=/root
2020-06-22 23:32:32,012 [myid:] - INFO  [main:Environment@100] - Client environment:user.dir=/usr/local/src/zookeeper-3.4.11
2020-06-22 23:32:32,013 [myid:] - INFO  [main:ZooKeeper@441] - Initiating client connection, connectString=localhost:2181 sessionTimeout=30000 watcher=org.apache.zookeeper.ZooKeeperMain$MyWatcher@68de145
Welcome to ZooKeeper!
JLine support is enabled
2020-06-22 23:32:32,093 [myid:] - INFO  [main-SendThread(localhost:2181):ClientCnxn$SendThread@1035] - Opening socket connection to server localhost/0:0:0:0:0:0:0:1:2181. Will not attempt to authenticate using SASL (unknown error)
2020-06-22 23:32:32,193 [myid:] - INFO  [main-SendThread(localhost:2181):ClientCnxn$SendThread@877] - Socket connection established to localhost/0:0:0:0:0:0:0:1:2181, initiating session
2020-06-22 23:32:32,206 [myid:] - INFO  [main-SendThread(localhost:2181):ClientCnxn$SendThread@1302] - Session establishment complete on server localhost/0:0:0:0:0:0:0:1:2181, sessionid = 0x100010afa8f000e, negotiated timeout = 30000

WATCHER::

WatchedEvent state:SyncConnected type:None path:null
[zk: localhost:2181(CONNECTED) 0] ls
[zk: localhost:2181(CONNECTED) 1] 
[zk: localhost:2181(CONNECTED) 1] ls /
[cluster, controller_epoch, controller, brokers, zookeeper, admin, isr_change_notification, consumers, config]
[zk: localhost:2181(CONNECTED) 2] ls /consumers
[console-consumer-93103, group_test]
[zk: localhost:2181(CONNECTED) 3]