今天我们来品一品这个ELK哈，首先要明确概念，ELK是什么呢，还是像往常一样，百度一哈👾
关于ELK
?Logstash
Logstash概述
?LogStash的主要组件
?LogStash主机分类
ElasticSearch
ElasticSearch概述
Elasticsearch的基础核心概念
主要功能：
?
ELK的日志处理流程
?
?部署ELK日志分析系统
搭建环境
node1 192.168.152.130 主要软件：Elasticsearch kibana
node2 192.168.152.129 主要软件： Elasticsearch
web 192.168.152.12 主要软件： logstash apache搭建过程
配置elasticsearch环境
首先三台都关闭防火墙，修改主机名字，这里以node1主机为例：
[root@server ~]# systemctl stop firewalld.service
[root@server ~]# systemctl disable firewalld.service
Removed symlink /etc/systemd/system/multi-user.target.wants/firewalld.service.
Removed symlink /etc/systemd/system/dbus-org.fedoraproject.FirewallD1.service.
[root@server ~]# setenforce 0
[root@server ~]# hostname node1
[root@server ~]# su
三台主机都安装Java环境：
[root@node1 ~]# java -version
openjdk version "1.8.0_181"
#从本机里导入安装包
[root@node1 ~]# ls
anaconda-ks.cfg initial-setup-ks.cfg jdk-8u91-linux-x64.tar.gz 公共 模板 视频 图片 文档 下载 音乐 桌面
[root@node1 ~]# tar xf jdk-8u91-linux-x64.tar.gz -C /usr/local/ #解压
[root@node1 ~]# cd /usr/local/
[root@node1 local]# ls
bin etc games include jdk1.8.0_91 lib lib64 libexec sbin share src
[root@node1 local]# mv jdk1.8.0_91/ jdk
[root@node1 local]# vim /etc/profile
#在里面添加进以下内容
export JAVA_HOME=/usr/local/jdk
export JRE_HOME=${JAVA_HOME}/jre
export CLASSPATH=.:${JAVA_HOME}/lib:${JRE_HOME}/lib
export PATH=${JAVA_HOME}/bin:$PATH
[root@node1 local]# source /etc/profile
[root@node1 local]# java -version
java version "1.8.0_91"
Java(TM) SE Runtime Environment (build 1.8.0_91-b14)
Java HotSpot(TM) 64-Bit Server VM (build 25.91-b14, mixed mode)
设置本地主机映射文件，node1和node2节点操作：
[root@node1 ~]# vim /etc/hosts
127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4
::1 localhost localhost.localdomain localhost6 localhost6.localdomain6
192.168.152.130 node1
192.168.152.129 node2配置elasticsearch软件（node1.node2做相同操作）
[root@node1 local]# cd /opt
#上传安装包
[root@node1 opt]# ls
elasticsearch-5.5.0.rpm elasticsearch-head.tar.gz httpd-2.4.6-95.el7.centos.x86_64.rpm rh
[root@node1 opt]#
[root@node1 opt]# rpm -ivh elasticsearch-5.5.0.rpm
#加载系统服务
[root@node1 opt]# systemctl daemon-reload
[root@node1 opt]# systemctl enable elasticsearch.service
Created symlink from /etc/systemd/system/multi-user.target.wants/elasticsearch.service to /usr/lib/systemd/system/elasticsearch.service.
[root@node1 opt]#
#修改主配置文件
[root@node1 opt]# cd /etc/elasticsearch/
[root@node1 elasticsearch]# cp -a elasticsearch.yml elasticsearch.yml.bak
[root@node1 elasticsearch]# vim elasticsearch.yml
17/ cluster.name: my-elk-cluster
#集群名字
23/ node.name: node1
#节点名宁字
33/ path.data: /data/elk_data
#数据存放路径
37/ path. logs: /var/log/elasticsearch/
#日志存放路径
43/ bootstrap.memory_lock: false
#锁定物理内存地址，防止es内存被交换出去，也就是避免es使用swap交换分区，频繁的交换，会导致Ios变高(性能测试:每秒的读写次数)。
55/ network.host: 0.0.0.0
#提供服务绑定的IP地址，0.0.0.0代表所有地址
59/ http.port: 9200
#侦听端口为9200
68/ discoveryp zen.ping.unicast.hosts:["node1", "node2"]
#集群发现通过单播实现单播
[root@node1 elasticsearch]# grep -v "^#" /etc/elasticsearch/elasticsearch.yml
cluster.name: my-elk-cluster
node.name: node1
path.data: /data/elk_data
path.logs: /var/log/elasticsearch/
bootstrap.memory_lock: false
network.host: 0.0.0.0
http.port: 9200
discovery.zen.ping.unicast.hosts: ["node1", "node2"]
[root@node1 elasticsearch]#
#创建数据存放路径并授权
[root@node1 elasticsearch]# mkdir -p /data/elk_data
[root@node1 elasticsearch]# chown elasticsearch:elasticsearch /data/elk_data/
#启动elasticsearch
[root@node1 elasticsearch]# systemctl start elasticsearch.service
[root@node1 elasticsearch]# netstat -antp | grep 9200
tcp6 0 0 :::9200 :::* LISTEN 14330/java查看节点信息
?
查看集群健康状态信息
?
?安装elasticsearch-head插件（node1.node2做相同操作）
#安装node组件
[root@node1 ~]# cd /opt
[root@node1 opt]# yum install gcc gcc-c++ make -y
[root@node1 opt]# ls
elasticsearch-5.5.0.rpm elasticsearch-head.tar.gz httpd-2.4.6-95.el7.centos.x86_64.rpm node-v8.2.1.tar.gz rh
[root@node1 opt]# tar xzvf node-v8.2.1.tar.gz
[root@node1 opt]# cd node-v8.2.1/
[root@node1 node-v8.2.1]# ./configure
[root@node1 node-v8.2.1]# make -j3
[root@node1 node-v8.2.1]# make install
#安装phantomjs前端框架
[root@node1 node-v8.2.1]# cd ..
[root@node1 opt]# ls
elasticsearch-5.5.0.rpm httpd-2.4.6-95.el7.centos.x86_64.rpm node-v8.2.1.tar.gz rh
elasticsearch-head.tar.gz node-v8.2.1 phantomjs-2.1.1-linux-x86_64.tar.bz2
[root@node1 opt]#
[root@node1 opt]# tar xjvf phantomjs-2.1.1-linux-x86_64.tar.bz2 -C /usr/local/src/
[root@node1 opt]# cd /usr/local/src/
[root@node1 src]# ls
phantomjs-2.1.1-linux-x86_64
[root@node1 phantomjs-2.1.1-linux-x86_64]# ls
bin ChangeLog examples LICENSE.BSD README.md third-party.txt
[root@node1 phantomjs-2.1.1-linux-x86_64]# cd bin
[root@node1 bin]# ls
phantomjs
[root@node1 bin]# cp phantomjs /usr/local/bin/
[root@node1 bin]#
#安装elasticsearch-head数据可视化工具
[root@node1 bin]# cd /opt
[root@node1 opt]# tar xzvf elasticsearch-head.tar.gz -C /usr/local/src
[root@node1 opt]# cd /usr/local/src/
[root@node1 src]# ls
elasticsearch-head phantomjs-2.1.1-linux-x86_64
[root@node1 src]# cd elasticsearch-head/
[root@node1 elasticsearch-head]# npm install
#修改主配置文件
[root@node1 elasticsearch-head]# cd
[root@node1 ~]# vim /etc/elasticsearch/elasticsearch.yml
#在最后补充
http.cors.enabled: true #开启跨域访问支持，默认为false
http.cors.allow-origin: "*" #跨域访问允许的域名地址
#启动elasticsearch-head
[root@node1 ~]# cd /usr/local/src/elasticsearch-head/
[root@node1 elasticsearch-head]# npm run start &
[1] 60746
[root@node1 elasticsearch-head]#
> elasticsearch-head@0.0.0 start /usr/local/src/elasticsearch-head
> grunt server
Running "connect:server" (connect) task
Waiting forever...
Started connect web server on http://localhost:9100?
[root@node1 elasticsearch-head]# npm run start &
[1] 64014
[root@node1 elasticsearch-head]#
> elasticsearch-head@0.0.0 start /usr/local/src/elasticsearch-head
> grunt server
Running "connect:server" (connect) task
Waiting forever...
Started connect web server on http://localhost:9100
[root@node1 ~]# netstat -antp | grep 9100
[root@node2 elasticsearch-head]# npm run start &
[1] 119353
[root@node2 elasticsearch-head]#
> elasticsearch-head@0.0.0 start /usr/local/src/elasticsearch-head
> grunt server
Running "connect:server" (connect) task
Waiting forever...
Started connect web server on http://localhost:9100
[root@node2 ~]# netstat -antp | grep 9100 ?
?
[root@node1 elasticsearch-head]# cd
[root@node1 ~]#
[root@node1 ~]#
[root@node1 ~]# curl -XPUT 'localhost:9200/index-demo/test/1?pretty&pretty' -H 'content-Type: application/json' -d '{"user":"aaa","mesg":"hello world"}'
{
"_index" : "index-demo",
"_type" : "test",
"_id" : "1",
"_version" : 1,
"result" : "created",
"_shards" : {
"total" : 2,
"successful" : 2,
"failed" : 0
},
"created" : true
}?
?
?
apache服务器部署logstash相关
?安装服务并开启服务
[root@apache ~]# yum -y install httpd
[root@apache ~]# systemctl start httpd
[root@apache ~]# netstat -ntap |grep httpd
tcp6 0 0 :::80 :::* LISTEN 19669/httpd
[root@apache ~]#
#安装logstash服务并启动
[root@apache ~]# cd /opt
[root@apache opt]# ls
logstash-5.5.1.rpm rh
[root@apache opt]# rpm -ivh logstash-5.5.1.rpm
[root@apache opt]# ln -s /usr/share/logstash/bin/logstash /usr/local/bin/ #创建一个软链接
[root@apache opt]# systemctl start logstash.service
[root@apache opt]# systemctl enable logstash.service
Created symlink from /etc/systemd/system/multi-user.target.wants/logstash.service to /etc/systemd/system/logstash.service.与elasticsearch（node）做对接测试：
Logstash这个命令测试，字段描述解释：
-f 通过这个选项可以指定logstash的配置文件，根据配置文件配置logstash
-e 后面跟着字符串 该字符串可以被当做logstash的配置（如果是” ”,则默认使用stdin做为输入、stdout作为输出）
-t 测试配置文件是否正确，然后退出
#输入采用标准输入 输出采用标准输出，进行测试
[root@apache opt]# logstash -e 'input { stdin{} } output { stdout{} }'
?使用rubydebug显示详细输出，codec为一种编解码器
[root@apache opt]# logstash -e 'input { stdin{} } output { stdout{ codec=>rubydebug} }'
?使用logstash将信息写入elasticsearch中，输入输出对接
[root@apache opt]# logstash -e 'input { stdin{} } output { elasticsearch { hosts=>["192.168.152.130:9200"]} }'
?
查看索引信息
做对接配置
Logstash配置文件主要由三部分组成：input、output、filter（根据需要来处理）
[root@apache opt]# chmod o+r /var/log/messages #给其他用户读的权限
[root@apache opt]# ll /var/log/messages
-rw----r--. 1 root root 288219 8月 14 10:52 /var/log/messages
[root@apache opt]# vim /etc/logstash/conf.d/system.conf
input {
file{
path => "/var/log/messages" #收集数据的路径
type => "system" #类型
start_position => "beginning" #从开头收集数据
}
}
output {
elasticsearch {
hosts => ["192.168.152.130:9200"] #输出到
index => "system-%{+YYYY.MM.dd}" #索引
}
}
[root@apache opt]# systemctl restart logstash.service 
?node1主机安装kibana
配置过程
#安装kibana：
[root@node1 ~]# cd /usr/local/src/
[root@node1 src]# ls
elasticsearch-head kibana-5.5.1-x86_64.rpm phantomjs-2.1.1-linux-x86_64
[root@node1 src]# rpm -ivh kibana-5.5.1-x86_64.rpm
#修改配置文件：
[root@node1 src]# cd /etc/kibana/
[root@node1 kibana]# cp kibana.yml kibana.yml.bak
[root@node1 kibana]# vim kibana.yml
2# server.port: 5601 #kibana打开的端口
7# server.host: "0.0.0.0" #kibana侦听的地址
21# elasticsearch.url: "http: //192.168.152.130:9200" #利和elasticsearch建立联系
30# kibana .index : ".kibana" #在elasticsearch中添加.kibana索引
[root@node1 kibana]# grep -v "^#" kibana.yml
server.port: 5601
server.host: "0.0.0.0"
elasticsearch.url: "http://192.168.152.130:9200"
kibana.index: ".kibana"
#启动服务
[root@node1 kibana]# systemctl start kibana.service
[root@node1 kibana]# systemctl enable kibana.service 宿主机登陆测试，查看kibana
?
对接apache的日志?
配置过程?
[root@apache opt]# cd /etc/logstash/conf.d/
[root@apache conf.d]# ls
system.conf
[root@apache conf.d]# vim apache_log.conf
input {
file{
path => "/etc/httpd/logs/access_log"
type => "access"
start_position => "beginning"
}
file{
path => "/etc/httpd/logs/error_log"
type => "error"
start_position => "beginning"
}
}
output {
if [type] == "access" {
elasticsearch {
hosts => ["192.168.152.130:9200"]
index => "apache_access-%{+YYYY.MM.dd}"
}
}
if [type] == "error" {
elasticsearch {
hosts => ["192.168.152.130:9200"]
index => "apache_error-%{+YYYY.MM.dd}"
}
}
}
[root@apache conf.d]# /usr/share/logstash/bin/logstash -f apache_log.conf
# 指定配置文件做实验查看索引
?
创建索引名称?
?
?
?
?
?