55 logstach应用详解、ELK Stack
01logstash应用详解配置环境：
node3192.168.1.133CentOS Linux release 7.2
node4192.168.1.134CentOS Linux release 7.2
# cd /etc/logstash/conf.d/
# vim filesample.conf
input {
file {
path =>["/var/log/messages"]
type =>"system"
start_position=>"beginning"
}
}
output {
stdout {
codec =>rubydebug
}
}
# logstash -f filesample.conf --configtest
Configuration OK
# rpm -ivh epel-release-latest-7.noarch.rpm
# yum -y install collectd
# vim /etc/collectd.conf
修改
#Hostname "localhost"
为
Hostname "node3"
修改
#LoadPlugin df
为
LoadPlugin df#监控磁盘
修改
#LoadPlugin network
为
LoadPlugin network
在程序端后添加
# systemctl start collectd.service
# vim udpsample.conf
input {
udp {
port =>25826
codec =>collectd {}
type =>"collectd"
}
}
output {
stdout {
codec =>rubydebug
}
}
# logstash -f udpsample.conf --configtest
Configuration OK
# logstash -f udpsample.conf
# yum -y install httpd
# systemctl start httpd.service
# vim groksample.conf
input {
stdin {}
}
filter {
grok {
match =>{ "message" =>"%{IP:clientip} %{WORD:method} %{URIPATHPARAM:request} %{NUMBER:bytes} %{NUMBER:duration}" }
}
}
output {
stdout {
codec =>rubydebug
}
}
# logstash -f groksample.conf --configtest
Configuration OK
# logstash -f groksample.conf
Logstash startup completed
1.1.1.1 GET /index.html 30 0.23
{
"message" => "1.1.1.1 GET /index.html 30 0.23",
"@version" => "1",
"@timestamp" => "2017-01-03T13:37:24.978Z",
"host" => "node4",
"clientip" => "1.1.1.1",
"method" => "GET",
"request" => "/index.html",
"bytes" => "30",
"duration" => "0.23"
}
# vim apachelogsample.conf
input {
file {
path =>["/var/log/httpd/access_log"]
type =>"apachelog"
start_position=>"beginning"
}
}
filter {
grok {
match =>{ "message" =>"%{COMBINEDAPACHELOG}" }
}
}
output {
stdout {
codec =>rubydebug
}
}
# logstash -f apachelogsample.conf --configtest
Configuration OK
# vim /opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-patterns-core-0.3.0/patterns/grok-patterns
在末尾添加
# nginx Logs
NGUSERNAME +
NGUSER %{NGUSERNAME}
NGINXACCESS %{IPORHOST:clientip} - %{NOTSPACE:remote_user} \[%{HTTPDATE:timestamp}\] "(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})\" %{NUMBER:response} (?:%{NUMBER:bytes}|-) %{QS:referrer} %{OS:agent} %{NOTSPACE:http_x_forwarded_for}
# systemctl stop httpd.service
# yum -y install nginx
# systemctl start nginx.service
# cd /var/log/nginx/
# ls
access.logerror.log
# tail access.log
192.168.1.204 - - "GET / HTTP/1.1" 200 3700 "-" "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.3; WOW64; Trident/7.0; .NET4.0E; .NET4.0C)" "-"
192.168.1.204 - - "GET /nginx-logo.png HTTP/1.1" 200 368 "http://192.168.1.134/" "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.3; WOW64; Trident/7.0; .NET4.0E; .NET4.0C)" "-"
192.168.1.204 - - "GET /poweredby.png HTTP/1.1" 200 2811 "http://192.168.1.134/" "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.3; WOW64; Trident/7.0; .NET4.0E; .NET4.0C)" "-"
# cd -
# cp apachelogsample.conf nginxlogsample.conf
# vim nginxlogsample.conf
input {
file {
path =>["/var/log/nginx/access.log"]
type =>"nginxlog"
start_position=>"beginning"
}
}
filter {
grok {
match =>{ "message" =>"%{NGINXACCESS}" }
}
}
output {
stdout {
codec =>rubydebug
}
}
# logstash -f nginxlogsample.conf
02ELK Stack
# yum install redis
# vim /etc/redis.conf
修改
bind 127.0.0.1
为
bind 0.0.0.0
# systemctl start redis.service
# redis-cli
127.0.0.1:6379> help
redis-cli 2.8.19
Type: "help @" to get a list of commands in
"help " for help on
"help " to get a list of possible help topics
"quit" to exit
# cd /etc/logstash/conf.d/
# cp nginxlogsample.conf nglogredissample.conf
# vim nglogredissample.conf
input {
file {
path =>["/var/log/nginx/access.log"]
type =>"nginxlog"
start_position=>"beginning"
}
}
filter {
grok {
match =>{ "message" =>"%{NGINXACCESS}" }
}
}
output {
redis {
port =>6379
host =>["127.0.0.1"]
data_type =>"list"
key =>"logstash-%"
}
}
# logstash -f nglogredissample.conf --configtest
Configuration OK
# vim /etc/profile.d/java.sh
export JAVA_HOME=/usr
# yum install -y logstash-1.5.4-1.noarch.rpm
# cd /etc/logstash/conf.d/
# vim server.conf
input {
redis {
port =>"6370"
host => "192.168.1.134"
data_type =>"list"
key =>"logstash-nginxlog"
}
}
output {
stdout {
codec =>rubydebug
}
}
# vim /etc/profile.d/logstash.sh
export PATH=/opt/logstash/bin:$PATH
# . /etc/profile.d/logstash.sh
# logstash -f server.conf --configtest
Configuration OK
# yum makecache
# yum install java-1.7.0-openjdk-devel.x86_64
# vim /etc/profile.d/java.sh
export JAVA_HOME=/usr
# yum install elasticsearch-1.7.2.noarch.rpm -y
# vim /etc/elasticsearch/elasticsearch.yml
修改
#cluster.name: elasticsearch
为
cluster.name: loges
修改
#node.name: "Franz Kafka"
为
node.name: "node1"
# systemctl daemon-reload
# systemctl start elasticsearch.service
# /usr/share/elasticsearch/bin/plugin -i bigedsk -u file:///root/bigdesk-latest.zip
# tar xf kibana-4.1.2-linux-x64.tar.gz -C /usr/local/
# cd /usr/local/
# ln -s kibana-4.1.2-linux-x64/ kibana
# cd kibana
# ls
binconfigLICENSE.txtnodepluginsREADME.txtsrc
# cd config/
# vim kibana.yml
修改
elasticsearch_url: "http://localhost:9200"
为
elasticsearch_url: "http://192.168.1.131:9200"
#启动
# /usr/local/kibana/bin/kibana
# vim server.conf
input {
redis {
port =>"6370"
host => "192.168.1.134"
data_type =>"list"
key =>"logstash-nginxlog"
}
}
output {
elasticsearch {
cluster =>"loges"
index =>"logstash-%{+YYYY.MM.dd}"
}
}
# logstash -f server.conf --configtest
Configuration OK
# logstash -f server.conf
# curl -XGET 'localhost:9200/_cat/indices'
yellow open .kibana 1 1 1 0 2.4kb 2.4kb
该节视频到71:55(65382)由于错误太多无法继续进行
页:
[1]