elasticsearch 日志抓取使用

zhuce

　　[root@server5 elk]# rpm -ivh  logstash-2.3.3-1.noarch.rpm
[root@server5 logstash]# /opt/logstash/bin/logstash -e 'input {stdin { } } output { stdout { codec => rubydebug } }'
Settings: Default pipeline workers: 1
Pipeline main started
hello #随便写
{
"message" => "hello",
"@version" => "1",
"@timestamp" => "2018-12-08T03:58:53.761Z",
"host" => "server5"
}
redhat
{
"message" => "redhat",
"@version" => "1",
"@timestamp" => "2018-12-08T03:59:05.366Z",
"host" => "server5"
}
^CSIGINT received. Shutting down the agent. {:level=>:warn}
stopping pipeline {:id=>"main"}
　　Pipeline main has been shutdown
[root@server5 logstash]# /opt/logstash/bin/logstash -e 'input {stdin { } } output { stdout { codec => rubydebug } elasticsearch { hosts => ["172.25.135.5"] index => "logstash-%{+YYYY.MM.dd}"} }'
[root@server5 logstash]# cd /etc/logstash/conf.d/
[root@server5 conf.d]# ls
[root@server5 conf.d]# vim es.conf

[root@server5 conf.d]# /opt/logstash/bin/logstash -f /etc/logstash/conf.d/es.conf

[root@server5 conf.d]# cat  /tmp/testfile
custom format: asdjkl
[root@server5 conf.d]# chmod 644 /var/log/messages
[root@server5 conf.d]# logger  "hello" #没有日志文件写一些，有可略过
[root@server5 conf.d]# logger  "hello"
[root@server5 conf.d]# logger  "hello"
[root@server5 conf.d]# logger  "hello"

　　[root@server5 conf.d]# vim es.conf
input {
file {
path => "/var/log/messages"
start_position => "beginning"
}
}
　　output {

stdout {

codec => rubydebug

}

    elasticsearch {
hosts => ["172.25.135.5"]
index => "message-%{+YYYY.MM.dd}"
}
file {

path => "/tmp/testfile"

codec => line { format => "custom format: %{message}"}

}
　　}
[root@server5 conf.d]# /opt/logstash/bin/logstash -f /etc/logstash/conf.d/es.conf
[root@server5 conf.d]# cd
[root@server5 ~]# l.
.              .bashrc            .pki
..             .cache             .sincedb_452905a167cf4509fd08acb964fdb20c
.bash_history  .cshrc             .ssh
.bash_logout   .mysql_history     .tcshrc
.bash_profile  .oracle_jre_usage  .viminfo
[root@server5 ~]# cat .sincedb_452905a167cf4509fd08acb964fdb20c
267849 0 64768 517
[root@server5 ~]# ls -i /var/log/messages
267849 /var/log/messages   #两者id一样


[root@server5 conf.d]# vim /etc/rsyslog.conf
.* @@172.25.135.5:514   #末尾添加，server7，8同样添加，同样重启以下服务
[root@server5 conf.d]# /etc/init.d/elasticsearch  restart
[root@server5 conf.d]# /etc/init.d/rsyslog restart   
[root@server5 conf.d]# vim es.conf
input {

file {

path => "/var/log/messages"

start_position => "beginning"

}

   syslog {
port  => 514
}
　　}
　　output {
stdout {
codec => rubydebug
}
elasticsearch {
hosts => ["172.25.135.5"]
index => "message-%{+YYYY.MM.dd}"
}

file {

path => "/tmp/testfile"

codec => line { format => "custom format: %{message}"}

}
　　}
　　[root@server5 conf.d]# /opt/logstash/bin/logstash -f /etc/logstash/conf.d/es.conf
#为了方便看：
可以给server7，8加点日志
[root@server7 ~]# logger  server2
[root@server7 ~]# logger  server2
[root@server7 ~]# logger  server2
[root@server8 vm]# logger  server3
[root@server8 vm]# logger  server3
[root@server8 vm]# logger  server3
刷新可以看出：

[root@server5 conf.d]# cd
[root@server5 ~]# l.
.              .bash_profile  .mysql_history                             .ssh
..             .bashrc        .oracle_jre_usage                          .tcshrc
.bash_history  .cache         .pki                                       .viminfo
.bash_logout   .cshrc         .sincedb_452905a167cf4509fd08acb964fdb20c
[root@server5 ~]# rm -fr  .sincedb_452905a167cf4509fd08acb964fdb20c
[root@server5 ~]# cd -
/etc/logstash/conf.d
[root@server5 conf.d]# ls
es.conf
[root@server5 conf.d]# vim es.conf
input {
file {
path => "/var/log/elasticsearch/my-es.log"
start_position => "beginning"
}

syslog {

port  => 514

}
　　}
#filter {

multiline {

type => "eslog"

pattern => "^["

negate => true

what => "previous"

}
　　#}
output {

stdout {

codec => rubydebug

}

    elasticsearch {
hosts => ["172.25.135.5"]
index => "es-%{+YYYY.MM.dd}"
}
file {

path => "/tmp/testfile"

codec => line { format => "custom format: %{message}"}

}
　　}
　　[root@server5 conf.d]# /opt/logstash/bin/logstash -f /etc/logstash/conf.d/es.conf    #新建es  log
刷新浏览器可以看出：