日志收集三：rsyslog + elasticsearch + kibana

meikkiie

rsyslog v8
　　配置文件语法改变较大，添加了很多其他输入输出模块，可以更好的进行扩展。
在v5的基础上，添加elasticsearch的发送。
v8配置参考：请添加链接描述

v8采集端完整配置

module(load="imuxsock") # provides support for local system logging (e.g. via logger command)
module(load="imklog")   # provides kernel logging support (previously done by rklogd)
module(load="omelasticsearch")
module(load="imfile" mode="inotify" readtimeout="5" )
#input(type="imfile" file="/var/log/zabbix_*.log" tag="C+eebo.test.test+CD" ruleset="fileinput" facility="local6" PersistStateInterval="100" startmsg.regex="^[[:digit:]]{5}:[[:digit:]]{8}:[[:digit:]]{6}.[[:digit:]]{3}" maxsubmitatonce="1024000" readMode="2")
module(load="imudp" threads="2") # needs to be done just once
input(type="imudp" port="514" address="127.0.0.1" rcvbufSize="1m" Ruleset="udpinput")
### server model need to load and set
#module(load="imtcp" keepalive="on" keepalive.time="30" KeepAlive.Probes="1") # needs to be done just once
#input(type="imtcp" port="514" Ruleset="tcpinput")
#### GLOBAL DIRECTIVES ####
# Use default timestamp format
$ActionFileDefaultTemplate RSYSLOG_TraditionalFileFormat
# Include all config files in /etc/rsyslog.d/
$IncludeConfig /etc/rsyslog.d/*.conf
# Spool files
#$WorkDirectory /var/spool/rsyslog
$WorkDirectory /var/lib/rsyslog
# Filter duplicate messages
$RepeatedMsgReduction on
$EscapeControlCharactersOnReceive off
#### RULES ####
# Log all kernel messages to the console.
# Logging much else clutters up the screen.
#kern.*                                                 /dev/console
# Log anything (except mail) of level info or higher.
# Don't log private authentication messages!
#*.info;mail.none;authpriv.none;cron.none;local0.none;local7.none                /var/log/messages
auth.info;authpriv.info;daemon.info;kern.info;syslog.info;user.info;lpr.info;mark.info                /var/log/messages
# The authpriv file has restricted access.
authpriv.*                                              /var/log/secure
# Log all the mail messages in one place.
mail.*                                                  /var/log/maillog
# Log cron stuff
cron.*                                                  /var/log/cron
# Everybody gets emergency messages
*.emerg                                                 :omusrmsg:*
# Save news errors of level crit and higher in a special file.
uucp,news.crit                                          /var/log/spooler
# Save boot messages also to boot.log
local7.*                                                /var/log/boot.log
template(name="msg" type="string" string="%rawmsg%\n")
template(name="msg" type="string" string="%rawmsg%\n")
template(name="timemsg" type="string" string="%timegenerated:8:15% %msg%\n")
template(name="udppfile" type="string" string="/data/rsyslogs/%HOSTNAME%/%syslogtag%/%$year%-%$month%-%$day%.log")
template(name="tcppfile" type="string" string="/data/rsyslogs/%HOSTNAME%/%syslogtag%/%$year%-%$month%-%$day%_%fromhost-ip%.log")
template(name="tcpnfile" type="string" string="/data/rsyslogs/nginx/%syslogtag:F,66:1%/%$year%-%$month%-%$day%_%fromhost-ip%_%syslogtag:F,66:2%.log")
### get local code log from syslog to elasearchtich
template(name="udppname" type="string" string="%HOSTNAME%-%$year%.%$month%.%$day%")
template(name="udpelastic" type="list" option.json="on"){
constant(value="{")
constant(value="\"PROGRAM_NAME\":\"") property(name="hostname")
constant(value="\",\"ENV\":\"") property(name="syslogtag")
constant(value="\",\"LOG_TYPE\":\"code_syslog\",")
constant(value="\"HOSTNAME\":\"") property(name="$!myhostname") #get host ip
constant(value="\",\"TIME\":\"") property(name="timereported" dateFormat="rfc3339")
constant(value="\",\"MSG\":\"") property(name="msg")
constant(value="\"}")
}
### get local nginx log from syslog to elasticsearch
#ACCESS:access_log syslog:server=127.0.0.1:514,facility=local1,tag=m_test_2haohr_com,severity=info main;
#ERROR:error_log syslog:server=127.0.0.1:514,facility=local1,tag=m_test_2haohr_com,severity=debug;
template(name="websitename" type="string" string="%syslogtag:F,66:1%-%$year%.%$month%.%$day%")
template(name="nginxelastic" type="list" option.json="on"){
constant(value="{")
constant(value="\"PROGRAM_NAME\":\"") property(name="syslogtag")
constant(value="\",\"ENV\":\"") property(name="$!env")
constant(value="\",\"LOG_TYPE\":\"") property(name="$!logtype")
constant(value="\",\"HOSTNAME\":\"") property(name="hostname") #get host ip
constant(value="\",\"TIME\":\"") property(name="timereported" dateFormat="rfc3339")
constant(value="\",\"MSG\":\"") property(name="msg")
constant(value="\"}")
}
ruleset(name="udpinput") {
if prifilt("local0.*") then {
action(type="omfile" dynaFile="udppfile" template="timemsg")
action(type="omfwd" protocol="tcp" target="10.10.10.10" port="514" zipLevel="5" keepalive="on")
set $!myhostname = "10-10-3-215";
action(type="omelasticsearch" server="10.10.166.166" serverport="9200" template="udpelastic" dynSearchIndex="on" searchIndex="udppname" searchType="events")
}else if $syslogfacility-text == "local1" then {
set $!sitename = $syslogtag;
if $syslogtag contains "_dev" then {
set $!env = "dev";
} else if $syslogtag contains "_test" then {
set $!env = "test";
} else {
set $!env = "production";
}
if $syslogseverity-text == "info" then {
set $!logtype = "nginx_access";
} else if $syslogseverity-text == "debug" then {
set $!logtype = "nginx_error";
}
action(type="omfwd" protocol="tcp" target="10.10.10.10" port="514" zipLevel="5" keepalive="on")
action(type="omelasticsearch" server="10.10.166.166" serverport="9200" template="nginxelastic" dynSearchIndex="on" searchIndex="websitename" bulkmode="on" Action.ResumeRetryCount="-1")
} else{
action(type="omfile" file="var/log/test.test.log")
stop
}
}
template(name="filepname" type="string" string="%syslogtag:F,43:2%-%$year%.%$month%.%$day%")
template(name="fileselasticsearch" type="list" option.json="on"){
constant(value="{")
constant(value="\"PROGRAM_NAME\":\"") property(name="$!programname")
constant(value="\",\"ENV\":\"") property(name="$!env")
constant(value="\",\"LOG_TYPE\":\"") property(name="$!logtype")
constant(value="\",\"HOSTNAME\":\"") property(name="hostname") #get host ip
constant(value="\",\"TIME\":\"") property(name="timereported" dateFormat="rfc3339")
constant(value="\",\"MSG\":\"") property(name="msg")
constant(value="\"}")
}
ruleset(name="fileinput"){
if prifilt("local6.*") then {
set $!programname = field($syslogtag,"+",2);
if re_match($syslogtag,"G+.*+.*E") then {
set $!logtype = "gunicorn_error";
} else if re_match($syslogtag,"G+.*+.*A") then {
set $!logtype = "gunicorn_access";
} else if re_match($syslogtag,"S+.*+.*G") then {
set $!logtype = "gunicorn_supervisor";
} else if re_match($syslogtag,"C+.*+.*C") then {
set $!logtype = "celery_log";
} else if re_match($syslogtag,"S+.*+.*C") then {
set $!logtype = "celery_supervisor";
}
if $syslogtag contains "D" then {
set $!env = "dev";
} else if $syslogtag contains "T" then {
set $!env = "test";
} else if $syslogtag contains "P" then {
set $!env = "production";
}
action(type="omfwd" protocol="tcp" target="10.10.10.10" port="514" zipLevel="5" keepalive="on")
action(type="omelasticsearch" server="10.10.166.166" serverport="9200" template="fileselasticsearch" dynSearchIndex="on" searchIndex="filepname" bulkmode="on" Action.ResumeRetryCount="-1")
}
stop
}
使用 ansible-playbook 替换所有机器

---
- hosts: GZtest
remote_user: root
vars:
rsyslog_port: 514
protocol: udp
tasks:
- name: add rsyslogd repository
yum_repository:
name: rsyslog_v8
description: Adiscon CentOS-$releasever - local packages for $basearch
baseurl: http://rpms.adiscon.com/v8-stable/epel-$releasever/$basearch
enabled: yes
gpgcheck: no  
gpgkey: http://rpms.adiscon.com/RPM-GPG-KEY-Adiscon
file: rsyslog
protect: yes
- name: install rsyslog-main
yum:
name: rsyslog
state: latest
- name: install rsyslog-elastic module
yum:
name: rsyslog-elasticsearch
state: latest
- name: copy rsyslog.conf
copy:
backup: yes
src: /etc/rsyslog.d/rsyslog_v8_client.conf
dest: /etc/rsyslog.conf
#    shell: sed -i "s/222/{{ansible_hostname}}/g" /root/shell/test.txt
- name: change localhost
replace:
dest: /etc/rsyslog.conf
regexp: '10-10-3-215'
replace: '{{ansible_hostname}}'
notify:
- restart rsyslogd
handlers:
- name: restart rsyslogd
service:
name: rsyslog
enabled: yes
state: restarted
其他：

elasticsearsh安装配置
　　单机es存储日志量有限，可以考虑保存最近一个礼拜的，其他的删除，有需要去文件中查看。
考虑添加队列，提升es实时性能

kibana安装配置
　　权限不好控制，暂时无法通过插件限制开发只使用discovery模块。
安全很重要，至少要通过Basic HTTP authentication验证。

效果
　　elasticsearch中保存的index

kibana查看日志：