Elasticsearch + Logstash + Kibana（ELK）安装部署方法

maxc1017

Elasticsearch + Logstash + Kibana（ELK）是一套开源的日志管理方案，分析网站的访问情况时我们一般会借助Google/百度/CNZZ等方式嵌入JS做数据统计，但是当网站访问异常或者被***时我们需要在后台分析如Nginx的具体日志，而Nginx日志分割/GoAccess/Awstats都是相对简单的单节点解决方案，针对分布式集群或者数据量级较大时会显得心有余而力不足，而ELK的出现可以使我们从容面对新的挑战。

Logstash：负责日志的收集，处理和储存
Elasticsearch：负责日志检索和分析
Kibana：负责日志的可视化


学习网站 elk
https://www.elastic.co/
http://kibana.logstash.es/content/logstash/examples/nginx-access.html
https://www.gitbook.com/book/fuxiaopang/learnelasticsearch/details


国内的klb
http://blog.csdn.net/ebw123/article/details/46707559
http://www.tuicool.com/articles/m6RNBrY
http://baidu.blog.运维网.com/71938/1676798
http://dl528888.blog.运维网.com/2382721/1703059
http://dl528888.blog.运维网.com/2382721/1703059
http://caochun.blog.运维网.com/4497308/1715462


http://www.linuxdiyf.com/linux/15787.html
http://www.linuxidc.com/Linux/2015-12/126587.htm
http://blog.csdn.net/super_scan/article/details/45694289


http://www.cnblogs.com/xiaouisme/p/3977721.html
http://blog.sina.com.cn/s/blog_7b837d030101ckia.html
http://www.tuicool.com/articles/bUvmUrb
http://www.360doc.com/content/15/0512/09/1073512_469853970.shtml


博客写的好
http://www.360doc.com/userhome/1073512
http://www.wklken.me/


国外的网站
https://github.com/instruct-br/webinar-elk
https://github.com/instruct-br/webinar-elk/tree/master/puppet/environments/production/modules


1. 主机规划

(1)要求两台主机的时间同步
(2)ssl 设置

2.修改两台主机的hostname

3.服务器端安装elk

（2）安装Elasticsearch


#下载安装
wget https://download.elastic.co/elasticsearch/elasticsearch/elasticsearch-1.7.1.noarch.rpm
yum localinstall elasticsearch-1.7.1.noarch.rpm

#启动相关服务
service elasticsearch start
service elasticsearch status

#查看Elasticsearch的配置文件
rpm -qc elasticsearch

/etc/elasticsearch/elasticsearch.yml
/etc/elasticsearch/logging.yml
/etc/init.d/elasticsearch
/etc/sysconfig/elasticsearch
/usr/lib/sysctl.d/elasticsearch.conf
/usr/lib/systemd/system/elasticsearch.service
/usr/lib/tmpfiles.d/elasticsearch.conf
#查看端口使用情况
netstat -nltp

#测试访问
curl -X GET http://localhost:9200/

（3）安装Kibana
#下载tar包
wget  https://download.elastic.co/kibana/kibana/kibana-4.1.1-linux-x64.tar.gz
#解压
tar  zxf  kibana-4.1.1-linux-x64.tar.gz -C /usr/local/
cd /usr/local/
mv  kibana-4.1.1-linux-x64 kibana


#创建kibana服务
vi /etc/rc.d/init.d/kibana
#!/bin/bash
### BEGIN INIT INFO
# Provides:          kibana
# Default-Start:     2 3 4 5
# Default-Stop:      0 1 6
# Short-Description: Runs kibana daemon
# Description: Runs the kibana daemon as a non-root user
### END INIT INFO
# Process name
NAME=kibana
DESC="Kibana4"
PROG="/etc/init.d/kibana"
# Configure location of Kibana bin
KIBANA_BIN=/usr/local/kibana/bin
# PID Info
PID_FOLDER=/var/run/kibana/
PID_FILE=/var/run/kibana/$NAME.pid
LOCK_FILE=/var/lock/subsys/$NAME
PATH=/bin:/usr/bin:/sbin:/usr/sbin:$KIBANA_BIN
DAEMON=$KIBANA_BIN/$NAME
# Configure User to run daemon process
DAEMON_USER=root
# Configure logging location
KIBANA_LOG=/var/log/kibana.log
# Begin Script
RETVAL=0
if [ `id -u` -ne 0 ]; then
        echo "You need root privileges to run this script"
        exit 1
fi
# Function library
. /etc/init.d/functions

start() {
        echo -n "Starting $DESC : "
pid=`pidofproc -p $PID_FILE kibana`
        if [ -n "$pid" ] ; then
                echo "Already running."
                exit 0
        else
        # Start Daemon
if [ ! -d "$PID_FOLDER" ] ; then
                        mkdir $PID_FOLDER
                fi
daemon --user=$DAEMON_USER --pidfile=$PID_FILE $DAEMON 1>"$KIBANA_LOG" 2>&1 &
                sleep 2
                pidofproc node > $PID_FILE
                RETVAL=$?
                [[ $? -eq 0 ]] && success || failure
echo
                [ $RETVAL = 0 ] && touch $LOCK_FILE
                return $RETVAL
        fi
}
reload()
{
    echo "Reload command is not implemented for this service."
    return $RETVAL
}
stop() {
        echo -n "Stopping $DESC : "
        killproc -p $PID_FILE $DAEMON
        RETVAL=$?
echo
        [ $RETVAL = 0 ] && rm -f $PID_FILE $LOCK_FILE
}

case "$1" in
  start)
        start
;;
  stop)
        stop
        ;;
  status)
        status -p $PID_FILE $DAEMON
        RETVAL=$?
        ;;
  restart)
        stop
        start
        ;;
  reload)
reload
;;
  *)
# Invalid Arguments, print the following message.
        echo "Usage: $0 {start|stop|status|restart}" >&2
exit 2
        ;;
esac#修改启动权限
chmod +x /etc/rc.d/init.d/kibana

#启动kibana服务
service kibana start
service kibana status


#查看端口
netstat -nltp





（4）设置ssl，之前设置的FQDN是elk.zzxtbl.com

openssl req -subj '/CN=elk.zzxtbl.com/' -x509 -days 3650 -batch -nodes -newkey rsa:2048 -keyout private/logstash-forwarder.key -out certs/logstash-forwarder.cr

（5） Logstash
#下载rpm包
wget  https://download.elastic.co/logstash/logstash/packages/centos/logstash-1.5.4-1.noarch.rpm
#安装
yum  localinstall logstash-1.5.4-1.noarch.rpm
#创建一个01-logstash-initial.conf文件
#创建一个01-logstash-initial.conf文件
cat > /etc/logstash/conf.d/01-logstash-initial.conf  5000
    type => "logs"
    ssl_certificate => "/etc/pki/tls/certs/logstash-forwarder.crt"
    ssl_key => "/etc/pki/tls/private/logstash-forwarder.key"
  }
}
filter {
  if [type] == "syslog" {
    grok {
      match => { "message" => "%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:syslog_message}" }
      add_field => [ "received_at", "%{@timestamp}" ]
      add_field => [ "received_from", "%{host}" ]
    }
    syslog_pri { }
    date {
      match => [ "syslog_timestamp", "MMM  d HH:mm:ss", "MMM dd HH:mm:ss" ]
    }
  }
}
output {
  elasticsearch { host => localhost }
  stdout { codec => rubydebug }
}
EOF#启动logstash服务
service logstash start
service logstash status

#查看5000端口
netstat -nltp

#增加节点和客户端配置一样，注意同步证书
scp /etc/pki/tls/certs/logstash-forwarder.crt  node:/opt
4.客户端安装Logstash Forwarder
#登陆到客户端，安装Logstash Forwarder
wget https://download.elastic.co/logstash-forwarder/binaries/logstash-forwarder-0.4.0-1.x86_64.rpm
yum localinstall logstash-forwarder-0.4.0-1.x86_64.rpm
#查看logstash-forwarder的配置文件位置
rpm -qc logstash-forwarder
/etc/logstash-forwarder.conf
#备份配置文件
cp /etc/logstash-forwarder.conf /etc/logstash-forwarder.conf.save
#编辑 /etc/logstash-forwarder.conf，需要根据实际情况进行修改
cat > /etc/logstash-forwarder.conf  5000
    type => "logs"
    ssl_certificate => "/etc/pki/tls/certs/logstash-forwarder.crt"
    ssl_key => "/etc/pki/tls/private/logstash-forwarder.key"
  }
}
filter {
  if [type] == "syslog" {
    grok {
      match => { "message" => "%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:syslog_message}" }
      add_field => [ "received_at", "%{@timestamp}" ]
      add_field => [ "received_from", "%{host}" ]
    }
    syslog_pri { }
    date {
      match => [ "syslog_timestamp", "MMM  d HH:mm:ss", "MMM dd HH:mm:ss" ]
    }
  }
  if [type] == "nginx" {
    grok {
       match => { "message" => "%{NGINXACCESS}" }
    }
  }
}
output {
  elasticsearch { host => localhost }
  stdout { codec => rubydebug }
}

重启服务端elk服务

显示页面

以上就是简单的elk分享！！