将apache日志输出为json格式并发送给logstash处理

234rew

1、Apache日志格式定义在apache配置文件中增加：

LogFormat "{ \
            \"@timestamp\": \"%{%Y-%m-%dT%H:%M:%S%z}t\", \
            \"@version\": \"1\", \
            \"tags\":[\"apache\"], \
            \"message\": \"%h %l %u %t \\\"%r\\\" %>s %b\", \
            \"clientip\": \"%a\", \
            \"duration\": %D, \
            \"status\": %>s, \
            \"request\": \"%U%q\", \
            \"urlpath\": \"%U\", \
            \"urlquery\": \"%q\", \
            \"bytes\": %B, \
            \"method\": \"%m\", \
            \"site\": \"%{Host}i\", \
            \"referer\": \"%{Referer}i\", \
            \"useragent\": \"%{User-agent}i\" \
           }" ls_apache_json
CustomLog logs/access_log.ls_json ls_apache_json

2、logforwarder配置文件增加以下文件定义内容
    {
      "paths": [ "/var/log/httpd/access_log.ls_json" ],
      "fields": { "type": "apache_json" }
    }

3、服务端logstash filter配置filter {
  if [type] == "apache_json" {
    json {
      source => "message"
    }

    if [useragent] != "-" and [useragent] != "" {
      useragent {
        add_tag => [ "UA" ]
        source => "useragent"
        prefix => "UA-"
      }
    }

    mutate {
      convert => ['duration', 'float']
    }

    ruby {
      code => "event['duration']/=1000000"
    }

    if [bytes]     == 0                    { mutate { remove_field => "[bytes]" } }
    if [urlquery]  == ""                 { mutate { remove_field => "urlquery" } }
    if [method]    =~ "(HEAD|OPTIONS)"         { mutate { remove_field => "method" } }
    if [useragent] == "-"                 { mutate { remove_field => "useragent" } }
    if [referer]   == "-"                 { mutate { remove_field => "referer" } }

    if "UA" in [tags] {
      if [device] == "Other" { mutate { remove_field => "device" } }
      if [name]   == "Other" { mutate { remove_field => "name" } }
      if [os]     == "Other" { mutate { remove_field => "os" } }
    }

  }
}

4、检查输入结果{
       "message" => "192.168.0.90 - - [27/Nov/2015:12:07:26 +0800] \"POST /zabbix/jsrpc.php?output=json-rpc HTTP/1.1\" 200 64",
      "@version" => "1",
    "@timestamp" => "2015-11-27T04:07:26.000Z",
          "file" => "/var/log/httpd/access_log.ls_json",
          "host" => "zabbix",
        "offset" => "1154812",
          "type" => "apache_json",
          "tags" => [
        [0] "apache",
        [1] "UA"
    ],
      "clientip" => "192.168.0.90",
      "duration" => 0.126574,
        "status" => 200,
       "request" => "/zabbix/jsrpc.php?output=json-rpc",
       "urlpath" => "/zabbix/jsrpc.php",
      "urlquery" => "?output=json-rpc",
         "bytes" => 64,
        "method" => "POST",
          "site" => "10.20.20.65",
       "referer" => "http://10.20.20.65/zabbix/dashboard.php?ddreset=1&sid=e5260b4dda5e072e",
     "useragent" => "Mozilla/5.0 (Windows NT 10.0; WOW64; rv:42.0) Gecko/20100101 Firefox/42.0",
       "UA-name" => "Firefox",
         "UA-os" => "Windows 10",
    "UA-os_name" => "Windows 10",
     "UA-device" => "Other",
      "UA-major" => "42",
      "UA-minor" => "0"
}

参考：
https://deviantony.wordpress.com ... -apache-access-log/