David
# 编辑first-pipleline.conf文件 # logstash使用http协议连接elasticsearchoutput {
elasticsearch {
hosts => [ "localhost:9200" ]
}
}
# 最后first-pipeline.conf文件
input {
beats {
port => "5043"
}
}
filter {
grok {
match => { "message" => "%{COMBINEDAPACHELOG}"}
}
geoip {
source => "clientip"
}
}
output {
elasticsearch {
hosts => [ "localhost:9200" ]
}
}
# 删除filebeat 注册文件
sudo rm /usr/share/filebeat/bin/data/registry
# 重启服务
/usr/share/filebeat/bin/filebeat -e -c /etc/filebeat/filebeat.yml -d "publish"
#查询数据
# 将$DATA换成特定的日期YYYY.MM.DD格式
curl -XGET 'localhost:9200/logstash-$DATE/_search?pretty&q=response=200'
{
"took" : 21,
"timed_out" : false,
"_shards" : {
"total" : 5,
"successful" : 5,
"failed" : 0
},
"hits" : {
"total" : 98,
"max_score" : 3.745223,
"hits" : [
{
"_index" : "logstash-2016.10.11",
"_type" : "log",
"_id" : "AVe14gMiYMkU36o_eVsA",
"_score" : 3.745223,
"_source" : {
"request" : "/presentations/logstash-monitorama-2013/images/frontend-response-codes.png",
"agent" : "\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36\"",
"geoip" : {
"timezone" : "Europe/Moscow",
"ip" : "83.149.9.216",
"latitude" : 55.7522,
"continent_code" : "EU",
"city_name" : "Moscow",
"country_code2" : "RU",
"country_name" : "Russia",
"dma_code" : null,
"country_code3" : "RU",
"region_name" : "Moscow",
"location" : [
37.6156,
55.7522
],
"postal_code" : "101194",
"longitude" : 37.6156,
"region_code" : "MOW"
},
"offset" : 2932,
"auth" : "-",
"ident" : "-",
"input_type" : "log",
"verb" : "GET",
"source" : "/path/to/file/logstash-tutorial.log",
"message" : "83.149.9.216 - - \"GET /presentations/logstash-monitorama-2013/images/frontend-response-codes.png HTTP/1.1\" 200 52878 \"http://semicomplete.com/presentations/logstash-monitorama-2013/\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36\"",
"type" : "log",
"tags" : [
"beats_input_codec_plain_applied"
],
"referrer" : "\"http://semicomplete.com/presentations/logstash-monitorama-2013/\"",
"@timestamp" : "2016-10-11T22:34:25.317Z",
"response" : "200",
"bytes" : "52878",
"clientip" : "83.149.9.216",
"@version" : "1",
"beat" : {
"hostname" : "My-MacBook-Pro.local",
"name" : "My-MacBook-Pro.local"
},
"host" : "My-MacBook-Pro.local",
"httpversion" : "1.1",
"timestamp" : "04/Jan/2015:05:13:45 +0000"
}
}
},
...
# 另一种查询方式
# 查询地址为Buffalo
# 将$DATA换成特定的日期YYYY.MM.DD格式
curl -XGET 'localhost:9200/logstash-2017.04.10/_search?pretty&q=geoip.city_name=Buffalo'
{
"took" : 3,
"timed_out" : false,
"_shards" : {
"total" : 5,
"successful" : 5,
"failed" : 0
},
"hits" : {
"total" : 3,
"max_score" : 2.6390574,
"hits" : [
{
"_index" : "logstash-2016.10.11",
"_type" : "log",
"_id" : "AVe14gMjYMkU36o_eVtO",
"_score" : 2.6390574,
"_source" : {
"request" : "/?flav=rss20",
"agent" : "\"-\"",
"geoip" : {
"timezone" : "America/New_York",
"ip" : "108.174.55.234",
"latitude" : 42.9864,
"continent_code" : "NA",
"city_name" : "Buffalo",
"country_code2" : "US",
"country_name" : "United States",
"dma_code" : 514,
"country_code3" : "US",
"region_name" : "New York",
"location" : [
-78.7279,
42.9864
],
"postal_code" : "14221",
"longitude" : -78.7279,
"region_code" : "NY"
},
"offset" : 21471,
"auth" : "-",
"ident" : "-",
"input_type" : "log",
"verb" : "GET",
"source" : "/path/to/file/logstash-tutorial.log",
"message" : "108.174.55.234 - - \"GET /?flav=rss20 HTTP/1.1\" 200 29941 \"-\" \"-\"",
"type" : "log",
"tags" : [
"beats_input_codec_plain_applied"
],
"referrer" : "\"-\"",
"@timestamp" : "2016-10-11T22:34:25.318Z",
"response" : "200",
"bytes" : "29941",
"clientip" : "108.174.55.234",
"@version" : "1",
"beat" : {
"hostname" : "My-MacBook-Pro.local",
"name" : "My-MacBook-Pro.local"
},
"host" : "My-MacBook-Pro.local",
"httpversion" : "1.1",
"timestamp" : "04/Jan/2015:05:27:45 +0000"
}
},
...
页:
[1]