修改elasticsearch映射模板——替换默认的logstash映射模板为nginx开头的模板
相信很多探索ELK的朋友和我一样，总是想把nginx访问日志的索引名称修改为自己想要的名称模式，例如：nginx-access-YY.MM.DD，不相信使用默认的必须以logstash-开头的，但是就这一个更改却
可能带来很多的问题，比较常见的是自定义的映射模板导入失败，参数不生效，geoip的定位信息无法
在kibana中调用，笔者也是吃尽了苦头，查阅了很多的技术博客，理解了模板映射的原理后，反复尝试
才成功使用上了自定义的模板映射文件。不知道很多前辈是踩过坑都不说还是之前的版本有新版有区别，
反正没有看到对此问题说的特别清晰的文章，所以笔者吃尽苦头之后，还是把自己的心路历程写出来，
希望对后人有所帮助。
cat /usr/share/logstash/vendor/bundle/jruby/2.3.0/gems/logstash-output-elasticsearch-9.2.0-java/lib/logstash/outputs/elasticsearch/elasticsearch-template-es7x.json
{
"template" : "logstash-*",
"version" : 60001,
"settings" : {
"index.refresh_interval" : "5s",
"number_of_shards": 1
},
"mappings" : {
"_doc" : {
"dynamic_templates" : [ {
"message_field" : {
"path_match" : "message",
"match_mapping_type" : "string",
"mapping" : {
"type" : "text",
"norms" : false
}
}
}, {
"string_fields" : {
"match" : "*",
"match_mapping_type" : "string",
"mapping" : {
"type" : "text", "norms" : false,
"fields" : {
"keyword" : { "type": "keyword", "ignore_above": 256 }
}
}
}
} ],
"properties" : {
"@timestamp": { "type": "date"},
"@version": { "type": "keyword"},
"geoip": {
"dynamic": true,
"properties" : {
"ip": { "type": "ip" },
"location" : { "type" : "geo_point" },
"latitude" : { "type" : "half_float" },
"longitude" : { "type" : "half_float" }
}
}
}
}
}
}
如何直接复制以上内容，修改索引名称后使用kibana提供的ES的控制台导入会报如下错误：
#! Deprecation: Deprecated field used, replaced by
{
"acknowledged": true
}
可以导入成功，那是因为ES会自我修正，把"template" : "nginx-*"修改为
"index_patterns" : ["nginx-*"]
所以正确的自定义义映射模板文件内容应该如下：
{
"index_patterns" : ["nginx-*"],
"version" : 60001,
"settings" : {
"index.refresh_interval" : "5s",
"number_of_shards": 1
},
"mappings" : {
"_doc" : {
"dynamic_templates" : [ {
"message_field" : {
"path_match" : "message",
"match_mapping_type" : "string",
"mapping" : {
"type" : "text",
"norms" : false
}
}
}, {
"string_fields" : {
"match" : "*",
"match_mapping_type" : "string",
"mapping" : {
"type" : "text", "norms" : false,
"fields" : {
"keyword" : { "type": "keyword", "ignore_above": 2048 }
}
}
}
} ],
"properties" : {
"@timestamp": { "type": "date"},
"@version": { "type": "keyword"},
"geoip": {
"dynamic": true,
"properties" : {
"ip": { "type": "ip" },
"location" : { "type" : "geo_point" },
"latitude" : { "type" : "half_float" },
"longitude" : { "type" : "half_float" }
}
}
}
}
}
}
如果你是第一时间就使用这种方法，笔者只能说你非常走运，可能两步就能摆平这个自定义映射模板的问题。
但我相信很多人的遭遇和笔者一样，走了很多的弯路，笔者当时就一直想使用logstash来自己管理这个映射
模板文件，但经过笔者多次尝试，如果直接使用curl 127.0.0.1:9200/_template/logstash?pretty导出
并重定向到一个文件，再修改的话就遇上大坑了。
curl 127.0.0.1:9200/_template/logstash?pretty > nginx.json
vim nginx.json
{
"nginx" : {
"order" : 0,
"version" : 60001,
"index_patterns" : [
"nginx-*"
],
"settings" : {
"index" : {
"refresh_interval" : "5s"
}
},
"mappings" : {
"_default_" : {
"dynamic_templates" : [
{
"message_field" : {
"path_match" : "message",
"match_mapping_type" : "string",
"mapping" : {
"type" : "text",
"norms" : false
}
}
},
{
"string_fields" : {
"match" : "*",
"match_mapping_type" : "string",
"mapping" : {
"type" : "text",
"norms" : false,
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 1024
}
}
}
}
}
],
"properties" : {
"@timestamp" : {
"type" : "date"
},
"@version" : {
"type" : "keyword"
},
"geoip" : {
"dynamic" : true,
"properties" : {
"ip" : {
"type" : "ip"
},
"location" : {
"type" : "geo_point"
},
"latitude" : {
"type" : "half_float"
},
"longitude" : {
"type" : "half_float"
}
}
}
}
}
},
"aliases" : { }
}
}
此时在logstash的配置文件中使用如下配置，启动logstash后，可以看到logstash会报错
elasticsearch {
hosts => ["192.168.10.101:9200"]
index => "nginx-%{+YYYY.MM.dd}"
template => "/etc/logstash/nginx.json"
template_name => "nginx"
template_overwrite => true
}
logstash日志报错如下：
Installing elasticsearch template to _template/nginx
Failed to install template. {:message=>"Got response code '400' contacting Elasticsearch at
URL 'http://192.168.10.101:9200/_template/nginx'", :class=>"LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError", :backtrace=>["/usr/shar
e/logstash/vendor/bundle/jruby/2.3.0/gems/logstash-output-elasticsearch-9.2.0-java/lib/logstash/outputs/elasticsearch/http_client/manticore_adapter.rb:80:in
`perform_request'", "/usr/share/logstash/vendor/bundle/jruby/2.3.0/gems/logstash-output-elasticsearch-9.2.0-java/lib/logstash/outputs/elasticsearch/http_clie
nt/pool.rb:291:in `perform_request_to_url'", "/usr/share/logstash/vendor/bundle/jruby/2.3.0/gems/logstash-output-elasticsearch-9.2.0-java/lib/logstash/output
s/elasticsearch/http_client/pool.rb:278:in `block in perform_request'", "/usr/share/logstash/vendor/bundle/jruby/2.3.0/gems/logstash-output-elasticsearch-9.2
.0-java/lib/logstash/outputs/elasticsearch/http_client/pool.rb:373:in `with_connection'", "/usr/share/logstash/vendor/bundle/jruby/2.3.0/gems/logstash-output
-elasticsearch-9.2.0-java/lib/logstash/outputs/elasticsearch/http_client/pool.rb:277:in `perform_request'", "/usr/share/logstash/vendor/bundle/jruby/2.3.0/ge
ms/logstash-output-elasticsearch-9.2.0-java/lib/logstash/outputs/elasticsearch/http_client/pool.rb:285:in `block in Pool'", "/usr/share/logstash/vendor/bundl
e/jruby/2.3.0/gems/logstash-output-elasticsearch-9.2.0-java/lib/logstash/outputs/elasticsearch/http_client.rb:348:in `template_put'", "/usr/share/logstash/ve
ndor/bundle/jruby/2.3.0/gems/logstash-output-elasticsearch-9.2.0-java/lib/logstash/outputs/elasticsearch/http_client.rb:86:in `template_install'", "/usr/shar
e/logstash/vendor/bundle/jruby/2.3.0/gems/logstash-output-elasticsearch-9.2.0-java/lib/logstash/outputs/elasticsearch/template_manager.rb:21:in `install'", "
/usr/share/logstash/vendor/bundle/jruby/2.3.0/gems/logstash-output-elasticsearch-9.2.0-java/lib/logstash/outputs/elasticsearch/template_manager.rb:9:in `inst
all_template'", "/usr/share/logstash/vendor/bundle/jruby/2.3.0/gems/logstash-output-elasticsearch-9.2.0-java/lib/logstash/outputs/elasticsearch/common.rb:118
:in `install_template'", "/usr/share/logstash/vendor/bundle/jruby/2.3.0/gems/logstash-output-elasticsearch-9.2.0-java/lib/logstash/outputs/elasticsearch/comm
on.rb:49:in `block in install_template_after_successful_connection'"]}
如果复制nginx.json的文件内容直接在kibana的ES控制台上导入的话，会直接看到报错提示
PUT _template/nginx-test
{
"nginx" : {
"order" : 0,
"version" : 60001,
"index_patterns" : [
"nginx-*"
],
"settings" : {
"index" : {
"refresh_interval" : "5s"
}
},
"mappings" : {
"_default_" : {
"dynamic_templates" : [
{
"message_field" : {
"path_match" : "message",
"match_mapping_type" : "string",
"mapping" : {
"type" : "text",
"norms" : false
}
}
},
{
"string_fields" : {
"match" : "*",
"match_mapping_type" : "string",
"mapping" : {
"type" : "text",
"norms" : false,
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 1024
}
}
}
}
}
],
"properties" : {
"@timestamp" : {
"type" : "date"
},
"@version" : {
"type" : "keyword"
},
"geoip" : {
"dynamic" : true,
"properties" : {
"ip" : {
"type" : "ip"
},
"location" : {
"type" : "geo_point"
},
"latitude" : {
"type" : "half_float"
},
"longitude" : {
"type" : "half_float"
}
}
}
}
}
},
"aliases" : { }
}
}
右边返回信息栏中报错如下：
{
"error": {
"root_cause": [
{
"type": "action_request_validation_exception",
"reason": "Validation Failed: 1: index patterns are missing;"
}
],
"type": "action_request_validation_exception",
"reason": "Validation Failed: 1: index patterns are missing;"
},
"status": 400
}
当初笔者就是为此报错抓狂过，但最后算是绕了一个很大的圈子找到了第一种方式的文件模板，最后才成功实现
给nginx自定义映射模板。所以，如果正在抓狂的你看到了我的这篇文章，是不是有一种雪中送炭的感觉了！，
如果感觉有，请点赞+转发！
页:
[1]