elasticsearch - 来自Logstash的Elasticsearch映射配置

Question

我正在尝试从Windows上的Elasticsearch配置Logstash的索引模板管理。

我有c:\ulyaoth\logstash-2.3.1\bin\logstash.json文件:

    input {
      beats {
       port => 5044
       type => "log"
      }
    }

    filter {
        grok {
            match => ["message","%{TIMESTAMP_ISO8601:timestamp_match}"]
            remove_field => ["_id","_index","_score","_type","beat.hostname","beat.name","count","fileds","host","input_type","offset","tags","type"]
        }

        mutate {
            remove_field => ["_id","_index","_score","_type","beat.hostname","beat.name","count","fileds","host","input_type","offset","tags","type"]
        }

        date {
            match => ["timestamp_match","YYYY-MM-dd HH:mm:ss.SSS"]
            target => "timestamp_match"
        }
    }

    output {
      elasticsearch {
        hosts => "localhost:9200"
        index => "%{[@metadata][beat]}-%{+YYYY.MM.dd}"
        document_type => "%{[@metadata][type]}"
        template => "c:/ulyaoth/logstash-2.3.1/bin/elasticsearch-template.custom.json"
        template_name => "elasticsearch-template"
        manage_template => true
        template_overwrite => true
      }
    }

和模板文件 c:/ulyaoth/logstash-2.3.1/bin/elasticsearch-template.custom.json。我从 c:\ulyaoth\logstash-2.3.1\vendor\bundle\jruby\1.9\gems\logstash-output-elasticsearch-2.5.5-java\lib\logstash\outputs\elasticsearch\中挖出了这个文件，并对其进行了编辑，以便:

    "source":{"index": "not_analyzed"}

这是整个文件:

    {
      "template" : "logstash-*",
      "settings" : {
        "index.refresh_interval" : "5s"
      },
      "mappings" : {
        "_default_" : {
          "_all" : {"enabled" : true, "omit_norms" : true},
          "dynamic_templates" : [ {
            "message_field" : {
              "match" : "message",
              "match_mapping_type" : "string",
              "mapping" : {
                "type" : "string", "index" : "analyzed", "omit_norms" : true,
                "fielddata" : { "format" : "disabled" }
              }
            }
          }, {
            "string_fields" : {
              "match" : "*",
              "match_mapping_type" : "string",
              "mapping" : {
                "type" : "string", "index" : "analyzed", "omit_norms" : true,
                "fielddata" : { "format" : "disabled" },
                "fields" : {
                  "raw" : {"type": "string", "index" : "not_analyzed", "doc_values" : true, "ignore_above" : 256}
                }
              }
            }
          }, {
            "float_fields" : {
              "match" : "*",
              "match_mapping_type" : "float",
              "mapping" : { "type" : "float", "doc_values" : true }
            }
          }, {
            "double_fields" : {
              "match" : "*",
              "match_mapping_type" : "double",
              "mapping" : { "type" : "double", "doc_values" : true }
            }
          }, {
            "byte_fields" : {
              "match" : "*",
              "match_mapping_type" : "byte",
              "mapping" : { "type" : "byte", "doc_values" : true }
            }
          }, {
            "short_fields" : {
              "match" : "*",
              "match_mapping_type" : "short",
              "mapping" : { "type" : "short", "doc_values" : true }
            }
          }, {
            "integer_fields" : {
              "match" : "*",
              "match_mapping_type" : "integer",
              "mapping" : { "type" : "integer", "doc_values" : true }
            }
          }, {
            "long_fields" : {
              "match" : "*",
              "match_mapping_type" : "long",
              "mapping" : { "type" : "long", "doc_values" : true }
            }
          }, {
            "date_fields" : {
              "match" : "*",
              "match_mapping_type" : "date",
              "mapping" : { "type" : "date", "doc_values" : true }
            }
          }, {
            "geo_point_fields" : {
              "match" : "*",
              "match_mapping_type" : "geo_point",
              "mapping" : { "type" : "geo_point", "doc_values" : true }
            }
          } ],
          "properties" : {
            "@timestamp": { "type": "date", "doc_values" : true },
            "@version": { "type": "string", "index": "not_analyzed", "doc_values" : true },
            "source":{"index": "not_analyzed"}
            "geoip"  : {
              "type" : "object",
              "dynamic": true,
              "properties" : {
                "ip": { "type": "ip", "doc_values" : true },
                "location" : { "type" : "geo_point", "doc_values" : true },
                "latitude" : { "type" : "float", "doc_values" : true },
                "longitude" : { "type" : "float", "doc_values" : true }
              }
            }
          }
        }
      }
    }

我的问题是模板未注册。 REST查询返回空对象，我也看到该字段仍在Kibana中进行了分析。

    GET /_template HTTP/1.1
    Host: 127.0.0.1:9200

另一个问题是 remove_field也不起作用-我仍然看到所有这些字段。

    remove_field => ["_id","_index","_score","_type","beat.hostname","beat.name","count","fileds","host","input_type","offset","tags","type"]

我看不到任何logstash日志(具有讽刺意味的是:)，在ES日志中，我看不到任何错误或模板问题。

如何解决这些问题？

编辑:

最终的工作配置为:

    {
      "template" : "filebeat-*",
      "settings" : {
        "index.refresh_interval" : "5s"
      },
      "mappings" : {
        "_default_" : {
          "_all" : {"enabled" : true, "omit_norms" : true},
          "dynamic_templates" : [ {
            "message_field" : {
              "match" : "message",
              "match_mapping_type" : "string",
              "mapping" : {
                "type" : "string", "index" : "analyzed", "omit_norms" : true,
                "fielddata" : { "format" : "disabled" }
              }
            }
          }, {
            "string_fields" : {
              "match" : "*",
              "match_mapping_type" : "string",
              "mapping" : {
                "type" : "string", "index" : "analyzed", "omit_norms" : true,
                "fielddata" : { "format" : "disabled" },
                "fields" : {
                  "raw" : {"type": "string", "index" : "not_analyzed", "doc_values" : true, "ignore_above" : 256}
                }
              }
            }
          }, {
            "float_fields" : {
              "match" : "*",
              "match_mapping_type" : "float",
              "mapping" : { "type" : "float", "doc_values" : true }
            }
          }, {
            "double_fields" : {
              "match" : "*",
              "match_mapping_type" : "double",
              "mapping" : { "type" : "double", "doc_values" : true }
            }
          }, {
            "byte_fields" : {
              "match" : "*",
              "match_mapping_type" : "byte",
              "mapping" : { "type" : "byte", "doc_values" : true }
            }
          }, {
            "short_fields" : {
              "match" : "*",
              "match_mapping_type" : "short",
              "mapping" : { "type" : "short", "doc_values" : true }
            }
          }, {
            "integer_fields" : {
              "match" : "*",
              "match_mapping_type" : "integer",
              "mapping" : { "type" : "integer", "doc_values" : true }
            }
          }, {
            "long_fields" : {
              "match" : "*",
              "match_mapping_type" : "long",
              "mapping" : { "type" : "long", "doc_values" : true }
            }
          }, {
            "date_fields" : {
              "match" : "*",
              "match_mapping_type" : "date",
              "mapping" : { "type" : "date", "doc_values" : true }
            }
          }, {
            "geo_point_fields" : {
              "match" : "*",
              "match_mapping_type" : "geo_point",
              "mapping" : { "type" : "geo_point", "doc_values" : true }
            }
          } ],
          "properties" : {
            "@timestamp": { "type": "date", "doc_values" : true },
            "@version": { "type": "string", "index": "not_analyzed", "doc_values" : true },
            "source":{ "type": "string", "index": "not_analyzed"}
            "geoip"  : {
              "type" : "object",
              "dynamic": true,
              "properties" : {
                "ip": { "type": "ip", "doc_values" : true },
                "location" : { "type" : "geo_point", "doc_values" : true },
                "latitude" : { "type" : "float", "doc_values" : true },
                "longitude" : { "type" : "float", "doc_values" : true }
              }
            }
          }
        }
      }
    }

已更改"template" : "filebeat-*"，
和"source":{ "type": "string", "index": "not_analyzed"}

Answer 1

source字段没有type。也许你的意思是:

"source":{ "type": "string", "index": "not_analyzed"},