json - 如何从Json文件中使用Logstash获取TimeStamp？ JSON中有多个日期字段

Question

我有如下的JSON输入，其中包含日期字段，并且需要从Json中提取日期时间字段，

{
  "Properties": {
         "Client Name": "Chubb",
         "Portfolio": "Chubb-Transfer"
  },
"Capture": [
         {
            "CaptureGUID": "caa1f5ba-1e93-4926-b3ac-e30d0d9d4cbb",
            "HTMLPath": "Captures\\C:\\",
            "ScreenName": "Amdocs CRM - ClearCallCenter - [Console]",
            "TimeStamp": "20170926110036"
          },
         {
            "CaptureGUID": "0faf6b54-999f-4bfd-b8d0-e81a589f9185",
            "HTMLPath": "Captures\\C:\\",
            "ScreenName": "Microsoft Excel - 1.0.1 1.0.6 1.0.8 Match 3.0.6 Hit NAIC Optimized.xlsx",
            "TimeStamp": "20170926105418"
          }
     ]
}

并且我的Logstash配置如下，如何将字符串日期(“TimeStamp”:“20170926105418”)转换为日期格式。已使用完整的Logstash文件进行了更新

input {
    file { 
        type => "json"
        path => "C:/ELK/data/Recordings/*.json"
        start_position => beginning
        codec => multiline {
           pattern => "^{"
           negate => "true"
           what => "previous"
           multiline_tag => "multi_tagged"
           max_lines => 30000
       }
    }
}
filter{
    date {
        match => ["Capture.TimeStamp", "yyyyMMddHHmmss"]
        target => "TimeStamp"
    }

    mutate { 
    replace => { "message" => "%{message}}" }
    gsub => [ 'message','\n','']
    }

    json { 
        source => "message" 
        remove_field => ["message"]
    }


}

output {
    elasticsearch {
    index => "test10"
    }
    stdout { codec => rubydebug }
}

Answer 1

从logstash配置文件中删除日期过滤器。映射索引时处理日期解析。以下是您的用例的映射。

PUT json
{
  "mappings": {
    "json": {
      "properties": {
        "Capture": {
          "type": "nested",
          "properties": {
            "CaptureGUID": {
              "type": "text",
              "fields": {
                "keyword": {
                  "type": "keyword",
                  "ignore_above": 256
                }
              }
            },
            "HTMLPath": {
              "type": "text",
              "fields": {
                "keyword": {
                  "type": "keyword",
                  "ignore_above": 256
                }
              }
            },
            "ScreenName": {
              "type": "text",
              "fields": {
                "keyword": {
                  "type": "keyword",
                  "ignore_above": 256
                }
              }
            },
            "TimeStamp": {
              "type": "date",
              "format": "yyyyMMddHHmmss"
            }
          }
        },
        "Properties": {
          "properties": {
            "Client Name": {
              "type": "text",
              "fields": {
                "keyword": {
                  "type": "keyword",
                  "ignore_above": 256
                }
              }
            },
            "Portfolio": {
              "type": "text",
              "fields": {
                "keyword": {
                  "type": "keyword",
                  "ignore_above": 256
                }
              }
            }
          }
        }
      }
    }
  }
}