node.js - 我无法过滤也无法聚合通过 LogStash 保存到 ElasticSearch 的文档

Question

我猜这个问题可能与我的logstash.conf有关，但我不知道到底该怎么做。我发现了一些优秀的教程，解释了如何仅使用 ElasticSearch 来完成此操作，但在我的例子中，所有数据都将通过 LogStash 来自 NodeJs。

我搜索有关启用 fieldData 的信息，但我不知道如何在 logstash.conf 中执行此操作。我应该创建索引模板吗？如果是这样怎么办？

上下文是我想在用户每次访问我们的应用程序时进行记录，然后根据每月的访问次数向他/她计费。

logstash.conf

input {
  tcp {
    port => 5000
    type => cpfTipo
  }
}

filter {
  json {
    source => "message"
  } 
}

output {
  elasticsearch { hosts => ["localhost:9200"] index => "mycostumer_indice" document_type => "cpfTipo"}
}

暂定过滤:

1)

curl -XGET http://127.0.0.1:9200/mycostumer_indice/cpfTipo/_search -d '{
    "query": {
        "filtered": {
            "query": {
                "match_all": {}
            },
            "filter": 
                    {
                        "term": {
                            "username": "a"
                        }
                    }
                ]
            }
        }
    }
}'
{"error":{"root_cause":[{"type":"parsing_exception","reason":"no [query] registered for [filtered]","line":3,"col":21}],"type":"parsing_exception","reason":"no [query] registered for [filtered]","line":3,"col":21},"status":400}demetrio@nodejs ~/tool

暂定聚合:

1)

curl -XGET http://127.0.0.1:9200/mycostumer_indice/cpfTipo/_search -d '{
{
    "aggs" : {
        "message" : {
            "terms" : {
                "field" : "cpfTipo",
                "size" : 5
            }
        }
    }
}'
{"error":{"root_cause":[{"type":"json_parse_exception","reason":"Unexpected character ('{' (code 123)): was expecting double-quote to start field name\n at [Source: org.elasticsearch.transport.netty4.ByteBufStreamInput@3ce63313; line: 2, column: 2]"}],"type":"json_parse_exception","reason":"Unexpected character ('{' (code 123)): was expecting double-quote to start field name\n at [Source: org.elasticsearch.transport.netty4.ByteBufStreamInput@3ce63313; line: 2, column: 2]"},"status":500}

2)

curl -XPOST 'http://127.0.0.1:9200/mycostumer_indice/_search?pretty' -d '
{
  "size": 0,
  "aggs": {
    "group_by_username": {
      "terms": {
        "field": "username"
      }
    }
  }
}'
{
  "error" : {
    "root_cause" : [
      {
        "type" : "illegal_argument_exception",
        "reason" : "Fielddata is disabled on text fields by default. Set fielddata=true on [username] in order to load fielddata in memory by uninverting the inverted index. Note that this can however use significant memory."
      }
    ],
    "type" : "search_phase_execution_exception",
    "reason" : "all shards failed",
    "phase" : "query",
    "grouped" : true,
    "failed_shards" : [
      {
        "shard" : 0,
        "index" : "mycostumer_indice",
        "node" : "-em7X-ssT3SL2JBtfs0VTQ",
        "reason" : {
          "type" : "illegal_argument_exception",
          "reason" : "Fielddata is disabled on text fields by default. Set fielddata=true on [username] in order to load fielddata in memory by uninverting the inverted index. Note that this can however use significant memory."
        }
      }
    ],
    "caused_by" : {
      "type" : "illegal_argument_exception",
      "reason" : "Fielddata is disabled on text fields by default. Set fielddata=true on [username] in order to load fielddata in memory by uninverting the inverted index. Note that this can however use significant memory."
    }
  },
  "status" : 400
}

mycostumer 索引如何显示:

curl http://127.0.0.1:9200/mycostumer_indice/cpfTipo/_search?pretty 
{
  "took" : 2,
  "timed_out" : false,
  "_shards" : {
    "total" : 5,
    "successful" : 5,
    "failed" : 0
  },
  "hits" : {
    "total" : 1,
    "max_score" : 1.0,
    "hits" : [
      {
        "_index" : "mycostumer_indice",
        "_type" : "cpfTipo",
        "_id" : "AVrxUi5cIZDJUBCguFI8",
        "_score" : 1.0,
        "_source" : {
          "password" : "a",
          "@timestamp" : "2017-03-21T14:42:54.466Z",
          "port" : 56012,
          "@version" : "1",
          "host" : "127.0.0.1",
          "message" : "{\"username\":\"a\",\"password\":\"a\"}",
          "type" : "cpfTipo",
          "username" : "a"
        }
      }
    ]
  }
}

在nodeJs中

var express = require('express');
var bodyParser = require('body-parser');
var Client = require('node-rest-client').Client;

var expressWinston = require('express-winston');
var winston = require('winston');
require('winston-logstash');

var client = new Client();

var Logstash = require('logstash-client');

var app = express();

expressWinston.requestWhitelist.push('body');
expressWinston.responseWhitelist.push('body')

app.use(bodyParser.json());
app.use(bodyParser.urlencoded({
  extended: true
}));

var port = process.env.PORT || 3000;

var router = express.Router();

var tokenRoute = router.route('/token');

tokenRoute.post(function (req, res) {

  var user = {
    username: req.body.username,
    password: req.body.password
  };
  logstash.send(user);

Answer 1

您的第一个搜索查询使用了已弃用的filtered查询，只需将其替换为bool即可:

curl -XGET http://127.0.0.1:9200/mycostumer_indice/cpfTipo/_search -d '{
    "query": {
        "bool": {
            "filter": 
                    {
                        "term": {
                            "username": "a"
                        }
                    }
                ]
            }
        }
    }
}'

您的第二个查询开头有太多左大括号，请改用这个。

curl -XGET http://127.0.0.1:9200/mycostumer_indice/cpfTipo/_search -d '{
    "aggs" : {
        "message" : {
            "terms" : {
                "field" : "cpfTipo",
                "size" : 5
            }
        }
    }
}'

您的第三个查询失败，因为您尝试聚合用户名，这是一个文本字段。您应该更改该字段的映射以使用 keyword type相反。