Elasticsearch 桶聚合返回错误的 doc_count

Question

我正在使用 Elasticsearch 桶聚合。我有一组文档，每个文档都有一个类别字段，它是一个数组。我需要获取每个类别的计数以及搜索结果。但目前，我得到了错误的 doc_count。

这是我的聚合查询

{

"aggs" : {
    "category" : {
        "terms" : { 
            "field" : "category.keyword"
        }
    }
}

下面是我使用错误的 doc_count 得到的结果。

{
"took": 1,
"timed_out": false,
"_shards": {
    "total": 1,
    "successful": 1,
    "skipped": 0,
    "failed": 0
},
"hits": {
    "total": {
        "value": 112,
        "relation": "eq"
    },
    "max_score": 1.0,
    "hits": [
        {
            "_index": "pages",
            "_type": "_doc",
            "_id": "ljrbn3ABGDJu_xG-W2Mm",
            "_score": 1.0,
            "_source": {
                "title": "testing code.",
                "location_en": "Europe",
                "location_fr": "Europe-Fr",
                "start_date": "2020-03-03 10:05:17",
                "end_date": "2020-03-13 10:05:17",
                "category": [
                    "job",
                    "university",
                    "europe",
                    "researcher",
                    "law"
                ]
            }
        },
        {
            "_index": "pages",
            "_type": "_doc",
            "_id": "lzrbn3ABGDJu_xG-W2PX",
            "_score": 1.0,
            "_source": {
                "title": "Test",
                "slug": "Test",
                "location_en": "Asia",
                "location_fr": "Asia-Fr",
                "start_date": "2020-03-03 10:05:18",
                "end_date": "2020-03-13 10:05:18",
                "category": [
                    "job",
                    "uppsala-university",
                    "asia",
                    "PhD",
                    "history"
                ]
            }
        },
        {
            "_index": "pages",
            "_type": "_doc",
            "_id": "mDrbn3ABGDJu_xG-W2Pl",
            "_score": 1.0,
            "_source": {
                "title": "Test",
                "slug": "Test",
                "location_en": "Europe",
                "location_fr": "Europe-Fr",
                "start_date": "2020-03-03 10:05:18",
                "end_date": "2020-03-13 10:05:18",
                "category": [
                    "job",
                    "university",
                    "europe",
                    "researcher",
                    "law"
                ]
            }
        },
        {
            "_index": "pages",
            "_type": "_doc",
            "_id": "mTrbn3ABGDJu_xG-XGOO",
            "_score": 1.0,
            "_source": {
                "title": "Test",
                "slug": "Test",
                "location_en": "Asia",
                "location_fr": "Asia-Fr",
                "start_date": "2020-03-03 10:05:18",
                "end_date": "2020-03-13 10:05:18",
                "category": [
                    "job",
                    "university",
                    "asia",
                    "PhD",
                    "history"
                ]
            }
        },
        {
            "_index": "pages",
            "_type": "_doc",
            "_id": "mjrbn3ABGDJu_xG-XGOb",
            "_score": 1.0,
            "_source": {
                "title": "testing world",
                "location_en": "Europe",
                "location_fr": "Europe-Fr",
                "start_date": "2020-03-03 10:05:18",
                "end_date": "2020-03-13 10:05:18",
                "category": [
                    "job",
                    "university",
                    "europe",
                    "researcher",
                    "law"
                ]
            }
        },
        {
            "_index": "pages",
            "_type": "_doc",
            "_id": "mzrbn3ABGDJu_xG-XWNG",
            "_score": 1.0,
            "_source": {
                "title": "hello",
                "slug": "Helloo",
                "short_description_en": "Helloo",
                "location_en": "Asia",
                "location_fr": "Asia-Fr",
                "start_date": "2020-03-03 10:05:18",
                "end_date": "2020-03-13 10:05:18",
                "category": [
                    "job",
                    "university",
                    "asia",
                    "PhD",
                    "history"
                ]
            }
        },
        {
            "_index": "pages",
            "_type": "_doc",
            "_id": "nDrbn3ABGDJu_xG-XWNU",
            "_score": 1.0,
            "_source": {
                "title": "Hello",
                "slug": "helloo",
                "short_description_en": "hello worldf",
                "location_en": "Europe",
                "location_fr": "Europe-Fr",
                "start_date": "2020-03-03 10:05:18",
                "end_date": "2020-03-13 10:05:18",
                "category": [
                    "job",
                    "university",
                    "europe",
                    "researcher",
                    "law"
                ]
            }
        },
        {
            "_index": "pages",
            "_type": "_doc",
            "_id": "nTrbn3ABGDJu_xG-XmMF",
            "_score": 1.0,
            "_source": {
                "title": "Test",
                "slug": "test",
                "short_description_en": "Test",
                "location_en": "Asia",
                "location_fr": "Asia-Fr",
                "start_date": "2020-03-03 10:05:18",
                "end_date": "2020-03-13 10:05:18",
                "category": [
                    "job",
                    "university",
                    "asia",
                    "PhD",
                    "history"
                ]
            }
        },
        {
            "_index": "pages",
            "_type": "_doc",
            "_id": "njrbn3ABGDJu_xG-XmMS",
            "_score": 1.0,
            "_source": {
                "title": "Test",
                "slug": "test",
                "short_description_en": "Test",
                "location_en": "Europe",
                "location_fr": "Europe-Fr",
                "start_date": "2020-03-03 10:05:18",
                "end_date": "2020-03-13 10:05:18",
                "category": [
                    "job",
                    "university",
                    "europe",
                    "researcher",
                    "law"
                ]
            }
        },
        {
            "_index": "pages",
            "_type": "_doc",
            "_id": "nzrbn3ABGDJu_xG-XmPZ",
            "_score": 1.0,
            "_source": {
                "title": "Researcher position in accelerator mass spectrometry (AMS)",
                "slug": "researcher-position-in-accelerator-mass-spectrometry-ams",
                "short_description_en": "Uppsala University is a comprehensive research-intensive university with a strong international standing. Our mission is to pursue top-quality research and education and to interact constructively with society. Our most important assets are all the individuals whose curiosity and...",
                "location_en": "Asia",
                "location_fr": "Asia-Fr",
                "start_date": "2020-03-03 10:05:18",
                "end_date": "2020-03-13 10:05:18",
                "category": [
                    "job",
                    "university",
                    "asia",
                    "PhD",
                    "history"
                ]
            }
        }
    ]
},
"aggregations": {
    "category": {
        "doc_count_error_upper_bound": 0,
        "sum_other_doc_count": 0,
        "buckets": [
            {
                "key": "job",
                "doc_count": 112
            },
            {
                "key": "university",
                "doc_count": 112
            },
            {
                "key": "PhD",
                "doc_count": 56
            },
            {
                "key": "asia",
                "doc_count": 56
            },
            {
                "key": "europe",
                "doc_count": 56
            },
            {
                "key": "history",
                "doc_count": 56
            },
            {
                "key": "law",
                "doc_count": 56
            },
            {
                "key": "researcher",
                "doc_count": 56
            }
        ]
    }
}

Answer 1

众所周知，从数组中获取唯一 (doc) 计数会引起头痛。尝试使用脚本化的指标聚合

1. 遍历找到类别字段的所有文档
2. 遍历所有单独的类别
3. 最后将它们保存在 HashMap 中

我认为这对应于您所追求的出现次数。

GET pages/_search
{
  "size": 0,
  "aggs": {
    "scripted_non_uniques": {
      "scripted_metric": {
        "init_script": "state.map = [:];",
        "map_script": """
              if (doc.containsKey('category')) {
                for (def val : doc['category.keyword']) {
                  if (state.map.containsKey(val)) {
                    // increment if existing
                    state.map[val] += 1;
                  } else {
                    // initialize to increment later
                    state.map[val] = 1;
                  }
                }
              }
        """,
        "combine_script": "              return state",
        "reduce_script": "              return states"
      }
    }
  }
}

产生以下结果(基于您示例中的 10 次匹配):

{
  "took" : 17,
  "timed_out" : false,
  "_shards" : {
    "total" : 1,
    "successful" : 1,
    "skipped" : 0,
    "failed" : 0
  },
  "hits" : {
    "total" : {
      "value" : 10,
      "relation" : "eq"
    },
    "max_score" : null,
    "hits" : [ ]
  },
  "aggregations" : {
    "scripted_non_uniques" : {
      "value" : [
        {
          "map" : {
            "law" : 5,
            "researcher" : 5,
            "university" : 9,
            "asia" : 5,
            "uppsala-university" : 1,
            "history" : 5,
            "job" : 10,
            "europe" : 5,
            "PhD" : 5
          }
        }
      ]
    }
  }
}

请注意，此脚本会多次计算一个 category 数组中的重复类别，因此您也应该考虑这种边缘情况。

例如，当您处于 for 循环中时，您可以通过调用 Debug.explain(val) 来调试脚本代码。您可以阅读有关脚本上下文的更多信息 here .