elasticsearch - 如何获得多个记录？

Question

我正在使用带logstash的JDBC从PostgreSQL查询中获取数据并将其导出到ES v7，这是配置文件:

input {
    jdbc {
        jdbc_connection_string => "jdbc:postgresql://ldatabase_rds_path?useSSL=true"
        jdbc_user => "username"
        jdbc_password => "password"
        jdbc_driver_library => "/home/z/Documents/postgresql-42.2.18.jar"
        jdbc_driver_class => "org.postgresql.Driver"
        tracking_column => "id"
        tracking_column_type => "numeric"
         clean_run => true
        schedule => "0 */1 * * *"
        statement => "SELECT id as id, type as type, z_id as z_id, sender_id as sender_id, receiver_id as receiver_id, status as status, amount as amount, fees as fees, created as created, metadata as metadata, funding_source_from_id as funding_source_from_id, funding_source_to_id as funding_source_to_id, is_parent as is_parent, destination_type as destination_type, source_type as source_type FROM payments_transfer"
    }
}
output {
        stdout { codec => json_lines }
        elasticsearch {
                hosts => ["localhost:9200"]
                manage_template => false
                index => "payments_transfer_data"
                document_id => "%{id}"
         }
}

从数据库仅获得1条记录需要花费大量时间!
我尝试了一些解决方案，例如显式定义映射，因此为这样的数据添加了映射:

PUT payments_transfer_data/_mapping/doc?include_type_name=true
{
  "properties": {
    "@timestamp": {
      "type": "date"
    },
    "@version": {
      "type": "text",
      "fields": {
        "keyword": {
          "type": "keyword"
        }
      }
    },
    "amount": {
      "type": "float"
    },
    "created": {
      "type": "date"
    },
    "destination_type": {
      "type": "text",
      "fields": {
        "keyword": {
          "type": "keyword"
        }
      }
    },
    "z_id": {
      "type": "text",
      "fields": {
        "keyword": {
          "type": "keyword"
        }
      }
    },
    "fees": {
      "type": "float"
    },
    "funding_source_from_id": {
      "type": "long"
    },
    "funding_source_to_id": {
      "type": "long"
    },
    "id": {
      "type": "long"
    },
    "is_parent": {
      "type": "boolean"
    },
    "metadata": {
      "type": "keyword"
    },
    "receiver_id": {
      "type": "long"
    },
    "sender_id": {
      "type": "long"
    },
    "source_type": {
      "type": "text",
      "fields": {
        "keyword": {
          "type": "keyword"
        }
      }
    },
    "status": {
      "type": "text",
      "fields": {
        "keyword": {
          "type": "keyword"
        }
      }
    },
    "type": {
      "type": "text",
      "fields": {
        "keyword": {
          "type": "keyword"
        }
      }
    }
  }
}

这是唯一获得的记录:

    source_type:customer sender_id:3 destination_type:funding-source type:funded_transfer z_id:863a240c-2011-e911-8114-bacd823e9f1d receiver_id:65 status:processed amount:550 funding_source_from_id:332 @timestamp:Nov 8, 2020 @ 16:00:08.809 fees:5.61 is_parent:false created:Jan 5, 2019 @ 21:28:21.847 @version:1 id:2,160 funding_source_to_id: - metadata: - _id:2160 _type:doc _index:payments_transfer_data _score: -

Answer 1

根据官方文档，您应该将tracking_column与use_column_value结合使用。使用您当前的设置，tracking_column不会有任何影响。
您是否尝试过使用Select * from查看是否所有内容都已被提取。