elasticsearch - 提示大小会影响文档的索引速度吗？

Question

目前，我想在aws ec2中托管的ES服务上为132 Million文档建立索引，我能够在一周内完成98 Million。
我注意到，随着索引大小的增加，索引速度逐渐降低，该索引当前位于44GB。
我尝试暂停该过程，并从某些 Angular 恢复该过程，但是速度绝对不一致。
索引大小和文档索引速度之间是否有关系？
在这种情况下，如果可以的话，将不胜感激有关如何提高索引速度的提示，谢谢。
集群设置

{
   "persistent":{
      
   },
   "transient":{
      
   },
   "defaults":{
      "cluster":{
         "routing":{
            "use_adaptive_replica_selection":"false",
            "rebalance":{
               "enable":"all"
            },
            "allocation":{
               "node_concurrent_incoming_recoveries":"2",
               "node_initial_primaries_recoveries":"4",
               "same_shard":{
                  "host":"false"
               },
               "total_shards_per_node":"-1",
               "type":"balanced",
               "disk":{
                  "threshold_enabled":"true",
                  "watermark":{
                     "low":"85%",
                     "flood_stage":"95%",
                     "high":"90%"
                  },
                  "include_relocations":"true",
                  "reroute_interval":"60s"
               },
               "awareness":{
                  "attributes":[
                     
                  ]
               },
               "balance":{
                  "index":"0.55",
                  "threshold":"1.0",
                  "shard":"0.45"
               },
               "enable":"all",
               "node_concurrent_outgoing_recoveries":"2",
               "allow_rebalance":"indices_all_active",
               "cluster_concurrent_rebalance":"2",
               "node_concurrent_recoveries":"2"
            }
         },
         "indices":{
            "tombstones":{
               "size":"500"
            },
            "close":{
               "enable":"true"
            }
         },
         "nodes":{
            "reconnect_interval":"10s"
         },
         "persistent_tasks":{
            "allocation":{
               "enable":"all"
            }
         },
         "blocks":{
            "read_only_allow_delete":"false",
            "read_only":"false"
         },
         "service":{
            "slow_task_logging_threshold":"30s"
         },
         "name":"roseland",
         "info":{
            "update":{
               "interval":"30s",
               "timeout":"15s"
            }
         }
      },
      "no":{
         "model":{
            "state":{
               "persist":"false"
            }
         }
      },
      "logger":{
         "level":"INFO"
      },
      "bootstrap":{
         "memory_lock":"false",
         "system_call_filter":"false",
         "ctrlhandler":"true"
      },
      "processors":"2",
      "ingest":{
         "grok":{
            "watchdog":{
               "max_execution_time":"1s",
               "interval":"1s"
            }
         }
      },
      "network":{
         "host":[
            "_local:ipv4_",
            "_eth0:ipv4_"
         ],
         "tcp":{
            "reuse_address":"true",
            "keep_alive":"true",
            "connect_timeout":"30s",
            "receive_buffer_size":"-1b",
            "no_delay":"true",
            "send_buffer_size":"-1b"
         },
         "bind_host":[
            "_local:ipv4_",
            "_eth0:ipv4_"
         ],
         "server":"true",
         "breaker":{
            "inflight_requests":{
               "limit":"100%",
               "overhead":"1.0"
            }
         },
         "publish_host":[
            "_local:ipv4_",
            "_eth0:ipv4_"
         ]
      },
      "pidfile":"/var/run/elasticsearch/elasticsearch.pid",
      "path":{
         "data":[
            "/data/elasticsearch"
         ],
         "logs":"/var/log/elasticsearch",
         "shared_data":"",
         "home":"/usr/share/elasticsearch",
         "repo":[
            
         ]
      },
      "search":{
         "default_search_timeout":"-1",
         "highlight":{
            "term_vector_multi_value":"true"
         },
         "default_allow_partial_results":"true",
         "max_buckets":"-1",
         "low_level_cancellation":"false",
         "keep_alive_interval":"1m",
         "remote":{
            "node":{
               "attr":""
            },
            "initial_connect_timeout":"30s",
            "connect":"true",
            "connections_per_cluster":"3"
         },
         "default_keep_alive":"5m",
         "max_keep_alive":"24h"
      },
      "security":{
         "manager":{
            "filter_bad_defaults":"true"
         }
      },
      "repositories":{
         "fs":{
            "compress":"false",
            "chunk_size":"9223372036854775807b",
            "location":""
         },
         "url":{
            "supported_protocols":[
               "http",
               "https",
               "ftp",
               "file",
               "jar"
            ],
            "allowed_urls":[
               
            ],
            "url":"http:"
         }
      },
      "action":{
         "auto_create_index":"true",
         "search":{
            "shard_count":{
               "limit":"9223372036854775807"
            }
         },
         "destructive_requires_name":"false",
         "master":{
            "force_local":"false"
         }
      },
      "client":{
         "type":"node",
         "transport":{
            "ignore_cluster_name":"false",
            "nodes_sampler_interval":"5s",
            "sniff":"false",
            "ping_timeout":"5s"
         }
      },
      "xpack":{
         "watcher":{
            "execution":{
               "scroll":{
                  "size":"0",
                  "timeout":""
               },
               "default_throttle_period":"5s"
            },
            "internal":{
               "ops":{
                  "bulk":{
                     "default_timeout":""
                  },
                  "index":{
                     "default_timeout":""
                  },
                  "search":{
                     "default_timeout":""
                  }
               }
            },
            "thread_pool":{
               "queue_size":"1000",
               "size":"10"
            },
            "index":{
               "rest":{
                  "direct_access":""
               }
            },
            "history":{
               "cleaner_service":{
                  "enabled":"true"
               }
            },
            "trigger":{
               "schedule":{
                  "ticker":{
                     "tick_interval":"500ms"
                  }
               }
            },
            "enabled":"true",
            "input":{
               "search":{
                  "default_timeout":""
               }
            },
            "encrypt_sensitive_data":"false",
            "transform":{
               "search":{
                  "default_timeout":""
               }
            },
            "stop":{
               "timeout":"30s"
            },
            "watch":{
               "scroll":{
                  "size":"0"
               }
            },
            "require_manual_start":"false",
            "actions":{
               "bulk":{
                  "default_timeout":""
               },
               "index":{
                  "default_timeout":""
               }
            }
         },
         "license":{
            "self_generated":{
               "type":"basic"
            }
         },
         "logstash":{
            "enabled":"true"
         },
         "notification":{
            "hipchat":{
               "host":"",
               "port":"443",
               "default_account":""
            },
            "pagerduty":{
               "default_account":""
            },
            "email":{
               "default_account":"",
               "html":{
                  "sanitization":{
                     "allow":[
                        "body",
                        "head",
                        "_tables",
                        "_links",
                        "_blocks",
                        "_formatting",
                        "img:embedded"
                     ],
                     "disallow":[
                        
                     ],
                     "enabled":"true"
                  }
               }
            },
            "reporting":{
               "retries":"40",
               "interval":"15s"
            },
            "jira":{
               "default_account":""
            },
            "slack":{
               "default_account":""
            }
         },
         "security":{
            "dls_fls":{
               "enabled":"true"
            },
            "transport":{
               "filter":{
                  "allow":[
                     
                  ],
                  "deny":[
                     
                  ],
                  "enabled":"true"
               },
               "ssl":{
                  "enabled":"false"
               }
            },
            "enabled":"true",
            "filter":{
               "always_allow_bound_address":"true"
            },
            "encryption":{
               "algorithm":"AES/CTR/NoPadding"
            },
            "audit":{
               "outputs":[
                  "logfile"
               ],
               "index":{
                  "bulk_size":"1000",
                  "rollover":"DAILY",
                  "flush_interval":"1s",
                  "events":{
                     "emit_request_body":"false",
                     "include":[
                        "ACCESS_DENIED",
                        "ACCESS_GRANTED",
                        "ANONYMOUS_ACCESS_DENIED",
                        "AUTHENTICATION_FAILED",
                        "REALM_AUTHENTICATION_FAILED",
                        "CONNECTION_DENIED",
                        "CONNECTION_GRANTED",
                        "TAMPERED_REQUEST",
                        "RUN_AS_DENIED",
                        "RUN_AS_GRANTED",
                        "AUTHENTICATION_SUCCESS"
                     ],
                     "exclude":[
                        
                     ]
                  },
                  "queue_max_size":"10000"
               },
               "enabled":"false",
               "logfile":{
                  "events":{
                     "emit_request_body":"false",
                     "include":[
                        "ACCESS_DENIED",
                        "ACCESS_GRANTED",
                        "ANONYMOUS_ACCESS_DENIED",
                        "AUTHENTICATION_FAILED",
                        "CONNECTION_DENIED",
                        "TAMPERED_REQUEST",
                        "RUN_AS_DENIED",
                        "RUN_AS_GRANTED"
                     ],
                     "exclude":[
                        
                     ]
                  },
                  "prefix":{
                     "emit_node_host_name":"false",
                     "emit_node_name":"true",
                     "emit_node_host_address":"false"
                  }
               }
            },
            "authc":{
               "anonymous":{
                  "authz_exception":"true",
                  "roles":[
                     
                  ],
                  "username":"_anonymous"
               },
               "password_hashing":{
                  "algorithm":"bcrypt"
               },
               "run_as":{
                  "enabled":"true"
               },
               "reserved_realm":{
                  "enabled":"true"
               },
               "token":{
                  "delete":{
                     "interval":"30m",
                     "timeout":"-1"
                  },
                  "enabled":"false",
                  "thread_pool":{
                     "queue_size":"1000",
                     "size":"1"
                  },
                  "timeout":"20m"
               }
            },
            "fips_mode":{
               "enabled":"false"
            },
            "encryption_key":{
               "length":"128",
               "algorithm":"AES"
            },
            "http":{
               "filter":{
                  "allow":[
                     
                  ],
                  "deny":[
                     
                  ],
                  "enabled":"true"
               },
               "ssl":{
                  "enabled":"false"
               }
            },
            "automata":{
               "max_determinized_states":"100000"
            },
            "user":null,
            "authz":{
               "store":{
                  "roles":{
                     "field_permissions":{
                        "cache":{
                           "max_size_in_bytes":"104857600"
                        }
                     },
                     "index":{
                        "cache":{
                           "ttl":"20m",
                           "max_size":"10000"
                        }
                     },
                     "cache":{
                        "max_size":"10000"
                     }
                  }
               }
            }
         },
         "http":{
            "default_connection_timeout":"10s",
            "proxy":{
               "host":"",
               "scheme":"",
               "port":"0"
            },
            "default_read_timeout":"10s",
            "max_response_size":"10mb"
         },
         "monitoring":{
            "enabled":"true",
            "collection":{
               "cluster":{
                  "stats":{
                     "timeout":"10s"
                  }
               },
               "node":{
                  "stats":{
                     "timeout":"10s"
                  }
               },
               "indices":[
                  
               ],
               "index":{
                  "stats":{
                     "timeout":"10s"
                  },
                  "recovery":{
                     "active_only":"false",
                     "timeout":"10s"
                  }
               },
               "interval":"10s",
               "enabled":"false",
               "ml":{
                  "job":{
                     "stats":{
                        "timeout":"10s"
                     }
                  }
               }
            },
            "history":{
               "duration":"168h"
            }
         },
         "graph":{
            "enabled":"true"
         },
         "ml":{
            "utility_thread_pool":{
               "queue_size":"500",
               "size":"80"
            },
            "max_machine_memory_percent":"30",
            "max_open_jobs":"20",
            "min_disk_space_off_heap":"5gb",
            "autodetect_process":"true",
            "datafeed_thread_pool":{
               "queue_size":"200",
               "size":"20"
            },
            "node_concurrent_job_allocations":"2",
            "max_model_memory_limit":"0b",
            "enabled":"true",
            "autodetect_thread_pool":{
               "queue_size":"80",
               "size":"80"
            }
         },
         "rollup":{
            "enabled":"true",
            "task_thread_pool":{
               "queue_size":"4",
               "size":"4"
            }
         },
         "sql":{
            "enabled":"true"
         }
      },
      "rest":{
         "action":{
            "multi":{
               "allow_explicit_index":"true"
            }
         }
      },
      "cache":{
         "recycler":{
            "page":{
               "limit":{
                  "heap":"10%"
               },
               "type":"CONCURRENT",
               "weight":{
                  "longs":"1.0",
                  "ints":"1.0",
                  "bytes":"1.0",
                  "objects":"0.1"
               }
            }
         }
      },
      "reindex":{
         "remote":{
            "whitelist":[
               
            ]
         }
      },
      "max":{
         "percent":{
            "date":{
               "errors":"25"
            },
            "outoforder":{
               "errors":"25"
            }
         },
         "anomaly":{
            "records":"500"
         }
      },
      "resource":{
         "reload":{
            "enabled":"true",
            "interval":{
               "low":"60s",
               "high":"5s",
               "medium":"30s"
            }
         }
      },
      "thread_pool":{
         "force_merge":{
            "queue_size":"-1",
            "size":"1"
         },
         "fetch_shard_started":{
            "core":"1",
            "max":"4",
            "keep_alive":"5m"
         },
         "listener":{
            "queue_size":"-1",
            "size":"1"
         },
         "index":{
            "queue_size":"200",
            "size":"2"
         },
         "refresh":{
            "core":"1",
            "max":"1",
            "keep_alive":"5m"
         },
         "generic":{
            "core":"4",
            "max":"128",
            "keep_alive":"30s"
         },
         "warmer":{
            "core":"1",
            "max":"1",
            "keep_alive":"5m"
         },
         "search":{
            "max_queue_size":"1000",
            "queue_size":"1000",
            "size":"4",
            "auto_queue_frame_size":"2000",
            "target_response_time":"1s",
            "min_queue_size":"1000"
         },
         "fetch_shard_store":{
            "core":"1",
            "max":"4",
            "keep_alive":"5m"
         },
         "flush":{
            "core":"1",
            "max":"1",
            "keep_alive":"5m"
         },
         "management":{
            "core":"1",
            "max":"5",
            "keep_alive":"5m"
         },
         "analyze":{
            "queue_size":"16",
            "size":"1"
         },
         "get":{
            "queue_size":"1000",
            "size":"2"
         },
         "bulk":{
            "queue_size":"200",
            "size":"2"
         },
         "estimated_time_interval":"200ms",
         "write":{
            "queue_size":"200",
            "size":"2"
         },
         "snapshot":{
            "core":"1",
            "max":"1",
            "keep_alive":"5m"
         }
      },
      "index":{
         "codec":"default",
         "store":{
            "type":"",
            "fs":{
               "fs_lock":"native"
            },
            "preload":[
               
            ]
         }
      },
      "monitor":{
         "jvm":{
            "gc":{
               "enabled":"true",
               "overhead":{
                  "warn":"50",
                  "debug":"10",
                  "info":"25"
               },
               "refresh_interval":"1s"
            },
            "refresh_interval":"1s"
         },
         "process":{
            "refresh_interval":"1s"
         },
         "os":{
            "refresh_interval":"1s"
         },
         "fs":{
            "refresh_interval":"1s"
         }
      },
      "transport":{
         "tcp":{
            "reuse_address":"true",
            "connect_timeout":"30s",
            "compress":"false",
            "port":"9300-9400",
            "keep_alive":"true",
            "receive_buffer_size":"-1b",
            "send_buffer_size":"-1b"
         },
         "bind_host":[
            
         ],
         "ping_schedule":"-1",
         "connections_per_node":{
            "recovery":"2",
            "state":"1",
            "bulk":"3",
            "reg":"6",
            "ping":"1"
         },
         "tracer":{
            "include":[
               
            ],
            "exclude":[
               "internal:discovery/zen/fd*",
               "cluster:monitor/nodes/liveness"
            ]
         },
         "type":"security4",
         "type.default":"netty4",
         "features":{
            "x-pack":"true"
         },
         "host":[
            
         ],
         "publish_port":"-1",
         "tcp_no_delay":"true",
         "publish_host":[
            
         ],
         "netty":{
            "receive_predictor_size":"64kb",
            "receive_predictor_max":"64kb",
            "worker_count":"4",
            "receive_predictor_min":"64kb",
            "boss_count":"1"
         }
      },
      "script":{
         "allowed_contexts":[
            
         ],
         "max_compilations_rate":"75/5m",
         "cache":{
            "max_size":"100",
            "expire":"0ms"
         },
         "painless":{
            "regex":{
               "enabled":"false"
            }
         },
         "max_size_in_bytes":"65535",
         "allowed_types":[
            
         ]
      },
      "node":{
         "data":"true",
         "enable_lucene_segment_infos_trace":"false",
         "local_storage":"true",
         "max_local_storage_nodes":"1",
         "name":"node01",
         "id":{
            "seed":"0"
         },
         "attr":{
            "xpack":{
               "installed":"true"
            },
            "ml":{
               "machine_memory":"4075884544",
               "max_open_jobs":"20",
               "enabled":"true"
            }
         },
         "portsfile":"false",
         "ingest":"true",
         "master":"true",
         "ml":"true"
      },
      "indices":{
         "cache":{
            "cleanup_interval":"1m"
         },
         "mapping":{
            "dynamic_timeout":"30s"
         },
         "memory":{
            "interval":"5s",
            "max_index_buffer_size":"-1",
            "shard_inactive_time":"5m",
            "index_buffer_size":"10%",
            "min_index_buffer_size":"48mb"
         },
         "breaker":{
            "request":{
               "limit":"60%",
               "type":"memory",
               "overhead":"1.0"
            },
            "total":{
               "limit":"70%"
            },
            "accounting":{
               "limit":"100%",
               "overhead":"1.0"
            },
            "fielddata":{
               "limit":"60%",
               "type":"memory",
               "overhead":"1.03"
            },
            "type":"hierarchy"
         },
         "fielddata":{
            "cache":{
               "size":"-1b"
            }
         },
         "query":{
            "bool":{
               "max_clause_count":"1024"
            },
            "query_string":{
               "analyze_wildcard":"false",
               "allowLeadingWildcard":"true"
            }
         },
         "admin":{
            "filtered_fields":"true"
         },
         "recovery":{
            "recovery_activity_timeout":"1800000ms",
            "retry_delay_network":"5s",
            "internal_action_timeout":"15m",
            "retry_delay_state_sync":"500ms",
            "internal_action_long_timeout":"1800000ms",
            "max_bytes_per_sec":"40mb"
         },
         "requests":{
            "cache":{
               "size":"1%",
               "expire":"0ms"
            }
         },
         "store":{
            "delete":{
               "shard":{
                  "timeout":"30s"
               }
            }
         },
         "analysis":{
            "hunspell":{
               "dictionary":{
                  "ignore_case":"false",
                  "lazy":"false"
               }
            }
         },
         "queries":{
            "cache":{
               "count":"10000",
               "size":"10%",
               "all_segments":"false"
            }
         }
      },
      "plugin":{
         "mandatory":[
            
         ]
      },
      "max_running_jobs":"20",
      "discovery":{
         "type":"zen",
         "zen":{
            "commit_timeout":"30s",
            "no_master_block":"write",
            "join_retry_delay":"100ms",
            "join_retry_attempts":"3",
            "ping":{
               "unicast":{
                  "concurrent_connects":"10",
                  "hosts":[
                     
                  ],
                  "hosts.resolve_timeout":"5s"
               }
            },
            "master_election":{
               "ignore_non_master_pings":"false",
               "wait_for_joins_timeout":"30000ms"
            },
            "send_leave_request":"true",
            "ping_timeout":"3s",
            "join_timeout":"60000ms",
            "publish_diff":{
               "enable":"true"
            },
            "publish":{
               "max_pending_cluster_states":"25"
            },
            "minimum_master_nodes":"-1",
            "hosts_provider":[
               
            ],
            "publish_timeout":"30s",
            "fd":{
               "connect_on_network_disconnect":"false",
               "ping_interval":"1s",
               "ping_retries":"3",
               "register_connection_listener":"true",
               "ping_timeout":"30s"
            },
            "max_pings_from_another_master":"3"
         },
         "initial_state_timeout":"30s"
      },
      "tribe":{
         "name":"",
         "on_conflict":"any",
         "blocks":{
            "metadata":"false",
            "read":{
               "indices":[
                  
               ]
            },
            "write.indices":[
               
            ],
            "write":"false",
            "metadata.indices":[
               
            ]
         }
      },
      "http":{
         "cors":{
            "max-age":"1728000",
            "allow-origin":"",
            "allow-headers":"X-Requested-With,Content-Type,Content-Length",
            "allow-credentials":"false",
            "allow-methods":"OPTIONS,HEAD,GET,POST,PUT,DELETE",
            "enabled":"false"
         },
         "max_chunk_size":"8kb",
         "compression_level":"3",
         "max_initial_line_length":"4kb",
         "type":"security4",
         "pipelining":"true",
         "enabled":"true",
         "type.default":"netty4",
         "content_type":{
            "required":"true"
         },
         "host":[
            
         ],
         "publish_port":"-1",
         "read_timeout":"0ms",
         "max_content_length":"100mb",
         "netty":{
            "receive_predictor_size":"64kb",
            "max_composite_buffer_components":"69905",
            "receive_predictor_max":"64kb",
            "worker_count":"4",
            "receive_predictor_min":"64kb"
         },
         "tcp":{
            "reuse_address":"true",
            "keep_alive":"true",
            "receive_buffer_size":"-1b",
            "send_buffer_size":"-1b"
         },
         "bind_host":[
            
         ],
         "reset_cookies":"false",
         "max_warning_header_count":"-1",
         "max_warning_header_size":"-1b",
         "detailed_errors":{
            "enabled":"true"
         },
         "port":"9200-9300",
         "max_header_size":"8kb",
         "pipelining.max_events":"10000",
         "tcp_no_delay":"true",
         "compression":"true",
         "publish_host":[
            
         ]
      },
      "gateway":{
         "recover_after_master_nodes":"0",
         "expected_nodes":"-1",
         "recover_after_data_nodes":"-1",
         "expected_data_nodes":"-1",
         "recover_after_time":"0ms",
         "expected_master_nodes":"-1",
         "recover_after_nodes":"-1"
      }
   }
}

Answer 1

正如@leandrojmp在评论中提到的那样，您需要为我们提供更多信息以提供特定的建议，但是要获得关于提高reindex(一次性)https://opster.com/blogs/improve-elasticsearch-reindex-performance/以及正在进行的索引性能的一般提示，请遵循https://opster.com/blogs/improve-elasticsearch-indexing-rate/。
另外，您可以运行free checkup tool，它提供了可以在集群中执行的某些优化。