docker - HAProxy Docker Container无法使用localhost在我的Cockroach DB Docker容器之间进行负载平衡

Question

我有一个包含我的HAProxy容器和3个Cockroach DB(CKDB)容器的docker compose文件。全部在网络模式主机上运行。

我可以使用localhost及其关联的端口加入CKDB节点，但是我的HAProxy容器无法使用“ localhost ”访问CKDB节点。如果我使用主机的IP地址，则可以使用。

这是对HAProxy的限制还是我的配置有问题？

这是HAProxy配置文件:

global
  maxconn 4096

defaults
    mode                tcp
    timeout connect     10s
    timeout client      1m
    timeout server      1m
    option              clitcpka

frontend ckdb_nodes
    bind *:26257
    option tcplog
    option httpchk GET /health?ready=1
    mode tcp
    use_backend ckdb_nodes

backend ckdb_nodes
    mode tcp
    balance roundrobin
    option ssl-hello-chk
    server cockroach1 localhost:26254 check port 8081
    server cockroach2 localhost:26255 check port 8082
    server cockroach3 localhost:26256 check port 8083

frontend ckdb_admin_ui
    bind *:8080
    option tcplog
    mode tcp
    use_backend ckdb_admin_ui

backend ckdb_admin_ui
    mode tcp
    balance roundrobin
    option ssl-hello-chk
    server cockroach1 localhost:8081 check
    server cockroach2 localhost:8082 check
    server cockroach3 localhost:8083 check

这是docker compose文件:

services:
  ckdb-e1-haproxy:
    image: ${IMAGE_STAGING_E1_HAPROXY}:${BUILD_VERSION_STATIC}
    ports:
      - ${COCKROACH_E1_HAPROXY_PORT}:${COCKROACH_E1_HAPROXY_PORT}
      - ${COCKROACH_ADMIN_UI_E1_HAPROXY_PORT}:${COCKROACH_ADMIN_UI_E1_HAPROXY_PORT}
    network_mode: "host"
    container_name: ${PROJECT}_ckdb-e1-haproxy
    depends_on:
      - ckdb-e1-node1
      - ckdb-e1-node2
      - ckdb-e1-node3

  ckdb-e1-node1:
    image: cockroachdb/cockroach:${CKDB_VERSION}
    command: start --certs-dir=${CKDB_CONTAINER_HOST_DIR} --port=${STAGING_E1_NODE1_CKDB_APP_PORT} --http-port=${STAGING_E1_NODE1_CKDB_ADMIN_UI_PORT}
    ports:
      - ${STAGING_E1_NODE1_CKDB_APP_PORT}:${STAGING_E1_NODE1_CKDB_APP_PORT}
      - ${STAGING_E1_NODE1_CKDB_ADMIN_UI_PORT}:${STAGING_E1_NODE1_CKDB_ADMIN_UI_PORT}
    volumes:
      - type: bind
        source: ${CKDB_STAGING_E1_CERTS_DIR}
        target: ${CONTAINER_CKDB_CERTS_PATH}
      - type: volume
        source: staging-app-db-data-e1-node1
        target: ${CONTAINER_CKDB_DATA_PATH}
    container_name: ${PROJECT}_ckdb-e1-node1
    network_mode: "host"

  ckdb-e1-node2:
    image: cockroachdb/cockroach:${CKDB_VERSION}
    command: start --certs-dir=${CKDB_CONTAINER_HOST_DIR} --port=${STAGING_E1_NODE2_CKDB_APP_PORT} --http-port=${STAGING_E1_NODE2_CKDB_ADMIN_UI_PORT} --join=localhost:${STAGING_E1_NODE1_CKDB_APP_PORT}
    ports:
      - ${STAGING_E1_NODE2_CKDB_APP_PORT}:${STAGING_E1_NODE2_CKDB_APP_PORT}
      - ${STAGING_E1_NODE2_CKDB_ADMIN_UI_PORT}:${STAGING_E1_NODE2_CKDB_ADMIN_UI_PORT}
    volumes:
      - type: bind
        source: ${CKDB_STAGING_E1_CERTS_DIR}
        target: ${CONTAINER_CKDB_CERTS_PATH}
      - type: volume
        source: staging-app-db-data-e1-node2
        target: ${CONTAINER_CKDB_DATA_PATH}
    container_name: ${PROJECT}_ckdb-e1-node2
    depends_on:
      - ckdb-e1-node1
    network_mode: "host"

  ckdb-e1-node3:
    image: cockroachdb/cockroach:${CKDB_VERSION}
    command: start --certs-dir=${CKDB_CONTAINER_HOST_DIR} --port=${STAGING_E1_NODE3_CKDB_APP_PORT} --http-port=${STAGING_E1_NODE3_CKDB_ADMIN_UI_PORT} --join=localhost:${STAGING_E1_NODE1_CKDB_APP_PORT}
    ports:
      - ${STAGING_E1_NODE3_CKDB_APP_PORT}:${STAGING_E1_NODE3_CKDB_APP_PORT}
      - ${STAGING_E1_NODE3_CKDB_ADMIN_UI_PORT}:${STAGING_E1_NODE3_CKDB_ADMIN_UI_PORT}
    volumes:
      - type: bind
        source: ${CKDB_STAGING_E1_CERTS_DIR}
        target: ${CONTAINER_CKDB_CERTS_PATH}
      - type: volume
        source: staging-app-db-data-e1-node3
        target: ${CONTAINER_CKDB_DATA_PATH}
    container_name: ${PROJECT}_ckdb-e1-node3
    depends_on:
      - ckdb-e1-node1
      - ckdb-e1-node2
    network_mode: "host"

Answer 1

这是我的完整配置供引用:

docker-compose.yml

version: '3.7'

networks:
  roachnet:
    driver: bridge

services:
  # CockroachDB Load Balancer
  crdb-lb:
    hostname: crdb-lb
    image: haproxy:2.0.7
    volumes: ['./crdb-lb.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro']
    networks: ["roachnet"]
    ports:
      - "26257:26257"
      - "8080:8080"
      - "8081:8081"
    links:
      - roach1
      - roach2
      - roach3

  # CockroachDB default topology - 3 nodes
  roach1:
    # see https://hub.docker.com/r/cockroachdb/cockroach/tags
    image: cockroachdb/cockroach:v19.1.5
    hostname: roach1
    networks:
      - roachnet
    volumes: ["./cockroach-data/roach1:/cockroach/cockroach-data"]
    command: "start --insecure --cache=.25 --max-sql-memory=.25"
    ulimits:
      nproc: 65535
      nofile:
        soft: 65535
        hard: 65535
  roach2:
    # see https://hub.docker.com/r/cockroachdb/cockroach/tags
    image: cockroachdb/cockroach:v19.1.5
    hostname: roach2
    networks: ["roachnet"]
    volumes: ["./cockroach-data/roach2:/cockroach/cockroach-data"]
    command: "start --insecure --join=roach1 --cache=.25 --max-sql-memory=.25"
    depends_on: ["roach1"]
    ulimits:
      nproc: 65535
      nofile:
        soft: 65535
        hard: 65535
  roach3:
    # see https://hub.docker.com/r/cockroachdb/cockroach/tags
    image: cockroachdb/cockroach:v19.1.5
    hostname: roach3
    networks: ["roachnet"]
    volumes: ["./cockroach-data/roach3:/cockroach/cockroach-data"]
    command: "start --insecure --join=roach1 --cache=.25 --max-sql-memory=.25"
    depends_on: ["roach1"]
    ulimits:
      nproc: 65535
      nofile:
        soft: 65535
        hard: 65535

我通过Docker映射的 crdb-lb.cfg 组成了 haproxy.cfg :

global
    maxconn 4096
    nbproc 1
    nbthread 4

defaults
    mode                tcp
    # Timeout values should be configured for your specific use.
    # See: https://cbonte.github.io/haproxy-dconv/1.8/configuration.html#4-timeout%20connect
    timeout connect     10s
    timeout client      30m
    timeout server      30m
    # TCP keep-alive on client side. Server already enables them.
    option              clitcpka

listen cockroach-db
    bind :26257
    mode tcp
    balance roundrobin
    option httpchk GET /health?ready=1
    server roach1 roach1:26257 check port 8080
    server roach2 roach2:26257 check port 8080
    server roach3 roach3:26257 check port 8080

listen cockroach-ui
    bind :8080
    mode tcp
    balance roundrobin
    option httpchk GET /health
    server roach1 roach1:8080 check port 8080
    server roach2 roach2:8080 check port 8080
    server roach3 roach3:8080 check port 8080

listen stats
    bind :8081
    mode http
    stats enable
    stats hide-version
    stats realm Haproxy\ Statistics
    stats uri /

当然，在连接之前，请确保您创建了要连接的用户。您可以通过执行 docker-compose exec roach1 ./cockroach sql --insecure轻松获得shell，然后可以从那里创建数据库和用户。

DROP DATABASE IF EXISTS mydb;
CREATE DATABASE mydb;
CREATE USER IF NOT EXISTS myuser;
GRANT ALL ON DATABASE mydb TO myuser;

然后在我的Golang应用程序中，我只需在端口 localhost上按 26257进行连接，如下所示:

db, err := sql.Open("postgres", "postgresql://myuser@localhost:26257/mydb?sslmode=disable")
if err != nil {
    log.Fatal("error connecting to the database: ", err)
}