linux - 最大化从haproxy到jetty应用服务器的tcp连接速率

Question

我使用 haproxy 前端和 nginx 来跨 jetty 应用程序服务器进行负载平衡，所有服务器都运行 ubuntu-14-04-x64。

Nginx 和 haproxy 共享一个 0.5G 1CPU 虚拟机。每个jetty服务器都运行在一个4G 2CPU虚拟机上。

Nginx 前端 haproxy，配置如下:

user myc;
worker_processes 1;
pid /run/nginx.pid;

events {
    worker_connections 65536;
    multi_accept on;
    use epoll;
}

http {
    include mime.types;
    types_hash_max_size 2048;
    default_type application/octet-stream;
    proxy_cache_path /home/myc/cache levels=1:2 keys_zone=one:10m inactive=7d;
    proxy_http_version 1.1;
    proxy_set_header Connection "";

    upstream loadbalancer {
        server unix:/tmp/haproxy.sock;
        keepalive 8192;
    }

    server {
        listen 80 backlog=16384;
        server_name example.org;
        access_log /var/log/nginx/access.log;
        error_log /var/log/nginx/error.log;

        gzip on;
        gzip_types text/plain application/javascript application/x-javascript text/javascript text/xml text/css application/json;
        gzip_vary on;
        gzip_min_length 10240;

        open_file_cache max=2000 inactive=20s;
        open_file_cache_valid 60s;
        open_file_cache_min_uses 5;
        open_file_cache_errors off;

        sendfile on;

        proxy_set_header X-Real-IP $remote_addr;
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;

        error_page 502 /502.html;
        location /502.html {
            root /home/myc/html;
        }

        proxy_cache one;

        location / {
            proxy_pass http://loadbalancer/;
            proxy_connect_timeout 60s;
            proxy_read_timeout 60s;

            add_header Cache-Control private; # Don't cache any of this publicly, because JSESSIONID will be cached, too

            add_header X-Proxy-Cache $upstream_cache_status;
        }

        location /api {
            proxy_pass http://loadbalancer/api;
            proxy_connect_timeout 60s;
            proxy_read_timeout 60s;

            proxy_ignore_headers Expires;
            proxy_hide_header Expires;
            proxy_ignore_headers Set-Cookie; # Make sure JSESSIONID is not cached with static content
            proxy_hide_header Set-Cookie;

            proxy_cache_valid 200 10s;

            add_header X-Proxy-Cache $upstream_cache_status;
        }

        location ~* ^.*\.(css|js|gif|jpe?g|png|ico)$ {
            proxy_pass http://loadbalancer$uri;
            proxy_connect_timeout 60s;
            proxy_read_timeout 60s;

            expires 1h;
            access_log off;
            log_not_found off;

            proxy_ignore_headers Expires;
            proxy_hide_header Expires;
            proxy_ignore_headers Set-Cookie; # Make sure JSESSIONID is not cached with static content
            proxy_hide_header Set-Cookie;
            add_header Cache-Control public;

            proxy_cache_valid 200 404 10s;

            add_header X-Proxy-Cache $upstream_cache_status;
        }
    }
}

HAProxy 配置如下:

global
    daemon
    maxconn 262144
    maxconnrate 128
    chroot /home/jail/

defaults
    mode http
    timeout connect 10s
    timeout client 120s
    timeout server 120s
    errorfile 502 /etc/haproxy/errors/502.http
    errorfile 503 /etc/haproxy/errors/503.http
    compression algo identity
    option http-keep-alive

frontend http-in
    bind /tmp/haproxy.sock user myc
    default_backend makeyourcase

backend makeyourcase
    option httpchk GET /
    appsession JSESSIONID len 52 timeout 1h
    server ny2-b-app01 <redacted>:8080 maxconn 1024 check inter 60000 fall 1 rise 1
    server ny2-b-app02 <redacted>:8080 maxconn 1024 check inter 60000 fall 1 rise 1

TCP 堆栈使用 nginx/haproxy 盒和两个 jetty 盒上的这些参数进行调整:

net.core.netdev_max_backlog = 65535
net.core.somaxconn = 65535
net.core.rmem_max = 16777216
net.core.wmem_max = 16777216
net.ipv4.tcp_rfc1337 = 1
net.ipv4.ip_local_port_range = 1024 65535
net.ipv4.tcp_fin_timeout = 30
net.ipv4.tcp_keepalive_intvl = 30
net.ipv4.tcp_max_syn_backlog = 65535
net.ipv4.tcp_moderate_rcvbuf = 1
net.ipv4.tcp_syncookies = 0

net.ipv4.tcp_rmem = 4096 87380 16777216
net.ipv4.tcp_wmem = 4096 16384 16777216

net.ipv4.tcp_tw_recycle = 1
net.ipv4.tcp_tw_reuse = 1
net.ipv4.tcp_timestamps = 1
net.ipv4.tcp_window_scaling = 1

我正在使用 The Grinder 来创建负载。一旦负载达到某个点，SYN_SENT 就会在 haproxy 端堆积。如果我将 maconnrate 降低到 96，连接速率就会降低到足以使 SYN_SENT 永远不会堆积的程度。后端的 CPU 负载永远不会超过 20%，平均负载也不会超过 1。我在 nginx、haproxy 和 tcp 堆栈中尝试了各种调整选项。

似乎可能有一些 tcp parm 可以让更多的连接/秒进入 jetty 服务器，从而让 haproxy 完全加载 jetty 服务器。

OTOH，根据 http://cbonte.github.io/haproxy-dconv/configuration-1.5.html ，“默认情况下，HAProxy 在持久性方面以保持事件模式运行然而，haproxy 和应用程序服务器之间的连接并未保持打开状态。我是否指定了某些 haproxy 选项来阻止这种情况发生？或者是否有某些选项可以实现这一点？

Answer 1

虽然我花了一些时间调整设置(如下所示)，但最终给出更合理结果(大约 14K 同时连接)的更改是将托管 nginx 和 haproxy 的虚拟机大小从 0.5G 1CPU 增加到 2G 2CPU。

这是当前的 nginx 配置:

user myc;
worker_processes 1;
pid /run/nginx.pid;
worker_rlimit_nofile 131072;

events {
    worker_connections 65536;
    multi_accept on;
    use epoll;
}

http {
    include mime.types;
    types_hash_max_size 2048;
    default_type application/octet-stream;
    proxy_cache_path /home/myc/cache levels=1:2 keys_zone=one:10m inactive=7d;

    upstream loadbalancer {
        server unix:/tmp/haproxy.sock;
        keepalive 32768;
    }

    upstream haproxy_admin {
        server 127.0.0.1:8081;
    }

    server {
        listen 80 backlog=16384;
        server_name makeyourcase.org;
        access_log /var/log/nginx/access.log;
        error_log /var/log/nginx/error.log;

        gzip on;
        gzip_types text/plain application/javascript application/x-javascript text/javascript text/xml text/css application/json;
        gzip_vary on;
        gzip_min_length 10240;

        open_file_cache max=2000 inactive=20s;
        open_file_cache_valid 60s;
        open_file_cache_min_uses 5;
        open_file_cache_errors off;

        sendfile on;

        proxy_http_version 1.1;
        proxy_set_header Connection "";
        proxy_set_header X-Real-IP $remote_addr;
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;

        error_page 502 /502.html;
        location /502.html {
            root /home/myc/html;
        }

        proxy_cache one;

        location ~ ^/api/auth/local/.*$ {
            proxy_pass http://loadbalancer$uri;
        }

        location /admin {
            proxy_pass http://loadbalancer/admin;
        }

        location /stats {
            proxy_pass http://haproxy_admin/;
        }

        location / {
            proxy_pass http://loadbalancer/;
            proxy_connect_timeout 60s;
            proxy_read_timeout 60s;

            add_header Cache-Control private; # Don't cache any of this publicly, because JSESSIONID will be cached, too

            add_header X-Proxy-Cache $upstream_cache_status;
        }

        location /api {
            proxy_pass http://loadbalancer/api;
            proxy_connect_timeout 60s;
            proxy_read_timeout 60s;

            proxy_ignore_headers Expires;
            proxy_hide_header Expires;
            proxy_ignore_headers Set-Cookie; # Make sure JSESSIONID is not cached with static content
            proxy_hide_header Set-Cookie;

            proxy_cache_valid 200 10s;

            add_header X-Proxy-Cache $upstream_cache_status;
        }

        location ~* ^.*\.(css|js|gif|jpe?g|png|ico)$ {
            proxy_pass http://loadbalancer$uri;
            proxy_connect_timeout 60s;
            proxy_read_timeout 60s;

            expires 1h;
            access_log off;
            log_not_found off;

            proxy_ignore_headers Expires;
            proxy_hide_header Expires;
            proxy_ignore_headers Set-Cookie; # Make sure JSESSIONID is not cached with static content
            proxy_hide_header Set-Cookie;
            add_header Cache-Control public;

            proxy_cache_valid 200 404 10s;

            add_header X-Proxy-Cache $upstream_cache_status;
        }
    }
}

和 HAProxy:

global
    daemon
    maxconn 32768
    maxconnrate 256
    chroot /home/jail/

defaults
    mode http
    timeout connect 10s
    timeout client 120s
    timeout server 120s
    errorfile 502 /etc/haproxy/errors/502.http
    errorfile 503 /etc/haproxy/errors/503.http
    compression algo identity

frontend http-in
    bind /tmp/haproxy.sock user myc
    default_backend makeyourcase

backend makeyourcase
    option httpchk GET /
    option http-server-close
    appsession JSESSIONID len 52 timeout 1h
    server ny2-b-app01 <redacted>:8080 maxconn 1024 check inter 60000 fall 1 rise 1
    server ny2-b-app02 <redacted>:8080 maxconn 1024 check inter 60000 fall 1 rise 1

listen stats localhost:8081
    mode   http
    stats  enable
    stats  hide-version
    stats  realm MYC
    stats  uri /

和 sysctl:

fs.file-max = 1048576

net.core.netdev_max_backlog = 250000
net.core.somaxconn = 262144
net.core.rmem_max = 134217728
net.core.wmem_max = 134217728

net.ipv4.tcp_rfc1337 = 1
net.ipv4.ip_local_port_range = 1024 65535
net.ipv4.tcp_fin_timeout = 10
net.ipv4.tcp_keepalive_intvl = 30
net.ipv4.tcp_max_syn_backlog = 262144
net.ipv4.tcp_moderate_rcvbuf = 1
net.ipv4.tcp_syncookies = 1
net.ipv4.tcp_rmem = 4096 87380 67108864
net.ipv4.tcp_wmem = 4096 65536 67108864
net.ipv4.tcp_low_latency = 1

net.ipv4.tcp_tw_recycle = 0
net.ipv4.tcp_tw_reuse = 1
net.ipv4.tcp_timestamps = 1
net.ipv4.tcp_window_scaling = 1

vm.swappiness = 0