0

We are using AWS to host Magento. We have 3 c4.4xlarge servers in different availability zones, in the same region, as backend web servers running HHVM and Nginx. Each connects to a multi-AZ RDS instance for its database.

In front of the backend servers, we have tried a few options to balance the load over the servers: AWS Application Load Balancer, AWS Classic Load Balancer, HAProxy and Nginx. We asked AWS to pre-warm the Classic Load Balancer as the usual traffic was almost zero.

We had intended to put HAProxy on a t2.medium server but also tried it on a c4.large server to be sure that isn't a bottleneck. The HAProxy, sysctl and Nginx configurations are below. We intended to use the internal IPs but have also tried using Elastic IPs.

HAProxy configuration

global
    log /dev/log    local0
    log /dev/log    local1 notice
    chroot /var/lib/haproxy
    stats socket /run/haproxy/admin.sock mode 660 level admin
    stats timeout 30s
    user haproxy
    group haproxy
    daemon
    maxconn 100000
    tune.bufsize 16384 # Tried without
    tune.maxrewrite 1024 # Tried without
    tune.maxaccept -1 # Tried without
    nbproc 4 # Tried without
    ca-base /etc/ssl/certs
    crt-base /etc/ssl/private

    ssl-default-bind-ciphers ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+3DES:!aNULL:!MD5:!DSS
    ssl-default-bind-options no-sslv3

defaults
    log global
    mode    http  # Tried tcp
    balance roundrobin  # Tried leastconn
    option  httplog
    option abortonclose
    option redispatch # Added
    option dontlognull
    option dontlog-normal # Added
    timeout connect 5000
    timeout client  50000
    timeout server  50000
    errorfile 400 /etc/haproxy/errors/400.http
    errorfile 403 /etc/haproxy/errors/403.http
    errorfile 408 /etc/haproxy/errors/408.http
    errorfile 500 /etc/haproxy/errors/500.http
    errorfile 502 /etc/haproxy/errors/502.http
    errorfile 503 /etc/haproxy/errors/503.http
    errorfile 504 /etc/haproxy/errors/504.http
    stats enable
    stats uri /haproxy?stats
    stats realm redacted
    stats auth redacted
    option http-server-close 
    option forwardfor
    option redispatch

frontend www-http
    bind 0.0.0.0:80
    reqadd X-Forwarded-Proto:\ http
    default_backend www-backend
    acl admin_tag path_beg /index.php/redacted
    acl admin_tag path_beg /redacted
    use_backend www-admin-backend if admin_tag

frontend www-https
    bind 0.0.0.0:443 ssl crt /etc/ssl/private/redacted.pem
    reqadd X-Forwarded-Proto:\ https
    default_backend www-backend
    acl admin_tag path_beg /index.php/redacted
    acl admin_tag path_beg /redacted
    use_backend www-admin-backend if admin_tag

backend www-backend
    redirect scheme https if !{ ssl_fc }
    cookie SERVERID insert indirect nocache 
    server redacted redacted:80 check cookie web1
    server redacted redacted:80 check cookie web2   
    server redacted redacted:80 check cookie web3

backend www-admin-backend
    redirect scheme https if !{ ssl_fc }
    cookie SERVERADMINID insert indirect nocache
    server redacted redacted:80 check cookie web1

Nginx configuration

upstream backend {
    least_conn;
    keepalive 8;
    server redacted max_fails=3 fail_timeout=10s;
    server redacted max_fails=3 fail_timeout=10s;
    server redacted max_fails=3 fail_timeout=10s;
}

server {
    listen 443 default_server;
    server_name _;

    ssl on;
    ssl_certificate    /redacted/cert_chain.crt;
    ssl_certificate_key    /redacted/private.key;
    ssl_session_cache shared:SSL:20m;
    ssl_session_timeout 10m;
    ssl_prefer_server_ciphers       on;
    ssl_protocols                   TLSv1 TLSv1.1 TLSv1.2;
    ssl_ciphers                     ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+3DES:!aNULL:!MD5:!DSS;

    location / {
        proxy_pass http://backend;
        proxy_set_header Host $host;
        proxy_set_header X-Real-IP $remote_addr;
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        proxy_set_header X-Forwarded-Proto $scheme;
    }
}

Sysctl config

fs.file-max = 10000000
fs.nr_open = 10000000
net.ipv4.tcp_mem = 786432 1697152 1945728
net.ipv4.tcp_rmem = 4096 4096 16777216
net.ipv4.tcp_wmem = 4096 4096 16777216
net.ipv4.netfilter.ip_conntrack_max = 50000
net.ipv4.ip_local_port_range = 1024 65000

In every configuration (different load balancer instances, different load balancer applications/services, different configuration) we can only get 50 requests per second. The CPU usage is about 30% on 1/3 of the cores, so about 10% overall.

We have verified this using Apachebench on an external server (starting with 120 concurrent connections and then 160 and 200 - response times get slower and throughput is almost the same). We have also verified this using blitz.io where we start at 200 users and climb slowly to 1600 - the throughput is the same, CPU usage is the same and response times slow down.

We have also tried load balancing the backend servers directly and we are able to handle about 50 requests/second on one backend server CPU usage obviously being higher.

There is nothing in any logs (Nginx, haproxy, kernel on any server). I have also spoken to AWS support and they have told me that there is no network rate limiting.

Am I missing something? It seems like there is a bottleneck on connections from the load balancer to the backend servers but I don't know what we haven't tested.

  • What version of magento do you use? – Orphans Nov 4 '16 at 12:46
  • You could set up a static backend behind nginx, serving, for example, a txt file and make a test to prove that it's really the balancer that is a bottleneck. – drookie Nov 4 '16 at 13:12
  • 1
    I don't see where you've mentioned how the back-end servers perform if you don't use anything in front of them at all. Have you confirmed that you don't simply have an issue with the back end generally? – Michael - sqlbot Nov 4 '16 at 15:08
  • "50 requests/second on one backend server CPU usage obviously being higher." as in you could serve 50tps on all 3 servers concurrently? Or just 50tps on a single server in isolation? It sounds like you have some shared resource in your application (DB?) that's bottlenecking you regardless of how many application servers you have. – thexacre Nov 6 '16 at 4:32

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Browse other questions tagged or ask your own question.