0

I have a mobile application in which we are using Nginx as a Reverse proxy which routes request to Nginx. The Nginx request in app server is passed to Node.js for processing. We are getting 504 Gateway timeout error when we are hitting more than 750 Users.we are seeing below error in Nginx Logs.

upstream timed out (110: Connection timed out) while connecting to upstream, client: LoadGenerator_IP, server: WebserverDNS, request: "GET /api/sample/profile HTTP/1.1", upstream: "https://app_server:443/api/sample/profile", host: "webserver_IP"

I tried to hit App server Nginx directly.We could able to hit more than 1000 users.But if we are using reverse proxy we are getting that error.

I tried alot of Linux system and Nginx settings.But did not overcome this issue.

NGINX.CONF

 user  nginx;
 worker_processes  auto;
 worker_rlimit_nofile 100000;
 error_log  /var/log/nginx/error.log error;
 pid        /var/run/nginx.pid;
 events {
      worker_connections 4096;
      use epoll;
      multi_accept on;
 }
 http {
      include       /etc/nginx/mime.types;
      default_type  application/octet-stream;
      log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                            '$status $body_bytes_sent "$http_referer" '
                            '"$http_user_agent" "$http_x_forwarded_for"'
                            'uct="$upstream_connect_time"'
                            'uht="$upstream_header_time"'
                            'urt="$upstream_response_time"'
                            'rt="$request_time "';
      access_log  /var/log/nginx/access.log  main;
      sendfile        on;
      #tcp_nopush     on;
      #tcp_nodelay on;
      # to boost I/O on HDD we can disable access logs
      access_log off;
      keepalive_timeout  120;
      keepalive_requests 10000;
      # allow the server to close connection on non responding client, this will free up memory
      reset_timedout_connection on;
      # request timed out -- default 60
      client_body_timeout 10;
      # if client stop responding, free up memory -- default 60
      send_timeout 2;
      #gzip on;
      # gzip_static on;
      #gzip_min_length 10240;
      #gzip_comp_level 1;
      #gzip_vary on;
      #gzip_disable msie6;
      #gzip_proxied expired no-cache no-store private auth;
      #gzip_types
           # text/html is always compressed by HttpGzipModule
      #    text/css
      #    text/javascript
      #    text/xml
      #    text/plain
      #    text/x-component
      #    application/javascript
      #    application/x-javascript
      #    application/json
      #    application/xml
      #    application/rss+xml
      #    application/atom+xml
      #    font/truetype
      #    font/opentype
      #    application/vnd.ms-fontobject
      #    image/svg+xml;
      include /etc/nginx/conf.d/Default.conf;
 }

Default.conf

 upstream app_server {
    server app_server_ip:443;
 }
 server {
    listen       80;
    listen 443 ssl backlog=32768;
    server_name someIP;
    server_tokens off;
    location ~ {
      proxy_pass https://app_server_ip;
      proxy_set_header X-Real-IP $remote_addr;
      proxy_set_header Host $host;
      #proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
      proxy_connect_timeout 75;
      proxy_read_timeout 300;
      proxy_send_timeout 300;
      proxy_http_version 1.1;
      proxy_set_header Connection "";
    }
    ssl_certificate /etc/ssl/certs/uatweb.crt;
    ssl_certificate_key /etc/ssl/certs/uatweb.key;
    ssl_protocols TLSv1.1 TLSv1.2;
    ssl_prefer_server_ciphers On;
    ssl_ciphers ECDH+AESGCM:ECDH+CHACHA20:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:RSA+AESGCM:RSA+AES:!aNULL:!MD5:!DSS;
    #charset koi8-r;
    #access_log  /var/log/nginx/host.access.log  main;
    location / {
      root   /usr/share/nginx/html;
      index  index.html index.htm;
    }
    #error_page  404              /404.html;
    # redirect server error pages to the static page /50x.html
    #
    error_page   500 502 503 504  /50x.html;
    location = /50x.html {
      root   /usr/share/nginx/html;
    }   
 }
 server {
    listen       8114 ssl;
    server_name someServerName;
    ssl_certificate /etc/ssl/certs/uatweb.crt;
    ssl_certificate_key /etc/ssl/certs/uatweb.key;
    ssl_protocols TLSv1.1 TLSv1.2;
    location ~ {
      proxy_pass https://someIP;
    }
    error_page   500 502 503 504  /50x.html;
    location = /50x.html {
      root   /usr/share/nginx/html;
    }
 }

Please let me know if any changes are required in Config Files.

1

This looks like a problem with your app server, not nginx. I'd say you're suffering from the proxy_connect_timeout - but this is set to 75 seconds which looks like a lot. You may try to increase the value but the problem is probably elsewhere.

When testing the app backend itself, what does it mean you were able to "hit more than 1000 users"? How did you test it? Are these concurrent users requesting the same resource? What response times did you get?

  • we tried proxy_connect_timeout=300 also ,got same errors.All the 1000 users are concurrent hitting same rest api just to check user scaling.There is no huge drop in response times apart from connection drops. – SuryaVeepuri May 17 at 7:46

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Not the answer you're looking for? Browse other questions tagged or ask your own question.