Redirect non-https traffic when using Varnish and nginx

Question

I am using Stunnel, Varnish, and nginx with a nodejs app, and I'm having trouble redirecting non-https pages (http://manager.domain.com) to their respective https page (https://manager.domain.com). I just get stuck in a redirect loop because all the traffic passes through varnish first. The main reason for this set up is to use sockets with ssl.

Stunnel listens on port 443. It terminates SSL connections and passes traffic to Varnish on port 80. Varnish listens on port 80 and splits other traffic as required between Nginx on 81 and Node.js on port 3000. Nginx listens on port 81. It serves static files and other non-Node.js pages.

Here is the nginx conf file

upstream nodejs {
    server 127.0.0.1:3000;
    server 127.0.0.1:3000;
}

server {
    listen 81;
    server_name www.domain.net;
    rewrite ^(.*) http://domain.com$1 permanent;

}

server {
    listen 81;
    server_name manager.domain.com;
    rewrite ^(.*) https://manager.domain.com$1 permanent;

}

server {
    listen 81;
    server_name domain.com manager.domain.com help.domain.com;

    access_log /srv/www/domain.com/logs/access.log timed;
    error_log /srv/www/domain.com/logs/error.log info;
    root /srv/www/domain.com/public;

    #everything else
    location / {
        proxy_set_header X-Real-IP $remote_addr;
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        proxy_set_header Host $http_host;
        proxy_set_header X-NginX-Proxy true;

        proxy_pass http://nodejs/;
        proxy_redirect off;


    }



    gzip on;
    gzip_comp_level 6;
    gzip_proxied any;
    gzip_min_length  1000;
    gzip_disable     "MSIE [1-6]\."
    gzip_types text/plain text/css application/json application/x-javascript text/xml application/xml application/xml+rss text/javascript;
}

Any here is the varnish config file

backend default {
    .host = "127.0.0.1";
    .port = "81";
    .connect_timeout = 5s;
    .first_byte_timeout = 30s;
    .between_bytes_timeout = 60s;
    .max_connections = 800;
}

backend nodejs {
    .host = "127.0.0.1";
    .port = "3000";
    .connect_timeout = 1s;
    .first_byte_timeout = 2s;
    .between_bytes_timeout = 60s;
    .max_connections = 800;
}

sub vcl_recv {
    set req.backend = default;
    set req.grace = 120s;

    #set the correct IP so my backends don’t log all requests as coming from Varnish
    if (req.restarts == 0) {
        if (req.http.x-forwarded-for) {
            set req.http.X-Forwarded-For =
            req.http.X-Forwarded-For + ", " + client.ip;
        } else {
            set req.http.X-Forwarded-For = client.ip;
        }
    }

    #remove port, so that hostname is normalized
    set req.http.Host = regsub(req.http.Host, ":[0-9]+", "");

    #Removed: code for purging

    #part of Varnish’s default config
    if (req.request != "GET" &&
        req.request != "HEAD" &&
        req.request != "PUT" &&
        req.request != "POST" &&
        req.request != "TRACE" &&
        req.request != "OPTIONS" &&
        req.request != "DELETE") {
        /* Non-RFC2616 or CONNECT which is weird. */
        return (pipe);
    }
    if (req.request != "GET" && req.request != "HEAD") {
        return (pass);
    }

    #pipe websocket connections directly to Node.js
    if (req.http.Upgrade ~ "(?i)websocket") {
        set req.backend = nodejs;
        return (pipe);
    }

    #do not cache large static files
    if (req.url ~ "\.(avi|flv|mp(e?)g|mp4|mp3|gz|tgz|bz2|tbz|ogg)$") {
        return(pass);
    }

    #general URL manipulation and cookie removal
    #lines 60-109 from https://github.com/mattiasgeniar/varnish-3.0-configuration-templates/blob/d86d6c1d7d3d0ddaf92019dd5ef5ce66c9e53700/default.vcl

    if(req.http.Host ~"^(www\.)?domain.com"){
        #Removed: Redirect for URL normalization using error 701
        # Requests made to this path, relate to websockets - pass does not seem to work (even for XHR polling)
        if (req.url ~ "^/socket.io/") {
            set req.backend = nodejs;
            return (pipe);
        }
    #My other PHP/MySQL sites get included here, each in its own block
    }

    # part of Varnish’s default config
    if (req.http.Authorization || req.http.Cookie) {
        /* Not cacheable by default */
        return (pass);
    }
    return (lookup);
}

sub vcl_pipe {
    #we need to copy the upgrade header
    if (req.http.upgrade) {
        set bereq.http.upgrade = req.http.upgrade;
    }
    #closing the connection is necessary for some applications – I haven’t had any issues with websockets keeping the line below uncommented
    #set bereq.http.Connection = "close";
     return (pipe);
}






sub vcl_pass {
     return (pass);
}

 sub vcl_hash {
     hash_data(req.url);
     if (req.http.host) {
         hash_data(req.http.host);
     } else {
         hash_data(server.ip);
     }
     return (hash);
 }

 sub vcl_hit {
     return (deliver);
 }

 sub vcl_miss {
     return (fetch);
 }

 sub vcl_fetch {
     if (beresp.ttl <= 0s ||
         beresp.http.Set-Cookie ||
         beresp.http.Vary == "*") {
        /*
         * Mark as "Hit-For-Pass" for the next 2 minutes
         */
        set beresp.ttl = 120 s;
        return (hit_for_pass);
     }
     return (deliver);
 }

 sub vcl_deliver {
     return (deliver);
 }

 sub vcl_error {
     set obj.http.Content-Type = "text/html; charset=utf-8";
     set obj.http.Retry-After = "5";
     synthetic {"
 <?xml version="1.0" encoding="utf-8"?>
 <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
  "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
 <html>
   <head>
     <title>"} + obj.status + " " + obj.response + {"</title>
   </head>
   <body>
     <h1>Error "} + obj.status + " " + obj.response + {"</h1>
     <p>"} + obj.response + {"</p>
     <h3>Guru Meditation:</h3>
     <p>XID: "} + req.xid + {"</p>
     <hr>
     <p>Varnish cache server</p>
   </body>
 </html>
 "};
     return (deliver);
 }

 sub vcl_init {
    return (ok);
}

 sub vcl_fini {
    return (ok);
 }

Answer 1

You have two server blocks on port 81 with manager.domain.com configured as a host header - nginx has no way to know whether a request went through stunnel or not, so the first one wins and the redirect always occurs.

I'd recommend either having Varnish do the redirecting based on whether the request came from stunnel or not (check the request's client.ip - 127.0.0.1 means it's from stunnel), or have Varnish mark the requests from stunnel with a header so that nginx can decide how to handle them.