0

I am on a IaaS Kubernetes k8s on a Virtual Machine.

I have setup a nginx and some other apps which are working well. (I can reach my apps)

I am using nginx ingress controller.

When I try to reach jenkins outside of the node, I get an empty reply but the ingress logs show a 403

When I curl jenkins inside of the node I can reach it without any problem

When I restart the Jenkins container, I can access the Please wait while Jenkins is getting ready to work ... page but after it is ready, I get a 403 again.

Any idea why this is happening?

Ingress ressource:

apiVersion: extensions/v1beta1
kind: Ingress
metadata:
  annotations:
    kubernetes.io/ingress.class: nginx
  labels:
    project: cicd
  managedFields:
  - apiVersion: networking.k8s.io/v1beta1
    fieldsType: FieldsV1
    fieldsV1:
      f:status:
        f:loadBalancer:
          f:ingress: {}
    manager: nginx-ingress-controller
    operation: Update
    time: "2020-04-02T17:11:25Z"
  - apiVersion: extensions/v1beta1
    fieldsType: FieldsV1
    fieldsV1:
      f:metadata:
        f:annotations:
          .: {}
          f:kubectl.kubernetes.io/last-applied-configuration: {}
          f:kubernetes.io/ingress.class: {}
        f:labels:
          .: {}
          f:project: {}
      f:spec:
        f:rules: {}
    manager: kubectl
    operation: Update
    time: "2020-04-02T17:12:56Z"
  name: jenkins
  namespace: jenkins
spec:
  rules:
  - host: jenkins.com
    http:
      paths:
      - backend:
          serviceName: jenkins
          servicePort: 8080
        path: /
        pathType: ImplementationSpecific

nginx.conf

## start server jenkins.com
    server {
        server_name jenkins.com ;

        listen 80  ;
        listen 443  ssl http2 ;

        set $proxy_upstream_name "-";

        ssl_certificate_by_lua_block {
            certificate.call()
        }

        location / {

            set $namespace      "jenkins";
            set $ingress_name   "jenkins";
            set $service_name   "jenkins";
            set $service_port   "8080";
            set $location_path  "/";

            rewrite_by_lua_block {
                lua_ingress.rewrite({
                    force_ssl_redirect = false,
                    ssl_redirect = true,
                    force_no_ssl_redirect = false,
                    use_port_in_redirects = false,
                })
                balancer.rewrite()
                plugins.run()
            }

            header_filter_by_lua_block {

                plugins.run()
            }
            body_filter_by_lua_block {

            }

            log_by_lua_block {

                balancer.log()

                monitor.call()

                plugins.run()
            }

            port_in_redirect off;

            set $balancer_ewma_score -1;
            set $proxy_upstream_name "jenkins-jenkins-8080";
            set $proxy_host          $proxy_upstream_name;
            set $pass_access_scheme  $scheme;
            set $pass_server_port    $server_port;
            set $best_http_host      $http_host;
            set $pass_port           $pass_server_port;

            set $proxy_alternative_upstream_name "";

            client_max_body_size                    10m;

            proxy_set_header Host                   $best_http_host;

            # Pass the extracted client certificate to the backend

            # Allow websocket connections
            proxy_set_header                        Upgrade           $http_upgrade;

            proxy_set_header                        Connection        $connection_upgrade;

            proxy_set_header X-Request-ID           $req_id;
            proxy_set_header X-Real-IP              $remote_addr;

            proxy_set_header X-Forwarded-For        $remote_addr;

            proxy_set_header X-Forwarded-Host       $best_http_host;
            proxy_set_header X-Forwarded-Port       $pass_port;
            proxy_set_header X-Forwarded-Proto      $pass_access_scheme;

            proxy_set_header X-Scheme               $pass_access_scheme;

            # Pass the original X-Forwarded-For
            proxy_set_header X-Original-Forwarded-For $http_x_forwarded_for;

            # mitigate HTTPoxy Vulnerability
            # https://www.nginx.com/blog/mitigating-the-httpoxy-vulnerability-with-nginx/
            proxy_set_header Proxy                  "";

            # Custom headers to proxied server

            proxy_connect_timeout                   5s;
            proxy_send_timeout                      86400s;
            proxy_read_timeout                      86400s;

            proxy_buffering                         off;
            proxy_buffer_size                       4k;
            proxy_buffers                           4 4k;

            proxy_max_temp_file_size                1024m;

            proxy_request_buffering                 on;
            proxy_http_version                      1.1;

            proxy_cookie_domain                     off;
            proxy_cookie_path                       off;

            # In case of errors try the next upstream server before returning an error
            proxy_next_upstream                     error timeout;
            proxy_next_upstream_timeout             0;
            proxy_next_upstream_tries               3;

            proxy_pass http://upstream_balancer;

            proxy_redirect                          off;

        }

    }

curl -v command

* Connected to jenkins.com (xxx.xxx.xxx.xxx) port 80 (#0)
> GET / HTTP/1.1
> User-Agent: curl/7.29.0
> Host: jenkins.com
> Accept: */*
> 
< HTTP/1.1 403 Forbidden
< Server: openresty/1.15.8.2
< Date: Sun, 05 Apr 2020 18:09:23 GMT
< Content-Type: text/html;charset=utf-8
< Content-Length: 793
< Connection: keep-alive
< Vary: Accept-Encoding
< X-Content-Type-Options: nosniff
< Set-Cookie: JSESSIONID.fe94dd2e=node01swwo52ouan0zf0m57265cwjo109.node0; Path=/; HttpOnly
< Expires: Thu, 01 Jan 1970 00:00:00 GMT
< X-Hudson: 1.395
< X-Jenkins: 2.222.1
< X-Jenkins-Session: efb06340
< X-Hudson-CLI-Port: 50000
< X-Jenkins-CLI-Port: 50000
< X-Jenkins-CLI2-Port: 50000
< X-You-Are-Authenticated-As: anonymous
< X-You-Are-In-Group-Disabled: JENKINS-39402: use -Dhudson.security.AccessDeniedException2.REPORT_GROUP_HEADERS=true or use /whoAmI to diagnose
< X-Required-Permission: hudson.model.Hudson.Read
< X-Permission-Implied-By: hudson.security.Permission.GenericRead
< X-Permission-Implied-By: hudson.model.Hudson.Administer
< 
<html><head><meta http-equiv='refresh' content='1;url=/login?from=%2F'/><script>window.location.replace('/login?from=%2F');</script></head><body style='background-color:white; color:white;'>


Authentication required
<!--
You are authenticated as: anonymous
Groups that you are in:

Permission you need to have (but didn't): hudson.model.Hudson.Read
 ... which is implied by: hudson.security.Permission.GenericRead
 ... which is implied by: hudson.model.Hudson.Administer

I am editing /etc/hosts in order to point jenkins.com to my node (On the machine trying to access jenkins and getting 403)

  • Could you include your Ingress object here and update the question? – Wytrzymały Wiktor Apr 3 '20 at 8:12
  • Sorry about that, It has been added – Sartigan Apr 3 '20 at 15:56
  • Hi Sartigan, welcome to S.F.; please post the actual curl -v experiences for both the good and bad outcomes, because we can't guess what's wrong with your setup – mdaniel Apr 4 '20 at 17:18
  • Sorry again about that, it has been added – Sartigan Apr 5 '20 at 18:11
  • @Sartigan Have you maybe tried this? – Wytrzymały Wiktor Apr 7 '20 at 7:06
1

This is a community wiki answer.

There are three most common solutions for this issue:

  1. If you don't mind reducing security of your Jenkins installation, you can uncheck "Prevent Cross Site Request Forgery exploits" in jenkins.com/configureSecurity section enter image description here

    1. A safer one: "Enable proxy compatibility" in the "Configure Global Security" options enter image description here

    2. If you access problem is due to crumb token issue, you can follow this short guide to resolve it.

Please let me know if that helped.

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Not the answer you're looking for? Browse other questions tagged or ask your own question.