daemon off; worker_processes 2; pid /run/nginx.pid; worker_rlimit_nofile 523264; events { multi_accept on; worker_connections 16384; use epoll; } http { real_ip_header X-Forwarded-For; set_real_ip_from 0.0.0.0/0; real_ip_recursive on; geoip_country /etc/nginx/GeoIP.dat; geoip_city /etc/nginx/GeoLiteCity.dat; geoip_proxy_recursive on; vhost_traffic_status_zone shared:vhost_traffic_status:10m; vhost_traffic_status_filter_by_set_key $geoip_country_code country::*; # lua section to return proper error codes when custom pages are used lua_package_path '.?.lua;./etc/nginx/lua/?.lua;/etc/nginx/lua/vendor/lua-resty-http/lib/?.lua;'; init_by_lua_block { require("error_page") } sendfile on; aio threads; tcp_nopush on; tcp_nodelay on; log_subrequest on; reset_timedout_connection on; keepalive_timeout 75s; client_header_buffer_size 1k; large_client_header_buffers 4 8k; types_hash_max_size 2048; server_names_hash_max_size 512; server_names_hash_bucket_size 64; map_hash_bucket_size 64; include /etc/nginx/mime.types; default_type text/html; gzip on; gzip_comp_level 5; gzip_http_version 1.1; gzip_min_length 256; gzip_types application/atom+xml application/javascript application/x-javascript application/json application/rss+xml application/vnd.ms-fontobject application/x-font-ttf application/x-web-app-manifest+json application/xhtml+xml application/xml font/opentype image/svg+xml image/x-icon text/css text/plain text/x-component; gzip_proxied any; server_tokens on; log_format upstreaminfo '$remote_addr - ' '[$proxy_add_x_forwarded_for] - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" ' '$request_length $request_time [$proxy_upstream_name] $upstream_addr $upstream_response_length $upstream_response_time $upstream_status'; map $request_uri $loggable { default 1; } access_log /var/log/nginx/access.log upstreaminfo if=$loggable; error_log /var/log/nginx/error.log notice; resolver 192.168.0.2 valid=30s; # Retain the default nginx handling of requests without a "Connection" header map $http_upgrade $connection_upgrade { default upgrade; '' close; } # trust http_x_forwarded_proto headers correctly indicate ssl offloading map $http_x_forwarded_proto $pass_access_scheme { default $http_x_forwarded_proto; '' $scheme; } map $http_x_forwarded_port $pass_server_port { default $http_x_forwarded_port; '' $server_port; } # map port 442 to 443 for header X-Forwarded-Port map $pass_server_port $pass_port { 442 443; default $pass_server_port; } # Map a response error watching the header Content-Type map $http_accept $httpAccept { default html; application/json json; application/xml xml; text/plain text; } map $httpAccept $httpReturnType { default text/html; json application/json; xml application/xml; text text/plain; } server_name_in_redirect off; port_in_redirect off; ssl_protocols TLSv1 TLSv1.1 TLSv1.2; # turn on session caching to drastically improve performance ssl_session_cache builtin:1000 shared:SSL:10m; ssl_session_timeout 10m; # allow configuring ssl session tickets ssl_session_tickets on; # slightly reduce the time-to-first-byte ssl_buffer_size 4k; # allow configuring custom ssl ciphers ssl_ciphers 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:AES:CAMELLIA:DES-CBC3-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA'; ssl_prefer_server_ciphers on; # In case of errors try the next upstream server before returning an error proxy_next_upstream error timeout invalid_header http_502 http_503 http_504; upstream default-kube-state-metrics-8080 { least_conn; server 172.16.39.4:8080 max_fails=0 fail_timeout=0; } upstream upstream-default-backend { least_conn; server 172.16.34.5:8080 max_fails=0 fail_timeout=0; } server { server_name _; listen [::]:80 ipv6only=off default_server reuseport backlog=511; listen 442 default_server reuseport backlog=511 ssl http2; # PEM sha: d6d43c441973c52ceecaf13c1c6f083275339cc4 ssl_certificate /ingress-controller/ssl/system-snake-oil-certificate.pem; ssl_certificate_key /ingress-controller/ssl/system-snake-oil-certificate.pem; more_set_headers "Strict-Transport-Security: max-age=15724800; includeSubDomains; preload"; vhost_traffic_status_filter_by_set_key $geoip_country_code country::$server_name; location / { set $proxy_upstream_name "upstream-default-backend"; port_in_redirect off; client_max_body_size "1m"; proxy_set_header Host $host; # Pass Real IP proxy_set_header X-Real-IP $remote_addr; # Allow websocket connections proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection $connection_upgrade; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Host $host; proxy_set_header X-Forwarded-Port $pass_port; proxy_set_header X-Forwarded-Proto $pass_access_scheme; # mitigate HTTPoxy Vulnerability # https://www.nginx.com/blog/mitigating-the-httpoxy-vulnerability-with-nginx/ proxy_set_header Proxy ""; # Custom headers proxy_connect_timeout 5s; proxy_send_timeout 60s; proxy_read_timeout 60s; proxy_redirect off; proxy_buffering off; proxy_buffer_size "4k"; proxy_http_version 1.1; proxy_pass http://upstream-default-backend; } # health checks in cloud providers require the use of port 80 location /healthz { access_log off; return 200; } # this is required to avoid error if nginx is being monitored # with an external software (like sysdig) location /nginx_status { allow 127.0.0.1; allow ::1; deny all; access_log off; stub_status on; } } server { server_name foo.com; listen [::]:80; vhost_traffic_status_filter_by_set_key $geoip_country_code country::$server_name; location / { set $proxy_upstream_name "default-kube-state-metrics-8080"; port_in_redirect off; client_max_body_size "1m"; proxy_set_header Host $host; # Pass Real IP proxy_set_header X-Real-IP $remote_addr; # Allow websocket connections proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection $connection_upgrade; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Host $host; proxy_set_header X-Forwarded-Port $pass_port; proxy_set_header X-Forwarded-Proto $pass_access_scheme; # mitigate HTTPoxy Vulnerability # https://www.nginx.com/blog/mitigating-the-httpoxy-vulnerability-with-nginx/ proxy_set_header Proxy ""; # Custom headers proxy_connect_timeout 5s; proxy_send_timeout 60s; proxy_read_timeout 60s; proxy_redirect off; proxy_buffering off; proxy_buffer_size "4k"; proxy_http_version 1.1; proxy_pass http://default-kube-state-metrics-8080; } } # default server, used for NGINX healthcheck and access to nginx stats server { # Use the port 18080 (random value just to avoid known ports) as default port for nginx. # Changing this value requires a change in: # https://github.com/kubernetes/contrib/blob/master/ingress/controllers/nginx/nginx/command.go#L104 listen [::]:18080 ipv6only=off default_server reuseport backlog=511; location /healthz { access_log off; return 200; } location /nginx_status { vhost_traffic_status_display; vhost_traffic_status_display_format html; } # this location is used to extract nginx metrics # using prometheus. # TODO: enable extraction for vts module. location /internal_nginx_status { allow 127.0.0.1; allow ::1; deny all; access_log off; stub_status on; } location / { set $proxy_upstream_name "upstream-default-backend"; proxy_pass http://upstream-default-backend; } } # default server for services without endpoints server { listen 8181; set $proxy_upstream_name "-"; location / { return 503; } } } stream { # map FQDN that requires SSL passthrough map $ssl_preread_server_name $stream_upstream { # send SSL traffic to this nginx in a different port default nginx-ssl-backend; } log_format log_stream '$remote_addr [$time_local] $protocol [$ssl_preread_server_name] [$stream_upstream] $status $bytes_sent $bytes_received $session_time'; access_log /var/log/nginx/access.log log_stream; error_log /var/log/nginx/error.log; # configure default backend for SSL upstream nginx-ssl-backend { server 127.0.0.1:442; } server { listen [::]:443 ipv6only=off; proxy_pass $stream_upstream; ssl_preread on; } # TCP services # UDP services }