update nginx template for TLS passthrough (#2166)

Update nginx template for TLS passthrough Problem: nginx configuration templates didn't support TLS passthrough Solution: I added a template setup fro stream servers
nginxinc · Jul 8, 2024 · 1bc8be4 · 1bc8be4
1 parent 04cfbc3
commit 1bc8be4
Show file tree

Hide file tree

Showing 31 changed files with 918 additions and 73 deletions.
diff --git a/charts/nginx-gateway-fabric/templates/deployment.yaml b/charts/nginx-gateway-fabric/templates/deployment.yaml
@@ -118,6 +118,8 @@ spec:
         volumeMounts:
         - name: nginx-conf
           mountPath: /etc/nginx/conf.d
+        - name: nginx-stream-conf
+          mountPath: /etc/nginx/stream-conf.d
         - name: module-includes
           mountPath: /etc/nginx/module-includes
         - name: nginx-secrets
@@ -153,6 +155,8 @@ spec:
         volumeMounts:
         - name: nginx-conf
           mountPath: /etc/nginx/conf.d
+        - name: nginx-stream-conf
+          mountPath: /etc/nginx/stream-conf.d
         - name: module-includes
           mountPath: /etc/nginx/module-includes
         - name: nginx-secrets
@@ -187,6 +191,8 @@ spec:
       volumes:
       - name: nginx-conf
         emptyDir: {}
+      - name: nginx-stream-conf
+        emptyDir: {}
       - name: module-includes
         emptyDir: {}
       - name: nginx-secrets

diff --git a/config/tests/static-deployment.yaml b/config/tests/static-deployment.yaml
@@ -70,6 +70,8 @@ spec:
         volumeMounts:
         - name: nginx-conf
           mountPath: /etc/nginx/conf.d
+        - name: nginx-stream-conf
+          mountPath: /etc/nginx/stream-conf.d
         - name: module-includes
           mountPath: /etc/nginx/module-includes
         - name: nginx-secrets
@@ -98,6 +100,8 @@ spec:
         volumeMounts:
         - name: nginx-conf
           mountPath: /etc/nginx/conf.d
+        - name: nginx-stream-conf
+          mountPath: /etc/nginx/stream-conf.d
         - name: module-includes
           mountPath: /etc/nginx/module-includes
         - name: nginx-secrets
@@ -117,6 +121,8 @@ spec:
       volumes:
       - name: nginx-conf
         emptyDir: {}
+      - name: nginx-stream-conf
+        emptyDir: {}
       - name: module-includes
         emptyDir: {}
       - name: nginx-secrets

diff --git a/deploy/manifests/nginx-gateway-experimental.yaml b/deploy/manifests/nginx-gateway-experimental.yaml
@@ -226,6 +226,8 @@ spec:
         volumeMounts:
         - name: nginx-conf
           mountPath: /etc/nginx/conf.d
+        - name: nginx-stream-conf
+          mountPath: /etc/nginx/stream-conf.d
         - name: module-includes
           mountPath: /etc/nginx/module-includes
         - name: nginx-secrets
@@ -254,6 +256,8 @@ spec:
         volumeMounts:
         - name: nginx-conf
           mountPath: /etc/nginx/conf.d
+        - name: nginx-stream-conf
+          mountPath: /etc/nginx/stream-conf.d
         - name: module-includes
           mountPath: /etc/nginx/module-includes
         - name: nginx-secrets
@@ -273,6 +277,8 @@ spec:
       volumes:
       - name: nginx-conf
         emptyDir: {}
+      - name: nginx-stream-conf
+        emptyDir: {}
       - name: module-includes
         emptyDir: {}
       - name: nginx-secrets

diff --git a/deploy/manifests/nginx-gateway.yaml b/deploy/manifests/nginx-gateway.yaml
@@ -222,6 +222,8 @@ spec:
         volumeMounts:
         - name: nginx-conf
           mountPath: /etc/nginx/conf.d
+        - name: nginx-stream-conf
+          mountPath: /etc/nginx/stream-conf.d
         - name: module-includes
           mountPath: /etc/nginx/module-includes
         - name: nginx-secrets
@@ -250,6 +252,8 @@ spec:
         volumeMounts:
         - name: nginx-conf
           mountPath: /etc/nginx/conf.d
+        - name: nginx-stream-conf
+          mountPath: /etc/nginx/stream-conf.d
         - name: module-includes
           mountPath: /etc/nginx/module-includes
         - name: nginx-secrets
@@ -269,6 +273,8 @@ spec:
       volumes:
       - name: nginx-conf
         emptyDir: {}
+      - name: nginx-stream-conf
+        emptyDir: {}
       - name: module-includes
         emptyDir: {}
       - name: nginx-secrets

diff --git a/deploy/manifests/nginx-plus-gateway-experimental.yaml b/deploy/manifests/nginx-plus-gateway-experimental.yaml
@@ -233,6 +233,8 @@ spec:
         volumeMounts:
         - name: nginx-conf
           mountPath: /etc/nginx/conf.d
+        - name: nginx-stream-conf
+          mountPath: /etc/nginx/stream-conf.d
         - name: module-includes
           mountPath: /etc/nginx/module-includes
         - name: nginx-secrets
@@ -261,6 +263,8 @@ spec:
         volumeMounts:
         - name: nginx-conf
           mountPath: /etc/nginx/conf.d
+        - name: nginx-stream-conf
+          mountPath: /etc/nginx/stream-conf.d
         - name: module-includes
           mountPath: /etc/nginx/module-includes
         - name: nginx-secrets
@@ -280,6 +284,8 @@ spec:
       volumes:
       - name: nginx-conf
         emptyDir: {}
+      - name: nginx-stream-conf
+        emptyDir: {}
       - name: module-includes
         emptyDir: {}
       - name: nginx-secrets

diff --git a/deploy/manifests/nginx-plus-gateway.yaml b/deploy/manifests/nginx-plus-gateway.yaml
@@ -229,6 +229,8 @@ spec:
         volumeMounts:
         - name: nginx-conf
           mountPath: /etc/nginx/conf.d
+        - name: nginx-stream-conf
+          mountPath: /etc/nginx/stream-conf.d
         - name: module-includes
           mountPath: /etc/nginx/module-includes
         - name: nginx-secrets
@@ -257,6 +259,8 @@ spec:
         volumeMounts:
         - name: nginx-conf
           mountPath: /etc/nginx/conf.d
+        - name: nginx-stream-conf
+          mountPath: /etc/nginx/stream-conf.d
         - name: module-includes
           mountPath: /etc/nginx/module-includes
         - name: nginx-secrets
@@ -276,6 +280,8 @@ spec:
       volumes:
       - name: nginx-conf
         emptyDir: {}
+      - name: nginx-stream-conf
+        emptyDir: {}
       - name: module-includes
         emptyDir: {}
       - name: nginx-secrets

diff --git a/internal/mode/static/nginx/conf/nginx-plus.conf b/internal/mode/static/nginx/conf/nginx-plus.conf
@@ -52,6 +52,21 @@ http {
   }
 }
 
+stream {
+	variables_hash_bucket_size 512;
+    variables_hash_max_size 1024;
+
+    map_hash_max_size 2048;
+    map_hash_bucket_size 256;
+
+    log_format stream-main '$remote_addr [$time_local] '
+                              '$protocol $status $bytes_sent $bytes_received '
+                              '$session_time "$ssl_preread_server_name"';
+    access_log /dev/stdout stream-main;
+
+	include /etc/nginx/stream-conf.d/*.conf;
+}
+
 mgmt {
   usage_report interval=0s;
 }
diff --git a/internal/mode/static/nginx/conf/nginx.conf b/internal/mode/static/nginx/conf/nginx.conf
@@ -36,3 +36,18 @@ http {
     }
   }
 }
+
+stream {
+	variables_hash_bucket_size 512;
+    variables_hash_max_size 1024;
+
+    map_hash_max_size 2048;
+    map_hash_bucket_size 256;
+
+    log_format stream-main '$remote_addr [$time_local] '
+                              '$protocol $status $bytes_sent $bytes_received '
+                              '$session_time "$ssl_preread_server_name"';
+    access_log /dev/stdout stream-main;
+
+	include /etc/nginx/stream-conf.d/*.conf;
+}
diff --git a/internal/mode/static/nginx/config/base_http_config_template.go b/internal/mode/static/nginx/config/base_http_config_template.go
@@ -2,4 +2,20 @@ package config
 
 const baseHTTPTemplateText = `
 {{- if .HTTP2 }}http2 on;{{ end }}
+
+# Set $gw_api_compliant_host variable to the value of $http_host unless $http_host is empty, then set it to the value
+# of $host. We prefer $http_host because it contains the original value of the host header, which is required by the
+# Gateway API. However, in an HTTP/1.0 request, it's possible that $http_host can be empty. In this case, we will use
+# the value of $host. See http://nginx.org/en/docs/http/ngx_http_core_module.html#var_host.
+map $http_host $gw_api_compliant_host {
+    '' $host;
+    default $http_host;
+}
+
+# Set $connection_header variable to upgrade when the $http_upgrade header is set, otherwise, set it to close. This
+# allows support for websocket connections. See https://nginx.org/en/docs/http/websocket.html.
+map $http_upgrade $connection_upgrade {
+    default upgrade;
+    '' close;
+}
 `
diff --git a/internal/mode/static/nginx/config/base_http_config_test.go b/internal/mode/static/nginx/config/base_http_config_test.go
@@ -47,5 +47,7 @@ func TestExecuteBaseHttp(t *testing.T) {
 		res := executeBaseHTTPConfig(test.conf)
 		g.Expect(res).To(HaveLen(1))
 		g.Expect(test.expCount).To(Equal(strings.Count(string(res[0].data), expSubStr)))
+		g.Expect(strings.Count(string(res[0].data), "map $http_host $gw_api_compliant_host {")).To(Equal(1))
+		g.Expect(strings.Count(string(res[0].data), "map $http_upgrade $connection_upgrade {")).To(Equal(1))
 	}
 }
diff --git a/internal/mode/static/nginx/config/generator.go b/internal/mode/static/nginx/config/generator.go
@@ -17,6 +17,9 @@ const (
 	// httpFolder is the folder where NGINX HTTP configuration files are stored.
 	httpFolder = configFolder + "/conf.d"
 
+	// streamFolder is the folder where NGINX Stream configuration files are stored.
+	streamFolder = configFolder + "/stream-conf.d"
+
 	// modulesIncludesFolder is the folder where the included "load_module" file is stored.
 	modulesIncludesFolder = configFolder + "/module-includes"
 
@@ -29,6 +32,9 @@ const (
 	// httpConfigFile is the path to the configuration file with HTTP configuration.
 	httpConfigFile = httpFolder + "/http.conf"
 
+	// streamConfigFile is the path to the configuration file with Stream configuration.
+	streamConfigFile = streamFolder + "/stream.conf"
+
 	// configVersionFile is the path to the config version configuration file.
 	configVersionFile = httpFolder + "/config-version.conf"
 
@@ -40,7 +46,7 @@ const (
 )
 
 // ConfigFolders is a list of folders where NGINX configuration files are stored.
-var ConfigFolders = []string{httpFolder, secretsFolder, includesFolder, modulesIncludesFolder}
+var ConfigFolders = []string{httpFolder, secretsFolder, includesFolder, modulesIncludesFolder, streamFolder}
 
 // Generator generates NGINX configuration files.
 // This interface is used for testing purposes only.
@@ -157,6 +163,9 @@ func (g GeneratorImpl) getExecuteFuncs() []executeFunc {
 		executeSplitClients,
 		executeMaps,
 		executeTelemetry,
+		executeStreamServers,
+		g.executeStreamUpstreams,
+		executeStreamMaps,
 	}
 }
 

diff --git a/internal/mode/static/nginx/config/generator_test.go b/internal/mode/static/nginx/config/generator_test.go
@@ -47,12 +47,25 @@ func TestGenerate(t *testing.T) {
 				Port: 443,
 			},
 		},
+		TLSPassthroughServers: []dataplane.Layer4VirtualServer{
+			{
+				Hostname:     "app.example.com",
+				Port:         443,
+				UpstreamName: "stream_up",
+			},
+		},
 		Upstreams: []dataplane.Upstream{
 			{
 				Name:      "up",
 				Endpoints: nil,
 			},
 		},
+		StreamUpstreams: []dataplane.Upstream{
+			{
+				Name:      "stream_up",
+				Endpoints: nil,
+			},
+		},
 		BackendGroups: []dataplane.BackendGroup{bg},
 		SSLKeyPairs: map[dataplane.SSLKeyPairID]dataplane.SSLKeyPair{
 			"test-keypair": {
@@ -81,7 +94,7 @@ func TestGenerate(t *testing.T) {
 
 	files := generator.Generate(conf)
 
-	g.Expect(files).To(HaveLen(6))
+	g.Expect(files).To(HaveLen(7))
 	arrange := func(i, j int) bool {
 		return files[i].Path < files[j].Path
 	}
@@ -98,7 +111,7 @@ func TestGenerate(t *testing.T) {
 	// Note: this only verifies that Generate() returns a byte array with upstream, server, and split_client blocks.
 	// It does not test the correctness of those blocks. That functionality is covered by other tests in this package.
 	g.Expect(httpCfg).To(ContainSubstring("listen 80"))
-	g.Expect(httpCfg).To(ContainSubstring("listen 443"))
+	g.Expect(httpCfg).To(ContainSubstring("listen unix:/var/run/nginx/https443.sock"))
 	g.Expect(httpCfg).To(ContainSubstring("upstream"))
 	g.Expect(httpCfg).To(ContainSubstring("split_clients"))
 
@@ -127,4 +140,12 @@ func TestGenerate(t *testing.T) {
 		Path:    "/etc/nginx/secrets/test-keypair.pem",
 		Content: []byte("test-cert\ntest-key"),
 	}))
+
+	g.Expect(files[6].Path).To(Equal("/etc/nginx/stream-conf.d/stream.conf"))
+	g.Expect(files[6].Type).To(Equal(file.TypeRegular))
+	streamCfg := string(files[6].Content)
+	g.Expect(streamCfg).To(ContainSubstring("listen unix:/var/run/nginx/app.example.com-443.sock"))
+	g.Expect(streamCfg).To(ContainSubstring("listen 443"))
+	g.Expect(streamCfg).To(ContainSubstring("app.example.com unix:/var/run/nginx/app.example.com-443.sock"))
+	g.Expect(streamCfg).To(ContainSubstring("example.com unix:/var/run/nginx/https443.sock"))
 }
diff --git a/internal/mode/static/nginx/config/http/config.go b/internal/mode/static/nginx/config/http/config.go
@@ -4,9 +4,9 @@ package http
 type Server struct {
 	SSL           *SSL
 	ServerName    string
+	Listen        string
 	Locations     []Location
 	Includes      []string
-	Port          int32
 	IsDefaultHTTP bool
 	IsDefaultSSL  bool
 	GRPC          bool
@@ -88,19 +88,6 @@ type SplitClientDistribution struct {
 	Value   string
 }
 
-// Map defines an NGINX map.
-type Map struct {
-	Source     string
-	Variable   string
-	Parameters []MapParameter
-}
-
-// Parameter defines a Value and Result pair in a Map.
-type MapParameter struct {
-	Value  string
-	Result string
-}
-
 // ProxySSLVerify holds the proxied HTTPS server verification configuration.
 type ProxySSLVerify struct {
 	TrustedCertificate string