Wasm integration tests for local and remote wasm files

test/integration/consul-container/libs/cluster/container.go

@@ -630,6 +630,8 @@ func newContainerRequest(config Config, opts containerOpts, ports ...int) (podRe
 			"9997/tcp", // Envoy App Listener
 			"9998/tcp", // Envoy App Listener
 			"9999/tcp", // Envoy App Listener
+
+			"80/tcp", // Nginx - http port used in wasm tests
 		},
 		Hostname: opts.hostname,
 		Networks: opts.addtionalNetworks,

test/integration/consul-container/libs/service/connect.go

@@ -157,8 +157,15 @@ type SidecarConfig struct {
 // "consul connect envoy", for service name (serviceName) on the specified
 // node. The container exposes port serviceBindPort and envoy admin port
 // (19000) by mapping them onto host ports. The container's name has a prefix
-// combining datacenter and name.
-func NewConnectService(ctx context.Context, sidecarCfg SidecarConfig, serviceBindPorts []int, node cluster.Agent) (*ConnectContainer, error) {
+// combining datacenter and name. The customContainerConf parameter can be used
+// to mutate the testcontainers.ContainerRequest used to create the sidecar proxy.
+func NewConnectService(
+	ctx context.Context,
+	sidecarCfg SidecarConfig,
+	serviceBindPorts []int,
+	node cluster.Agent,
+	customContainerConf func(request testcontainers.ContainerRequest) testcontainers.ContainerRequest,
+) (*ConnectContainer, error) {
 	nodeConfig := node.GetConfig()
 	if nodeConfig.ScratchDir == "" {
 		return nil, fmt.Errorf("node ScratchDir is required")
@@ -265,6 +272,11 @@ func NewConnectService(ctx context.Context, sidecarCfg SidecarConfig, serviceBin
 	exposedPorts := make([]string, len(appPortStrs))
 	copy(exposedPorts, appPortStrs)
 	exposedPorts = append(exposedPorts, adminPortStr)
+
+	if customContainerConf != nil {
+		req = customContainerConf(req)
+	}
+
 	info, err := cluster.LaunchContainerOnNode(ctx, node, req, exposedPorts)
 	if err != nil {
 		return nil, err

test/integration/consul-container/libs/service/examples.go

@@ -127,6 +127,42 @@ func (c exampleContainer) GetStatus() (string, error) {
 	return state.Status, err
 }
 
+// NewCustomService creates a new test service from a custom testcontainers.ContainerRequest.
+func NewCustomService(ctx context.Context, name string, httpPort int, grpcPort int, node libcluster.Agent, request testcontainers.ContainerRequest) (Service, error) {
+	namePrefix := fmt.Sprintf("%s-service-example-%s", node.GetDatacenter(), name)
+	containerName := utils.RandName(namePrefix)
+
+	pod := node.GetPod()
+	if pod == nil {
+		return nil, fmt.Errorf("node Pod is required")
+	}
+
+	var (
+		httpPortStr = strconv.Itoa(httpPort)
+		grpcPortStr = strconv.Itoa(grpcPort)
+	)
+
+	request.Name = containerName
+
+	info, err := libcluster.LaunchContainerOnNode(ctx, node, request, []string{httpPortStr, grpcPortStr})
+	if err != nil {
+		return nil, err
+	}
+
+	out := &exampleContainer{
+		ctx:         ctx,
+		container:   info.Container,
+		ip:          info.IP,
+		httpPort:    info.MappedPorts[httpPortStr].Int(),
+		grpcPort:    info.MappedPorts[grpcPortStr].Int(),
+		serviceName: name,
+	}
+
+	fmt.Printf("Custom service exposed http port %d, gRPC port %d\n", out.httpPort, out.grpcPort)
+
+	return out, nil
+}
+
 func NewExampleService(ctx context.Context, name string, httpPort int, grpcPort int, node libcluster.Agent, containerArgs ...string) (Service, error) {
 	namePrefix := fmt.Sprintf("%s-service-example-%s", node.GetDatacenter(), name)
 	containerName := utils.RandName(namePrefix)

test/integration/consul-container/libs/service/helpers.go

@@ -9,6 +9,7 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/require"
+	"github.com/testcontainers/testcontainers-go"
 
 	"github.com/hashicorp/consul/api"
 	libcluster "github.com/hashicorp/consul/test/integration/consul-container/libs/cluster"
@@ -50,7 +51,7 @@ type ServiceOpts struct {
 }
 
 // createAndRegisterStaticServerAndSidecar register the services and launch static-server containers
-func createAndRegisterStaticServerAndSidecar(node libcluster.Agent, httpPort int, grpcPort int, svc *api.AgentServiceRegistration, containerArgs ...string) (Service, Service, error) {
+func createAndRegisterStaticServerAndSidecar(node libcluster.Agent, httpPort int, grpcPort int, svc *api.AgentServiceRegistration, customContainerCfg func(testcontainers.ContainerRequest) testcontainers.ContainerRequest, containerArgs ...string) (Service, Service, error) {
 	// Do some trickery to ensure that partial completion is correctly torn
 	// down, but successful execution is not.
 	var deferClean utils.ResettableDefer
@@ -77,10 +78,11 @@ func createAndRegisterStaticServerAndSidecar(node libcluster.Agent, httpPort int
 			svc.Connect.SidecarService.Proxy != nil &&
 			svc.Connect.SidecarService.Proxy.Mode == api.ProxyModeTransparent,
 	}
-	serverConnectProxy, err := NewConnectService(context.Background(), sidecarCfg, []int{svc.Port}, node) // bindPort not used
+	serverConnectProxy, err := NewConnectService(context.Background(), sidecarCfg, []int{svc.Port}, node, customContainerCfg) // bindPort not used
 	if err != nil {
 		return nil, nil, err
 	}
+
 	deferClean.Add(func() {
 		_ = serverConnectProxy.Terminate()
 	})
@@ -91,7 +93,101 @@ func createAndRegisterStaticServerAndSidecar(node libcluster.Agent, httpPort int
 	return serverService, serverConnectProxy, nil
 }
 
-func CreateAndRegisterStaticServerAndSidecar(node libcluster.Agent, serviceOpts *ServiceOpts, containerArgs ...string) (Service, Service, error) {
+// createAndRegisterCustomServiceAndSidecar creates a custom service from the given testcontainers.ContainerRequest
+// and a sidecar proxy for the service. The customContainerCfg parameter is used to mutate the
+// testcontainers.ContainerRequest for the sidecar proxy.
+func createAndRegisterCustomServiceAndSidecar(node libcluster.Agent,
+	httpPort int,
+	grpcPort int,
+	svc *api.AgentServiceRegistration,
+	request testcontainers.ContainerRequest,
+	customContainerCfg func(testcontainers.ContainerRequest) testcontainers.ContainerRequest,
+) (Service, Service, error) {
+	// Do some trickery to ensure that partial completion is correctly torn
+	// down, but successful execution is not.
+	var deferClean utils.ResettableDefer
+	defer deferClean.Execute()
+
+	if err := node.GetClient().Agent().ServiceRegister(svc); err != nil {
+		return nil, nil, err
+	}
+
+	// Create a service and proxy instance
+	serverService, err := NewCustomService(context.Background(), svc.ID, httpPort, grpcPort, node, request)
+	if err != nil {
+		return nil, nil, err
+	}
+	deferClean.Add(func() {
+		_ = serverService.Terminate()
+	})
+	sidecarCfg := SidecarConfig{
+		Name:      fmt.Sprintf("%s-sidecar", svc.ID),
+		ServiceID: svc.ID,
+		Namespace: svc.Namespace,
+		EnableTProxy: svc.Connect != nil &&
+			svc.Connect.SidecarService != nil &&
+			svc.Connect.SidecarService.Proxy != nil &&
+			svc.Connect.SidecarService.Proxy.Mode == api.ProxyModeTransparent,
+	}
+	serverConnectProxy, err := NewConnectService(context.Background(), sidecarCfg, []int{svc.Port}, node, customContainerCfg) // bindPort not used
+	if err != nil {
+		return nil, nil, err
+	}
+
+	deferClean.Add(func() {
+		_ = serverConnectProxy.Terminate()
+	})
+
+	// disable cleanup functions now that we have an object with a Terminate() function
+	deferClean.Reset()
+
+	return serverService, serverConnectProxy, nil
+}
+
+func CreateAndRegisterCustomServiceAndSidecar(node libcluster.Agent,
+	serviceOpts *ServiceOpts,
+	request testcontainers.ContainerRequest,
+	customContainerCfg func(testcontainers.ContainerRequest) testcontainers.ContainerRequest) (Service, Service, error) {
+	// Register the static-server service and sidecar first to prevent race with sidecar
+	// trying to get xDS before it's ready
+	p := serviceOpts.HTTPPort
+	agentCheck := api.AgentServiceCheck{
+		Name:     "Static Server Listening",
+		TCP:      fmt.Sprintf("127.0.0.1:%d", p),
+		Interval: "10s",
+		Status:   api.HealthPassing,
+	}
+	if serviceOpts.RegisterGRPC {
+		p = serviceOpts.GRPCPort
+		agentCheck.TCP = ""
+		agentCheck.GRPC = fmt.Sprintf("127.0.0.1:%d", p)
+	}
+	req := &api.AgentServiceRegistration{
+		Name: serviceOpts.Name,
+		ID:   serviceOpts.ID,
+		Port: p,
+		Connect: &api.AgentServiceConnect{
+			SidecarService: &api.AgentServiceRegistration{
+				Proxy: &api.AgentServiceConnectProxyConfig{
+					Mode: api.ProxyMode(serviceOpts.Connect.Proxy.Mode),
+				},
+			},
+		},
+		Namespace: serviceOpts.Namespace,
+		Meta:      serviceOpts.Meta,
+		Check:     &agentCheck,
+	}
+	return createAndRegisterCustomServiceAndSidecar(node, serviceOpts.HTTPPort, serviceOpts.GRPCPort, req, request, customContainerCfg)
+}
+
+// CreateAndRegisterStaticServerAndSidecarWithCustomContainerConfig creates an example static server and a sidecar for
+// the service. The customContainerCfg parameter is a function of testcontainers.ContainerRequest to
+// testcontainers.ContainerRequest which can be used to mutate the container request for the sidecar proxy and inject
+// custom configuration and lifecycle hooks.
+func CreateAndRegisterStaticServerAndSidecarWithCustomContainerConfig(node libcluster.Agent,
+	serviceOpts *ServiceOpts,
+	customContainerCfg func(testcontainers.ContainerRequest) testcontainers.ContainerRequest,
+	containerArgs ...string) (Service, Service, error) {
 	// Register the static-server service and sidecar first to prevent race with sidecar
 	// trying to get xDS before it's ready
 	p := serviceOpts.HTTPPort
@@ -122,7 +218,11 @@ func CreateAndRegisterStaticServerAndSidecar(node libcluster.Agent, serviceOpts
 		Check:     &agentCheck,
 		Locality:  serviceOpts.Locality,
 	}
-	return createAndRegisterStaticServerAndSidecar(node, serviceOpts.HTTPPort, serviceOpts.GRPCPort, req, containerArgs...)
+	return createAndRegisterStaticServerAndSidecar(node, serviceOpts.HTTPPort, serviceOpts.GRPCPort, req, customContainerCfg, containerArgs...)
+}
+
+func CreateAndRegisterStaticServerAndSidecar(node libcluster.Agent, serviceOpts *ServiceOpts, containerArgs ...string) (Service, Service, error) {
+	return CreateAndRegisterStaticServerAndSidecarWithCustomContainerConfig(node, serviceOpts, nil, containerArgs...)
 }
 
 func CreateAndRegisterStaticServerAndSidecarWithChecks(node libcluster.Agent, serviceOpts *ServiceOpts) (Service, Service, error) {
@@ -149,7 +249,7 @@ func CreateAndRegisterStaticServerAndSidecarWithChecks(node libcluster.Agent, se
 		Meta: serviceOpts.Meta,
 	}
 
-	return createAndRegisterStaticServerAndSidecar(node, serviceOpts.HTTPPort, serviceOpts.GRPCPort, req)
+	return createAndRegisterStaticServerAndSidecar(node, serviceOpts.HTTPPort, serviceOpts.GRPCPort, req, nil)
 }
 
 func CreateAndRegisterStaticClientSidecar(
@@ -209,7 +309,7 @@ func CreateAndRegisterStaticClientSidecar(
 		EnableTProxy: enableTProxy,
 	}
 
-	clientConnectProxy, err := NewConnectService(context.Background(), sidecarCfg, []int{libcluster.ServiceUpstreamLocalBindPort}, node)
+	clientConnectProxy, err := NewConnectService(context.Background(), sidecarCfg, []int{libcluster.ServiceUpstreamLocalBindPort}, node, nil)
 	if err != nil {
 		return nil, err
 	}

test/integration/consul-container/libs/topology/service_topology.go

@@ -7,11 +7,12 @@ import (
 	"fmt"
 	"testing"
 
+	"github.com/stretchr/testify/require"
+
 	"github.com/hashicorp/consul/api"
 	libassert "github.com/hashicorp/consul/test/integration/consul-container/libs/assert"
 	libcluster "github.com/hashicorp/consul/test/integration/consul-container/libs/cluster"
 	libservice "github.com/hashicorp/consul/test/integration/consul-container/libs/service"
-	"github.com/stretchr/testify/require"
 )
 
 // CreateServices

test/integration/consul-container/test/envoy_extensions/testdata/wasm_test_files/Dockerfile

@@ -0,0 +1,3 @@
+FROM tinygo/tinygo:sha-598cb1e4ddce53d85600a1b7724ed39eea80e119
+COPY ./build.sh /
+ENTRYPOINT ["/build.sh"]

test/integration/consul-container/test/envoy_extensions/testdata/wasm_test_files/README.md

@@ -0,0 +1,14 @@
+# Building WASM test files
+
+We have seen some issues with building the wasm test files on the build runners for the integration test. Currently,
+the theory is that there may be some differences in the clang toolchain on different runners which cause panics in
+tinygo if the job is scheduled on particular runners but not others.
+
+In order to get around this, we are just building the wasm test file and checking it into the repo.
+
+To build the wasm test file, 
+
+```bash
+~/consul/test/integration/consul-container/test/envoy_extensions/testdata/wasm_test_files
+> docker run -v ./:/wasm --rm $(docker build -q .)
+```

test/integration/consul-container/test/envoy_extensions/testdata/wasm_test_files/build.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+cd /wasm
+tinygo build -o /wasm/wasm_add_header.wasm -scheduler=none -target=wasi /wasm/wasm_add_header.go

test/integration/consul-container/test/envoy_extensions/testdata/wasm_test_files/go.mod

@@ -0,0 +1,13 @@
+module main
+
+go 1.20
+
+require (
+	github.com/tetratelabs/proxy-wasm-go-sdk v0.21.0
+	github.com/tidwall/gjson v1.14.4
+)
+
+require (
+	github.com/tidwall/match v1.1.1 // indirect
+	github.com/tidwall/pretty v1.2.0 // indirect
+)

test/integration/consul-container/test/envoy_extensions/testdata/wasm_test_files/go.sum

@@ -0,0 +1,12 @@
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk=
+github.com/tetratelabs/proxy-wasm-go-sdk v0.21.0 h1:sxuh1wxy/zz4vXwMEC+ESVpwJmej1f22eYsrJlgVn7c=
+github.com/tetratelabs/proxy-wasm-go-sdk v0.21.0/go.mod h1:jqQTUvJBI6WJ+sVCZON3A4GwmUfBDuiNnZ4kuxsvLCo=
+github.com/tidwall/gjson v1.14.4 h1:uo0p8EbA09J7RQaflQ1aBRffTR7xedD2bcIVSYxLnkM=
+github.com/tidwall/gjson v1.14.4/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
+github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
+github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
+github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs=
+github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=

test/integration/consul-container/test/envoy_extensions/testdata/wasm_test_files/nginx.conf

@@ -0,0 +1,13 @@
+server {
+    # send wasm files as download rather than render as html
+    location ~ ^.*/(?P<request_basename>[^/]+\.(wasm))$  {
+        root /www/downloads;
+
+       add_header Content-disposition 'attachment; filename="$request_basename"';
+       types {
+        application/octet-stream .wasm;
+       }
+       default_type application/octet-stream;
+    }
+}
+

test/integration/consul-container/test/envoy_extensions/testdata/wasm_test_files/wasm_add_header.go

@@ -0,0 +1,47 @@
+package main
+
+import (
+	"github.com/tetratelabs/proxy-wasm-go-sdk/proxywasm"
+	"github.com/tetratelabs/proxy-wasm-go-sdk/proxywasm/types"
+)
+
+func main() {
+	proxywasm.SetVMContext(&vmContext{})
+}
+
+type vmContext struct {
+	// Embed the default VM context here,
+	// so that we don't need to reimplement all the methods.
+	types.DefaultVMContext
+}
+
+func (*vmContext) NewPluginContext(contextID uint32) types.PluginContext {
+	return &pluginContext{}
+}
+
+type pluginContext struct {
+	// Embed the default plugin context here,
+	// so that we don't need to reimplement all the methods.
+	types.DefaultPluginContext
+}
+
+func (p *pluginContext) NewHttpContext(contextID uint32) types.HttpContext {
+	return &httpHeaders{}
+}
+
+type httpHeaders struct {
+	// Embed the default http context here,
+	// so that we don't need to reimplement all the methods.
+	types.DefaultHttpContext
+}
+
+func (ctx *httpHeaders) OnHttpResponseHeaders(int, bool) types.Action {
+	proxywasm.LogDebug("adding header: x-test:true")
+
+	err := proxywasm.AddHttpResponseHeader("x-test", "true")
+	if err != nil {
+		proxywasm.LogCriticalf("failed to add test header to response: %v", err)
+	}
+
+	return types.ActionContinue
+}