[APP-257] Add integration tests to check CNAB bundle API

docker.Makefile

@@ -72,7 +72,7 @@ test-unit: build_dev_image ## run unit tests
 	docker run --rm -v $(CURDIR)/_build/test-results:/test-results $(DEV_IMAGE_NAME) make TEST_RESULTS_PREFIX=$(TEST_RESULTS_PREFIX) test-unit
 
 test-e2e: build_dev_image invocation-image ## run end-to-end tests
-	docker run -v /var/run:/var/run:ro --rm --network="host" $(DEV_IMAGE_NAME) make TEST_RESULTS_PREFIX=$(TEST_RESULTS_PREFIX) bin/$(BIN_NAME) test-e2e
+	docker run -v /var/run:/var/run:ro --rm --network="host" $(DEV_IMAGE_NAME) make TEST_RESULTS_PREFIX=$(TEST_RESULTS_PREFIX) bin/$(BIN_NAME) E2E_TESTS=$(E2E_TESTS) test-e2e
 
 COV_LABEL := com.docker.app.cov-run=$(TAG)
 coverage-run: build_dev_image ## run tests with coverage

e2e/compatibility_test.go

@@ -0,0 +1,185 @@
+package e2e
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strings"
+	"testing"
+	"time"
+
+	"gotest.tools/assert"
+	"k8s.io/apimachinery/pkg/util/wait"
+
+	"gotest.tools/fs"
+)
+
+const (
+	pollTimeout = 30 * time.Second
+)
+
+func loadAndTagImage(info dindSwarmAndRegistryInfo, tmpDir *fs.Dir, tag string, url string) error {
+
+	err := downloadImageTarball(tmpDir.Join("image.tar"), url)
+	if err != nil {
+		return err
+	}
+
+	combined := info.dockerCmd("load", "-q", "-i", tmpDir.Join("image.tar"))
+
+	digest := ""
+	for _, line := range strings.Split(combined, "\n") {
+		if strings.Contains(line, "sha256:") {
+			digest = strings.Split(line, "sha256:")[1]
+		}
+	}
+	if digest == "" {
+		return errors.New("Image digest not found in docker load's stdout")
+	}
+
+	digest = strings.Trim(digest, " \r\n")
+	info.dockerCmd("tag", digest, tag)
+
+	return nil
+}
+
+func downloadImageTarball(filepath string, url string) error {
+	client := http.Client{Timeout: time.Minute * 1}
+	res, err := client.Get(url)
+	if err != nil {
+		return err
+	}
+	defer res.Body.Close()
+
+	// Create the file
+	out, err := os.Create(filepath)
+	if err != nil {
+		return err
+	}
+	defer out.Close()
+	// Write the body to file
+	_, err = io.Copy(out, res.Body)
+	return err
+}
+
+func TestBackwardsCompatibilityV1(t *testing.T) {
+	runWithDindSwarmAndRegistry(t, func(info dindSwarmAndRegistryInfo) {
+		appName := "app-e2e"
+
+		data, err := ioutil.ReadFile(filepath.Join("testdata", "compatibility", "bundle-v0.9.0.json"))
+		assert.NilError(t, err)
+		// update bundle
+		bundleDir := filepath.Join(info.configDir, "app", "bundles", "docker.io", "library", "app-e2e", "_tags", "v0.9.0")
+		assert.NilError(t, os.MkdirAll(bundleDir, os.FileMode(0777)))
+		assert.NilError(t, ioutil.WriteFile(filepath.Join(bundleDir, "bundle.json"), data, os.FileMode(0644)))
+
+		// load images build with an old Docker App version
+		assert.NilError(t, loadAndTagImage(info, info.tmpDir, "app-e2e:0.1.0-invoc", "https://github.com/docker/app-e2e/raw/master/images/v0.9.0/app-e2e-invoc.tar"))
+		assert.NilError(t, loadAndTagImage(info, info.tmpDir, "app-e2e/backend", "https://github.com/docker/app-e2e/raw/master/images/v0.9.0/backend.tar"))
+		assert.NilError(t, loadAndTagImage(info, info.tmpDir, "app-e2e/frontend", "https://github.com/docker/app-e2e/raw/master/images/v0.9.0/frontend.tar"))
+
+		// list images
+		output := info.dockerCmd("app", "image", "ls")
+		checkContains(t, output, []string{appName})
+		// inspect bundle
+		output = info.dockerCmd("app", "image", "inspect", "app-e2e:v0.9.0", "--pretty")
+		checkContains(t, output,
+			[]string{
+				`name:\s+app-e2e`,
+				`backend\s+1\s+app-e2e/backend`,
+				`frontend\s+1\s+8080\s+app-e2e/frontend`,
+				`ports.frontend\s+8080`,
+			})
+
+		// render bundle
+		output = info.dockerCmd("app", "image", "render", "app-e2e:v0.9.0")
+		checkContains(t, output,
+			[]string{
+				"image: app-e2e/frontend",
+				"image: app-e2e/backend",
+				"published: 8080",
+				"target: 80",
+			})
+
+		// Install app
+		output = info.dockerCmd("app", "run", "app-e2e:v0.9.0", "--name", appName)
+		checkContains(t, output,
+			[]string{
+				fmt.Sprintf("Creating service %s_backend", appName),
+				fmt.Sprintf("Creating service %s_frontend", appName),
+				fmt.Sprintf("Creating network %s_default", appName),
+			})
+
+		// Status check -- poll app list
+		checkStatus := func(lastAction string) {
+			err = wait.Poll(2*time.Second, pollTimeout, func() (bool, error) {
+				output = info.dockerCmd("app", "ls")
+				expectedLines := []string{
+					`RUNNING APP\s+APP NAME\s+SERVICES\s+LAST ACTION\s+RESULT\s+CREATED\s+MODIFIED\s+REFERENCE`,
+					fmt.Sprintf(`%s\s+%s \(0.1.0\)\s+2/2\s+%s\s+success\s+.+second[s]?\sago\s+.+second[s]?\sago\s+`, appName, appName, lastAction),
+				}
+				matches := true
+				for _, expected := range expectedLines {
+					exp := regexp.MustCompile(expected)
+					matches = matches && exp.MatchString(output)
+				}
+				return matches, nil
+			})
+			assert.NilError(t, err)
+		}
+
+		queryService := func(port string) {
+			err = wait.Poll(2*time.Second, pollTimeout, func() (bool, error) {
+				// Check the frontend service responds
+				url := `http://localhost:` + port
+				output = info.execCmd("/usr/bin/wget", "-O", "-", url)
+				return strings.Contains(output, `Hi there, I love Docker!`), nil
+			})
+			assert.NilError(t, err)
+		}
+
+		// Check status on install
+		checkStatus("install")
+
+		// query deployed service
+		queryService("8080")
+
+		// Inspect app
+		output = info.dockerCmd("app", "inspect", appName, "--pretty")
+		checkContains(t, output,
+			[]string{
+				"Running App:",
+				fmt.Sprintf("Name: %s", appName),
+				"Result: success",
+				`ports.frontend: "8080"`,
+			})
+
+		// Update the application, changing the port
+		output = info.dockerCmd("app", "update", appName, "--set", "ports.frontend=8081")
+		checkContains(t, output,
+			[]string{
+				fmt.Sprintf("Updating service %s_backend", appName),
+				fmt.Sprintf("Updating service %s_frontend", appName),
+			})
+
+		// check status on upgrade
+		checkStatus("upgrade")
+
+		// Check the frontend service responds on the new port
+		queryService("8081")
+
+		// Uninstall the application
+		output = info.dockerCmd("app", "rm", appName)
+		checkContains(t, output,
+			[]string{
+				fmt.Sprintf("Removing service %s_backend", appName),
+				fmt.Sprintf("Removing service %s_frontend", appName),
+				fmt.Sprintf("Removing network %s_default", appName),
+			})
+	})
+}

e2e/helper_test.go

@@ -29,8 +29,13 @@ type dindSwarmAndRegistryInfo struct {
 	swarmAddress    string
 	registryAddress string
 	configuredCmd   icmd.Cmd
+	configDir       string
+	tmpDir          *fs.Dir
 	stopRegistry    func()
 	registryLogs    func() string
+	dockerCmd       func(...string) string
+	execCmd         func(...string) string
+	localCmd        func(...string) string
 }
 
 func runWithDindSwarmAndRegistry(t *testing.T, todo func(dindSwarmAndRegistryInfo)) {
@@ -40,15 +45,40 @@ func runWithDindSwarmAndRegistry(t *testing.T, todo func(dindSwarmAndRegistryInf
 	tmpDir := fs.NewDir(t, t.Name())
 	defer tmpDir.Remove()
 
+	var configDir string
+	for _, val := range cmd.Env {
+		if ok := strings.HasPrefix(val, "DOCKER_CONFIG="); ok {
+			configDir = strings.Replace(val, "DOCKER_CONFIG=", "", 1)
+		}
+	}
+
+	// Initialize the info struct
+	runner := dindSwarmAndRegistryInfo{configuredCmd: cmd, configDir: configDir, tmpDir: tmpDir}
+
+	// Func to execute command locally
+	runLocalCmd := func(params ...string) string {
+		if len(params) == 0 {
+			return ""
+		}
+		cmd := icmd.Command(params[0], params[1:]...)
+		result := icmd.RunCmd(cmd)
+		result.Assert(t, icmd.Success)
+		return result.Combined()
+	}
+	// Func to execute docker cli commands
+	runDockerCmd := func(params ...string) string {
+		runner.configuredCmd.Command = dockerCli.Command(params...)
+		result := icmd.RunCmd(runner.configuredCmd)
+		result.Assert(t, icmd.Success)
+		return result.Combined()
+	}
+
 	// The dind doesn't have the cnab-app-base image so we save it in order to load it later
-	saveCmd := icmd.Cmd{Command: dockerCli.Command("save", fmt.Sprintf("docker/cnab-app-base:%s", internal.Version), "-o", tmpDir.Join("cnab-app-base.tar.gz"))}
-	icmd.RunCmd(saveCmd).Assert(t, icmd.Success)
+	runDockerCmd("save", fmt.Sprintf("docker/cnab-app-base:%s", internal.Version), "-o", tmpDir.Join("cnab-app-base.tar.gz"))
 
 	// Busybox is used in a few e2e test, let's pre-load it
-	cmd.Command = dockerCli.Command("pull", "busybox:1.30.1")
-	icmd.RunCmd(cmd).Assert(t, icmd.Success)
-	saveCmd = icmd.Cmd{Command: dockerCli.Command("save", "busybox:1.30.1", "-o", tmpDir.Join("busybox.tar.gz"))}
-	icmd.RunCmd(saveCmd).Assert(t, icmd.Success)
+	runDockerCmd("pull", "busybox:1.30.1")
+	runDockerCmd("save", "busybox:1.30.1", "-o", tmpDir.Join("busybox.tar.gz"))
 
 	// we have a difficult constraint here:
 	// - the registry must be reachable from the client side (for cnab-to-oci, which does not use the docker daemon to access the registry)
@@ -57,38 +87,40 @@ func runWithDindSwarmAndRegistry(t *testing.T, todo func(dindSwarmAndRegistryInf
 	// Solution found is: use host external IP (not loopback) so accessing from within installer container will reach the right container
 
 	registry := NewContainer("registry:2", 5000)
-	registry.Start(t, "-e", "REGISTRY_VALIDATION_MANIFESTS_URLS_ALLOW=[^http]",
+	registry.Start(t, "--name", "registry", "-e", "REGISTRY_VALIDATION_MANIFESTS_URLS_ALLOW=[^http]",
 		"-e", "REGISTRY_HTTP_ADDR=0.0.0.0:5000")
 	defer registry.StopNoFail()
 	registryAddress := registry.GetAddress(t)
 
 	swarm := NewContainer("docker:19.03.3-dind", 2375, "--insecure-registry", registryAddress)
-	swarm.Start(t, "-e", "DOCKER_TLS_CERTDIR=") // Disable certificate generate on DinD startup
+	swarm.Start(t, "--name", "dind", "-e", "DOCKER_TLS_CERTDIR=") // Disable certificate generate on DinD startup
 	defer swarm.Stop(t)
 	swarmAddress := swarm.GetAddress(t)
 
-	cmd.Command = dockerCli.Command("context", "create", "swarm-context", "--docker", fmt.Sprintf(`"host=tcp://%s"`, swarmAddress), "--default-stack-orchestrator", "swarm")
-	icmd.RunCmd(cmd).Assert(t, icmd.Success)
+	// Initialize the info struct
+	runner.registryAddress = registryAddress
+	runner.swarmAddress = swarmAddress
+	runner.stopRegistry = registry.StopNoFail
+	runner.registryLogs = registry.Logs(t)
+
+	runDockerCmd("context", "create", "swarm-context", "--docker", fmt.Sprintf(`"host=tcp://%s"`, swarmAddress), "--default-stack-orchestrator", "swarm")
+
+	runner.configuredCmd.Env = append(runner.configuredCmd.Env, "DOCKER_CONTEXT=swarm-context", "DOCKER_INSTALLER_CONTEXT=swarm-context")
 
-	cmd.Env = append(cmd.Env, "DOCKER_CONTEXT=swarm-context", "DOCKER_INSTALLER_CONTEXT=swarm-context")
 	// Initialize the swarm
-	cmd.Command = dockerCli.Command("swarm", "init")
-	icmd.RunCmd(cmd).Assert(t, icmd.Success)
+	runDockerCmd("swarm", "init")
 	// Load the needed base cnab image into the swarm docker engine
-	cmd.Command = dockerCli.Command("load", "-i", tmpDir.Join("cnab-app-base.tar.gz"))
-	icmd.RunCmd(cmd).Assert(t, icmd.Success)
+	runDockerCmd("load", "-i", tmpDir.Join("cnab-app-base.tar.gz"))
 	// Pre-load busybox image used by a few e2e tests
-	cmd.Command = dockerCli.Command("load", "-i", tmpDir.Join("busybox.tar.gz"))
-	icmd.RunCmd(cmd).Assert(t, icmd.Success)
+	runDockerCmd("load", "-i", tmpDir.Join("busybox.tar.gz"))
 
-	info := dindSwarmAndRegistryInfo{
-		configuredCmd:   cmd,
-		registryAddress: registryAddress,
-		swarmAddress:    swarmAddress,
-		stopRegistry:    registry.StopNoFail,
-		registryLogs:    registry.Logs(t),
+	runner.localCmd = runLocalCmd
+	runner.dockerCmd = runDockerCmd
+	runner.execCmd = func(params ...string) string {
+		args := append([]string{"docker", "exec", "-t", "dind"}, params...)
+		return runLocalCmd(args...)
 	}
-	todo(info)
+	todo(runner)
 }
 
 func build(t *testing.T, cmd icmd.Cmd, dockerCli dockerCliCommand, ref, path string) {