Add CI to verify hydration works. (#50)

* Add CI to verify hydration works.

* Write a golang test to verify we can hydrate the manifests

* Trigger go tests under prow

* Related to #42

* Latest.

* Latest.

* Fix labels.

* * Fix the tests by moving go.mod to the root of the repo. It looks
  like that is what the go unittests task currently requires.

.gitignore

@@ -0,0 +1 @@
+.idea/**

go.mod

@@ -0,0 +1,5 @@
+module github.com/kubeflow/gcp-blueprints
+
+go 1.13
+
+require github.com/termie/go-shutil v0.0.0-20140729215957-bcacb06fecae

go.sum

@@ -0,0 +1,2 @@
+github.com/termie/go-shutil v0.0.0-20140729215957-bcacb06fecae h1:vgGSvdW5Lqg+I1aZOlG32uyE6xHpLdKhZzcTEktz5wM=
+github.com/termie/go-shutil v0.0.0-20140729215957-bcacb06fecae/go.mod h1:quDq6Se6jlGwiIKia/itDZxqC5rj6/8OdFyMMAwTxCs=

kubeflow/Makefile

@@ -25,6 +25,7 @@ ACM_KF_REPO=acm-repo
 BUILD_DIR=.build
 
 # The URL you want to fetch manifests from
+# TODO(jlewi): Should we make this a setter to make it easier to update programmatically?
 MANIFESTS_URL=https://github.com/kubeflow/manifests.git@master
 
 #***********************************************************************************************************************

kubeflow/instance/kustomize/kubeflow-apps/kustomization.yaml

@@ -5,5 +5,5 @@ patchesStrategicMerge:
 - default-install-config.yaml
 - pipeline-minio-install-config.yaml
 resources:
-- ../../../upstream/manifests/stacks/gcp # {"type":"string","x-kustomize":{"setBy":"kpt","partialSetters":[{"name":"kustomize_manifests_path","value":"../../../upstream/manifests"}]}}
 - ../../../upstream/manifests/pipeline/minio/installs/gcp-pd # {"type":"string","x-kustomize":{"setBy":"kpt","partialSetters":[{"name":"kustomize_manifests_path","value":"../../../upstream/manifests"}]}}
+- ../../../upstream/manifests/stacks/generic # {"type":"string","x-kustomize":{"setBy":"kpt","partialSetters":[{"name":"kustomize_manifests_path","value":"../../../upstream/manifests"}]}}

prow_config.yaml

@@ -0,0 +1,5 @@
+# This file configures the workflows to trigger in our Prow jobs.
+# see kubeflow/testing/py/run_e2e_workflow.py
+workflows:
+  - tekton_run: kubeflow/gcp-blueprints/tests/pipeline-runs/go-tests-run.yaml 
+    name: go-unittests

tests/README.md

@@ -0,0 +1,5 @@
+# Testing
+
+Directory contains testing related code for the blueprints. 
+
+Tests are written as tekton workflows.

tests/pipeline-runs/go-tests-run.yaml

@@ -0,0 +1,33 @@
+# Pipelinerun for go unittests
+apiVersion: tekton.dev/v1alpha1
+kind: PipelineRun
+metadata:
+  generateName: go-test-
+  namespace: kf-ci
+  labels:
+    pipeline: go-test
+    repo: gcp-blueprints
+spec:
+  params:
+  # When running under prow this value will automatically be replaced 
+  # by the appropriate GCS location for the test.
+  # So the value here only matters for manual runs.
+  - name: artifacts-gcs
+    value: "gs://kubeflow-ci-deployment/gabrielwen-testing-2"
+  serviceAccountName: kf-ci
+  resources:
+  - name: source-repo
+    resourceSpec:
+      type: git
+      params:
+      # The URL must match the repo under test in prow otherwise you well get the
+      # error "Couldn't find repo under test"
+      - name: url
+        value: https://github.com/kubeflow/gcp-blueprints.git
+      # When running under prow revision automatically be replaced 
+      # by the appropriate GCS location for the test.
+      # So the value here only matters for manual runs.
+      - name: revision
+        value: master
+  pipelineRef:
+    name: go-tests

tests/validate_test.go

@@ -0,0 +1,77 @@
+package tests
+
+import (
+	shutil "github.com/termie/go-shutil"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path"
+	"testing"
+)
+
+// TestHydrate verifies we can properly hydrate the config
+func TestHydrate(t *testing.T) {
+
+	testDir, err := ioutil.TempDir("", "gcpBlueprintsTest-")
+
+	if err != nil {
+		t.Fatalf("Couldn't create temporary directory: %v", err)
+	}
+
+	currDir, err := os.Getwd()
+
+	if err != nil {
+		t.Fatalf("Couldn't get working directory %v", err)
+	}
+
+	kDir := path.Join(currDir, "..", "kubeflow")
+
+	target := path.Join(testDir, "kubeflow")
+
+	t.Logf("Using %v", target)
+	shutil.CopyTree(kDir, target, nil)
+
+	// Get the upstream packages
+	cmd := exec.Command("make", "get-pkg")
+	cmd.Dir = target
+
+	out, err := cmd.CombinedOutput()
+	t.Logf("Run %v :\n%v", cmd, string(out))
+
+	if err != nil {
+		t.Fatalf("%v failed; error %v", cmd, err)
+	}
+
+	vars := map[string]string {
+		"mgmt-ctxt": "mgmt-ctxt",
+		"name": "kf-test",
+		"gcloud.core.project": "kubeflow-ci-deployment",
+		"gcloud.compute.zone": "us-east1-d",
+		"location": "us-east1",
+		"gcloud.compute.region": "us-east1",
+	}
+
+	for _, subDir := range []string{"instance", "upstream"} {
+		for k, v := range vars {
+			cmd := exec.Command("kpt", "cfg", "set", subDir, k, v)
+			cmd.Dir = target
+			out, err := cmd.CombinedOutput()
+
+			t.Logf("Run %v:\n%v", cmd, string(out))
+
+			if err != nil {
+				t.Fatalf("%v failed; error %v", cmd, err)
+			}
+		}
+	}
+
+	cmd = exec.Command("make", "hydrate")
+	cmd.Dir = target
+	out, err = cmd.CombinedOutput()
+
+	t.Logf("Run %v:\n%v", cmd, string(out))
+
+	if err != nil {
+		t.Fatalf("%v failed; error %v", cmd, err)
+	}
+}