Improve test stability

.github/workflows/ci.yml

@@ -24,14 +24,17 @@ jobs:
           run: make test-unit-coverage
     integration:
         strategy:
+          fail-fast: false
           matrix:
             go-version: [1.15.x]
             os: [ubuntu-latest]
             kubernetes:
-              - v1.17.17
+              # Only v1.18 is currently enabled because of the flakiness in the tests, specifically API calls failing with "etcdserver: request timed out"
+              #- v1.17.17
               - v1.18.15
-              - v1.19.7
-              - v1.20.2
+              #- v1.19.7
+              #- v1.20.2
+          max-parallel: 1
         runs-on: ${{ matrix.os }}
         steps:
         - name: Install Go
@@ -64,14 +67,17 @@ jobs:
           run: make test-integration
     e2e:
         strategy:
+          fail-fast: false
           matrix:
             go-version: [1.15.x]
             os: [ubuntu-latest]
             kubernetes:
-              - v1.17.17
+              # Only v1.18 is currently enabled because of the flakiness in the tests, specifically API calls failing with "etcdserver: request timed out"
+              #- v1.17.17
               - v1.18.15
-              - v1.19.7
-              - v1.20.2
+              #- v1.19.7
+              #- v1.20.2
+          max-parallel: 2
         runs-on: ${{ matrix.os }}
         steps:
         - name: Install Go
@@ -107,7 +113,7 @@ jobs:
             kubectl apply -f test/data/registry.yaml
             kubectl -n registry rollout status deployment registry --timeout=1m
         - name: Install ko
-          run: curl -fsL https://github.com/google/ko/releases/download/v0.8.0/ko_0.8.0_Linux_x86_64.tar.gz | sudo tar xzf - -C /usr/local/bin ko
+          run: curl -fsL https://github.com/google/ko/releases/download/v0.8.1/ko_0.8.1_Linux_x86_64.tar.gz | sudo tar xzf - -C /usr/local/bin ko
         - name: Install Shipwright Build
           run: |
             make install-operator-kind
@@ -117,5 +123,14 @@ jobs:
         - name: Build operator logs
           if: ${{ failure() }}
           run: |
-            POD_NAME=$(kubectl -n build-operator get pod -o json | jq -r '.items[0].metadata.name')
+            echo "# Pods:"
+            kubectl -n build-operator get pod
+            PODS=$(kubectl -n build-operator get pod -o json)
+            POD_NAME=$(echo "${PODS}" | jq -r '.items[] | select(.metadata.name | startswith("build-operator-")) | .metadata.name')
+            RESTART_COUNT=$(echo "${PODS}" | jq -r ".items[] | select(.metadata.name == \"${POD_NAME}\") | .status.containerStatuses[0].restartCount")
+            if [ "${RESTART_COUNT}" != "0" ]; then
+              echo "# Previous logs:"
+              kubectl -n build-operator logs "${POD_NAME}" --previous
+            fi
+            echo "# Logs:"
             kubectl -n build-operator logs "${POD_NAME}"

.travis.yml

@@ -7,7 +7,7 @@ services:
 install:
   - |
     # Install ko
-    curl -fsL https://github.com/google/ko/releases/download/v0.8.0/ko_0.8.0_Linux_x86_64.tar.gz | sudo tar xzf - -C /usr/local/bin ko
+    curl -fsL https://github.com/google/ko/releases/download/v0.8.1/ko_0.8.1_Linux_x86_64.tar.gz | sudo tar xzf - -C /usr/local/bin ko
 
 deploy:
   - provider: script

test/clusterbuildstrategy_samples.go

@@ -247,6 +247,29 @@ spec:
         memory: 128Mi
 `
 
+// ClusterBuildStrategySleep30s is a strategy that does only sleep 30 seconds
+const ClusterBuildStrategySleep30s = `
+apiVersion: build.dev/v1alpha1
+kind: ClusterBuildStrategy
+metadata:
+  name: noop
+spec:
+  buildSteps:
+  - name: sleep30
+    image: alpine:latest
+    command:
+    - sleep
+    args:
+    - "30s"
+    resources:
+      limits:
+        cpu: 250m
+        memory: 128Mi
+      requests:
+        cpu: 250m
+        memory: 128Mi
+`
+
 // ClusterBuildStrategyWithAnnotations is a cluster build strategy that contains annotations
 const ClusterBuildStrategyWithAnnotations = `
 apiVersion: build.dev/v1alpha1

test/e2e/samples.go

@@ -18,6 +18,7 @@ import (
 	framework "github.com/operator-framework/operator-sdk/pkg/test"
 	"knative.dev/pkg/apis"
 
+	corev1 "k8s.io/api/core/v1"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
@@ -81,7 +82,7 @@ func amendBuild(identifier string, b *operator.Build) {
 }
 
 // CreateBuild loads the builds definition from the file path, unifies the output image based on
-// the identifier and creates it in a namespace
+// the identifier, creates it in a namespace and waits for it to be registered
 func createBuild(ctx *framework.Context, namespace string, identifier string, filePath string, timeout time.Duration, retry time.Duration) {
 	Logf("Creating build %s", identifier)
 
@@ -98,6 +99,23 @@ func createBuild(ctx *framework.Context, namespace string, identifier string, fi
 	Expect(err).ToNot(HaveOccurred(), "Unable to create build %s", identifier)
 
 	Logf("Build %s created", identifier)
+
+	buildName := types.NamespacedName{
+		Namespace: namespace,
+		Name:      b.Name,
+	}
+
+	trueCondition := corev1.ConditionTrue
+	falseCondition := corev1.ConditionFalse
+
+	Eventually(func() corev1.ConditionStatus {
+		err = clientGet(buildName, b)
+		Expect(err).ToNot(HaveOccurred(), "Error retrieving a build")
+
+		Expect(b.Status.Registered).ToNot(Equal(falseCondition), "Build registered status is false")
+
+		return b.Status.Registered
+	}, time.Duration(20*getTimeoutMultiplier())*time.Second, time.Second).Should(Equal(trueCondition), "Build was not registered")
 }
 
 // retrieveBuildAndBuildRun will retrieve the build and buildRun

test/integration/build_to_buildruns_test.go

@@ -82,6 +82,9 @@ var _ = Describe("Integration tests Build and BuildRuns", func() {
 
 			Expect(tb.CreateBuild(buildObject)).To(BeNil())
 
+			buildObject, err = tb.GetBuildTillValidation(buildObject.Name)
+			Expect(err).To(BeNil())
+
 			Expect(tb.CreateBR(buildRunObject)).To(BeNil())
 
 			br, err := tb.GetBRTillCompletion(buildRunObject.Name)
@@ -104,6 +107,9 @@ var _ = Describe("Integration tests Build and BuildRuns", func() {
 
 			Expect(tb.CreateBuild(buildObject)).To(BeNil())
 
+			buildObject, err = tb.GetBuildTillValidation(buildObject.Name)
+			Expect(err).To(BeNil())
+
 			Expect(tb.CreateBR(buildRunObject)).To(BeNil())
 
 			br, err := tb.GetBRTillCompletion(buildRunObject.Name)
@@ -118,6 +124,9 @@ var _ = Describe("Integration tests Build and BuildRuns", func() {
 
 			Expect(tb.CreateBuild(buildObject)).To(BeNil())
 
+			buildObject, err = tb.GetBuildTillValidation(buildObject.Name)
+			Expect(err).To(BeNil())
+
 			buildRun, err := tb.Catalog.LoadBRWithNameAndRef(
 				BUILDRUN+tb.Namespace,
 				BUILD+tb.Namespace,
@@ -149,6 +158,9 @@ var _ = Describe("Integration tests Build and BuildRuns", func() {
 
 			Expect(tb.CreateBuild(buildObject)).To(BeNil())
 
+			buildObject, err = tb.GetBuildTillValidation(buildObject.Name)
+			Expect(err).To(BeNil())
+
 			Expect(tb.CreateBR(buildRunObject)).To(BeNil())
 
 			// Wait for BR to get an Starttime
@@ -176,6 +188,9 @@ var _ = Describe("Integration tests Build and BuildRuns", func() {
 
 			Expect(tb.CreateBuild(build)).To(BeNil())
 
+			buildObject, err = tb.GetBuildTillValidation(buildObject.Name)
+			Expect(err).To(BeNil())
+
 			Expect(tb.CreateBR(buildRunObject)).To(BeNil())
 
 			br, err := tb.GetBRTillStartTime(buildRunObject.Name)
@@ -237,6 +252,9 @@ var _ = Describe("Integration tests Build and BuildRuns", func() {
 
 			Expect(tb.CreateBuild(buildObject)).To(BeNil())
 
+			buildObject, err = tb.GetBuildTillValidation(buildObject.Name)
+			Expect(err).To(BeNil())
+
 			Expect(tb.CreateBR(buildRunObject)).To(BeNil())
 
 			br, err := tb.GetBRTillCompletion(buildRunObject.Name)
@@ -259,6 +277,9 @@ var _ = Describe("Integration tests Build and BuildRuns", func() {
 
 			Expect(tb.CreateBuild(buildObject)).To(BeNil())
 
+			buildObject, err = tb.GetBuildTillValidation(buildObject.Name)
+			Expect(err).To(BeNil())
+
 			buildRun, err := tb.Catalog.LoadBRWithNameAndRef(
 				BUILDRUN+tb.Namespace,
 				BUILD+tb.Namespace+"foobar",
@@ -288,6 +309,9 @@ var _ = Describe("Integration tests Build and BuildRuns", func() {
 
 			Expect(tb.CreateBuild(buildObject)).To(BeNil())
 
+			buildObject, err = tb.GetBuildTillValidation(buildObject.Name)
+			Expect(err).To(BeNil())
+
 			buildRun01, err := tb.Catalog.LoadBRWithNameAndRef(
 				BUILDRUN+tb.Namespace+"01",
 				BUILD+tb.Namespace,
@@ -340,6 +364,9 @@ var _ = Describe("Integration tests Build and BuildRuns", func() {
 
 			Expect(tb.CreateBuild(buildObject)).To(BeNil())
 
+			buildObject, err = tb.GetBuildTillValidation(buildObject.Name)
+			Expect(err).To(BeNil())
+
 			autoDeleteBuildRun, err := tb.Catalog.LoadBRWithNameAndRef(
 				BUILDRUN+tb.Namespace,
 				BUILD+tb.Namespace,
@@ -352,7 +379,7 @@ var _ = Describe("Integration tests Build and BuildRuns", func() {
 			_, err = tb.GetBRTillStartTime(autoDeleteBuildRun.Name)
 			Expect(err).To(BeNil())
 
-			br, err := tb.GetBR(BUILDRUN + tb.Namespace)
+			br, err := tb.GetBRTillOwner(BUILDRUN+tb.Namespace, buildObject.Name)
 			Expect(err).To(BeNil())
 			Expect(ownerReferenceNames(br.OwnerReferences)).Should(ContainElement(buildObject.Name))
 
@@ -369,6 +396,9 @@ var _ = Describe("Integration tests Build and BuildRuns", func() {
 
 			Expect(tb.CreateBuild(buildObject)).To(BeNil())
 
+			buildObject, err = tb.GetBuildTillValidation(buildObject.Name)
+			Expect(err).To(BeNil())
+
 			autoDeleteBuildRun, err := tb.Catalog.LoadBRWithNameAndRef(
 				BUILDRUN+tb.Namespace,
 				BUILD+tb.Namespace,
@@ -389,7 +419,7 @@ var _ = Describe("Integration tests Build and BuildRuns", func() {
 			err = tb.DeleteBuild(BUILD + tb.Namespace)
 			Expect(err).To(BeNil())
 
-			br, err := tb.GetBR(BUILDRUN + tb.Namespace)
+			br, err := tb.GetBRTillNotOwner(BUILDRUN+tb.Namespace, buildObject.Name)
 			Expect(err).To(BeNil())
 			Expect(ownerReferenceNames(br.OwnerReferences)).ShouldNot(ContainElement(buildObject.Name))
 
@@ -398,6 +428,9 @@ var _ = Describe("Integration tests Build and BuildRuns", func() {
 
 			Expect(tb.CreateBuild(buildObject)).To(BeNil())
 
+			buildObject, err = tb.GetBuildTillValidation(buildObject.Name)
+			Expect(err).To(BeNil())
+
 			autoDeleteBuildRun, err := tb.Catalog.LoadBRWithNameAndRef(
 				BUILDRUN+tb.Namespace,
 				BUILD+tb.Namespace,
@@ -418,7 +451,7 @@ var _ = Describe("Integration tests Build and BuildRuns", func() {
 			err = tb.DeleteBuild(BUILD + tb.Namespace)
 			Expect(err).To(BeNil())
 
-			br, err := tb.GetBR(BUILDRUN + tb.Namespace)
+			br, err := tb.GetBRTillNotOwner(BUILDRUN+tb.Namespace, buildObject.Name)
 			Expect(err).To(BeNil())
 			Expect(ownerReferenceNames(br.OwnerReferences)).ShouldNot(ContainElement(buildObject.Name))
 
@@ -427,6 +460,9 @@ var _ = Describe("Integration tests Build and BuildRuns", func() {
 
 			Expect(tb.CreateBuild(buildObject)).To(BeNil())
 
+			buildObject, err = tb.GetBuildTillValidation(buildObject.Name)
+			Expect(err).To(BeNil())
+
 			autoDeleteBuildRun, err := tb.Catalog.LoadBRWithNameAndRef(
 				BUILDRUN+tb.Namespace,
 				BUILD+tb.Namespace,
@@ -457,7 +493,7 @@ var _ = Describe("Integration tests Build and BuildRuns", func() {
 			Expect(err).To(BeNil())
 			Expect(patchedBuild.Annotations[v1alpha1.AnnotationBuildRunDeletion]).To(Equal("true"))
 
-			br, err := tb.GetBR(BUILDRUN + tb.Namespace)
+			br, err := tb.GetBRTillOwner(BUILDRUN+tb.Namespace, buildObject.Name)
 			Expect(err).To(BeNil())
 			Expect(ownerReferenceNames(br.OwnerReferences)).Should(ContainElement(buildObject.Name))
 

test/integration/buildruns_to_sa_test.go

@@ -6,6 +6,7 @@ package integration_test
 
 import (
 	"fmt"
+
 	corev1 "k8s.io/api/core/v1"
 
 	. "github.com/onsi/ginkgo"
@@ -71,6 +72,9 @@ var _ = Describe("Integration tests BuildRuns and Service-accounts", func() {
 
 			Expect(tb.CreateBuild(buildObject)).To(BeNil())
 
+			buildObject, err = tb.GetBuildTillValidation(buildObject.Name)
+			Expect(err).To(BeNil())
+
 			Expect(tb.CreateBR(buildRunObject)).To(BeNil())
 
 			_, err = tb.GetBRTillStartTime(buildRunObject.Name)
@@ -99,6 +103,9 @@ var _ = Describe("Integration tests BuildRuns and Service-accounts", func() {
 
 			Expect(tb.CreateBuild(buildObject)).To(BeNil())
 
+			buildObject, err = tb.GetBuildTillValidation(buildObject.Name)
+			Expect(err).To(BeNil())
+
 			Expect(tb.CreateBR(buildRunObject)).To(BeNil())
 
 			_, err = tb.GetBRTillStartTime(buildRunObject.Name)
@@ -116,6 +123,9 @@ var _ = Describe("Integration tests BuildRuns and Service-accounts", func() {
 			}
 			Expect(tb.CreateBuild(buildObject)).To(BeNil())
 
+			buildObject, err = tb.GetBuildTillValidation(buildObject.Name)
+			Expect(err).To(BeNil())
+
 			Expect(tb.CreateBR(buildRunObject)).To(BeNil())
 
 			_, err = tb.GetBRTillStartTime(buildRunObject.Name)
@@ -137,6 +147,9 @@ var _ = Describe("Integration tests BuildRuns and Service-accounts", func() {
 		It("it fails and updates buildrun conditions if the specified serviceaccount doesn't exist", func() {
 			Expect(tb.CreateBuild(buildObject)).To(BeNil())
 
+			buildObject, err = tb.GetBuildTillValidation(buildObject.Name)
+			Expect(err).To(BeNil())
+
 			Expect(tb.CreateBR(buildRunObject)).To(BeNil())
 
 			br, err := tb.GetBRTillCompletion(buildRunObject.Name)
@@ -149,4 +162,3 @@ var _ = Describe("Integration tests BuildRuns and Service-accounts", func() {
 		})
 	})
 })
-