Implement coverage and test result reporting

Azure · Apr 28, 2020 · 4559378 · 4559378
1 parent f860c81
commit 4559378
Show file tree

Hide file tree

Showing 2 changed files with 89 additions and 21 deletions.
diff --git a/Makefile b/Makefile
@@ -36,18 +36,15 @@ generate-test-certs:
 # Run Controller tests against the configured cluster
 test-integration-controllers: generate fmt vet manifests
 	TEST_RESOURCE_PREFIX=$(TEST_RESOURCE_PREFIX) TEST_USE_EXISTING_CLUSTER=true REQUEUE_AFTER=20 \
-	go test -v -tags "$(BUILD_TAGS)" -coverprofile=reports/integration-controllers-coverage-ouput.txt -coverpkg=./... -covermode count -parallel 4 -timeout 45m \
+	go test -v -tags "$(BUILD_TAGS)" -coverprofile=reports/integration-controllers-coverage-output.txt -coverpkg=./... -covermode count -parallel 4 -timeout 45m \
 	./controllers/... \
 	2>&1 | tee reports/integration-controllers-output.txt
 	go-junit-report < reports/integration-controllers-output.txt > reports/integration-controllers-report.xml
-	gocov convert reports/integration-controllers-coverage-ouput.txt > reports/integration-controllers-coverage-ouput.json
-	gocov-xml < reports/integration-controllers-coverage-ouput.json > reports/integration-controllers-coverage.xml
-	go tool cover -html=reports/integration-controllers-coverage-ouput.txt -o reports/integration-controllers-coverage.html
 
 # Run Resource Manager tests against the configured cluster
 test-integration-managers: generate fmt vet manifests
 	TEST_USE_EXISTING_CLUSTER=true TEST_CONTROLLER_WITH_MOCKS=false REQUEUE_AFTER=20 \
-	go test -v -coverprofile=coverage/coverage-managers.txt -covermode count  -parallel 4 -timeout 45m \
+	go test -v -coverprofile=reports/integration-managers-coverage-ouput.txt -coverpkg=./... -covermode count -parallel 4 -timeout 45m \
 	./api/... \
 	./pkg/resourcemanager/eventhubs/...  \
 	./pkg/resourcemanager/resourcegroups/...  \
@@ -57,9 +54,8 @@ test-integration-managers: generate fmt vet manifests
 	./pkg/resourcemanager/psql/firewallrule/... \
 	./pkg/resourcemanager/appinsights/... \
 	./pkg/resourcemanager/vnet/...
-	2>&1 | tee testlogs.txt
-	go-junit-report < testlogs.txt > report-managers.xml
-	go tool cover -html=coverage-managers.txt -o cover-managers.html
+	2>&1 | tee reports/integration-managers-output.txt
+	go-junit-report < reports/integration-managers-output.txt > reports/integration-managers-report.xml
 
 # Run all available tests. Note that Controllers are not unit-testable.
 test-unit: generate fmt vet manifests 
@@ -72,6 +68,13 @@ test-unit: generate fmt vet manifests
 	go-junit-report < testlogs.txt > report-unit.xml
 	go tool cover -html=coverage/coverage.txt -o cover-unit.html
 
+# Merge all the available test coverage results and publish a single report
+test-process-coverage:
+	find reports -name "*-coverage-output.txt" -type f -print | xargs gocovmerge > reports/merged-coverage-output.txt
+	gocov convert reports/merged-coverage-output.txt > reports/merged-coverage-output.json
+	gocov-xml < reports/merged-coverage-output.json > reports/merged-coverage.xml
+	go tool cover -html=reports/merged-coverage-output.txt -o reports/merged-coverage.html
+
 # Cleanup resource groups azure created by tests using pattern matching 't-rg-'
 test-cleanup-azure-resources: 	
 	# Delete the resource groups that match the pattern
@@ -185,9 +188,8 @@ endif
 install-aad-pod-identity:
 	kubectl apply -f https://raw.githubusercontent.com/Azure/aad-pod-identity/master/deploy/infra/deployment-rbac.yaml
 
-install-test-dependency:
-	go get -u github.com/jstemmer/go-junit-report \
+install-test-dependencies:
+	go get github.com/jstemmer/go-junit-report \
 	&& go get github.com/axw/gocov/gocov \
 	&& go get github.com/AlekSi/gocov-xml \
-	&& go get github.com/onsi/ginkgo/ginkgo \
-	&& go get golang.org/x/tools/cmd/cover
+	&& go get github.com/wadey/gocovmerge
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
@@ -12,6 +12,8 @@ variables:
   # Agent VM image name
   vmImageName: 'ubuntu-latest'
 
+
+
 stages:
 - stage: Build
   jobs:
@@ -23,19 +25,25 @@ stages:
         version: '1.13.7'
 
     - task: Go@0
-      displayName: Get Go dependencies
+      displayName: Go Get
       inputs:
         command: 'get'
         arguments: '-d'
         workingDirectory: '$(System.DefaultWorkingDirectory)'
 
     - bash: |
+        set -x
+        go env
         os=$(go env GOOS)
         arch=$(go env GOARCH)
-        curl -L https://go.kubebuilder.io/dl/2.3.1/${os}/${arch} | tar -xz -C /tmp/    
-        export KUBEBUILDER_ASSETS=/tmp/kubebuilder_2.3.1_${os}_${arch}
+        curl -L https://go.kubebuilder.io/dl/2.3.1/${os}/${arch} | tar -xz -C $(go env GOPATH)/bin
+        mv $(go env GOPATH)/bin/kubebuilder_2.3.1_${os}_${arch} /usr/local/kubebuilder
+
         export PATH=$PATH:/usr/local/kubebuilder/bin
-      displayName: Install kubebuilder
+        echo '##vso[task.prependpath]/usr/local/kubebuilder/bin'
+
+        make install-test-dependencies
+      displayName: Install kubebuilder and test dependencies
       
     - bash: make manager
       displayName: Build manager binary
@@ -74,7 +82,7 @@ stages:
 - stage: Deploy
   jobs:
   - job: InstallControllers
-    displayName: Install Controllers
+    displayName: Install CRD's
     steps:
     - task: Kubernetes@1
       displayName: Fetch kubeconfig
@@ -114,6 +122,7 @@ stages:
 - stage: Test
   jobs:
   - job:
+    displayName: Run integration tests
     steps:
     - task: Kubernetes@1
       displayName: Fetch kubeconfig
@@ -125,18 +134,75 @@ stages:
         checkLatest: true
 
     - bash: |
+        set -x
+        go env
+        os=$(go env GOOS)
+        arch=$(go env GOARCH)
+        curl -L https://go.kubebuilder.io/dl/2.3.1/${os}/${arch} | tar -xz -C $(go env GOPATH)/bin
+        mv $(go env GOPATH)/bin/kubebuilder_2.3.1_${os}_${arch} /usr/local/kubebuilder
+
+        export PATH=$PATH:/usr/local/kubebuilder/bin
+        echo '##vso[task.prependpath]/usr/local/kubebuilder/bin'
+
+        make install-test-dependencies
+      displayName: Install kubebuilder and test dependencies
+
+    - bash: |
+        export PATH=$PATH:$(go env GOPATH)/bin
+
+        make install-test-dependencies
         make test-integration-controllers
-        make test-integration-managers
-      displayName: Test controllers & managers
+      displayName: Test Controllers
       continueOnError: true
       env:
         AZURE_SUBSCRIPTION_ID: $(AZURE_SUBSCRIPTION_ID)
         AZURE_CLIENT_SECRET: $(AZURE_SP_SECRET)
         AZURE_CLIENT_ID: $(AZURE_CLIENT_ID)
         AZURE_OPERATOR_KEYVAULT: $(AZURE_OPERATOR_KEYVAULT)
         AZURE_TENANT_ID: $(AZURE_TENANT_ID)
+        BUILD_ID: $(Build.BuildId)
+
+    # The manager tests have been disabled as their addition casues the build pipeline to time out on a single hosted node
+    # It can be re-enabled once tests are run in parallel or more than 1 hour of execution time is permitted for the suite
+    # - bash: |
+    #     export PATH=$PATH:$(go env GOPATH)/bin
+    #     make install-test-dependencies
+
+    #     make test-integration-managers
+    #   displayName: Test Managers
+    #   continueOnError: true
+    #   env:
+    #     AZURE_SUBSCRIPTION_ID: $(AZURE_SUBSCRIPTION_ID)
+    #     AZURE_CLIENT_SECRET: $(AZURE_SP_SECRET)
+    #     AZURE_CLIENT_ID: $(AZURE_CLIENT_ID)
+    #     AZURE_OPERATOR_KEYVAULT: $(AZURE_OPERATOR_KEYVAULT)
+    #     AZURE_TENANT_ID: $(AZURE_TENANT_ID)
+    #     BUILD_ID: $(Build.BuildId)
+
+    - bash: |
+        export PATH=$PATH:$(go env GOPATH)/bin
+
+        make test-process-coverage
+      displayName: Render Coverage Reports
+      continueOnError: true
+      workingDirectory: '$(System.DefaultWorkingDirectory)'
+
+    - task: PublishTestResults@2
+      inputs:
+        testRunner: JUnit
+        testResultsFiles: $(System.DefaultWorkingDirectory)/reports/*-report.xml
+
+    - task: PublishCodeCoverageResults@1
+      inputs:
+        codeCoverageTool: Cobertura 
+        summaryFileLocation: $(System.DefaultWorkingDirectory)/reports/merged-coverage.xml
+
+    - task: PublishBuildArtifacts@1
+      inputs:
+        pathtoPublish: '$(System.DefaultWorkingDirectory)/reports'
+        artifactName: reports
 
-- stage: Clean
+- stage: Cleanup
   jobs:
   - job: ResourceCleanup
     continueOnError: true
@@ -146,4 +212,4 @@ stages:
         azureSubscription: 'tpark-jananiv-azureserviceoperators(4ef44fef-c51d-4d7c-a6ff-8635c02848b1)'
         scriptType: 'bash'
         scriptLocation: 'inlineScript'
-        inlineScript: 'make test-cleanup-azure-resources'
+        inlineScript: 'make test-cleanup-azure-resources'