From 45593785a8cf6c3dc991743e05a47d81470a5a6e Mon Sep 17 00:00:00 2001 From: Denis Kisselev Date: Mon, 13 Apr 2020 16:48:46 -0700 Subject: [PATCH] Implement coverage and test result reporting --- Makefile | 26 +++++++------- azure-pipelines.yml | 84 ++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 89 insertions(+), 21 deletions(-) diff --git a/Makefile b/Makefile index c7f80d5bc7c..bd5f83a4523 100644 --- a/Makefile +++ b/Makefile @@ -36,18 +36,15 @@ generate-test-certs: # Run Controller tests against the configured cluster test-integration-controllers: generate fmt vet manifests TEST_RESOURCE_PREFIX=$(TEST_RESOURCE_PREFIX) TEST_USE_EXISTING_CLUSTER=true REQUEUE_AFTER=20 \ - go test -v -tags "$(BUILD_TAGS)" -coverprofile=reports/integration-controllers-coverage-ouput.txt -coverpkg=./... -covermode count -parallel 4 -timeout 45m \ + go test -v -tags "$(BUILD_TAGS)" -coverprofile=reports/integration-controllers-coverage-output.txt -coverpkg=./... -covermode count -parallel 4 -timeout 45m \ ./controllers/... \ 2>&1 | tee reports/integration-controllers-output.txt go-junit-report < reports/integration-controllers-output.txt > reports/integration-controllers-report.xml - gocov convert reports/integration-controllers-coverage-ouput.txt > reports/integration-controllers-coverage-ouput.json - gocov-xml < reports/integration-controllers-coverage-ouput.json > reports/integration-controllers-coverage.xml - go tool cover -html=reports/integration-controllers-coverage-ouput.txt -o reports/integration-controllers-coverage.html # Run Resource Manager tests against the configured cluster test-integration-managers: generate fmt vet manifests TEST_USE_EXISTING_CLUSTER=true TEST_CONTROLLER_WITH_MOCKS=false REQUEUE_AFTER=20 \ - go test -v -coverprofile=coverage/coverage-managers.txt -covermode count -parallel 4 -timeout 45m \ + go test -v -coverprofile=reports/integration-managers-coverage-ouput.txt -coverpkg=./... -covermode count -parallel 4 -timeout 45m \ ./api/... \ ./pkg/resourcemanager/eventhubs/... \ ./pkg/resourcemanager/resourcegroups/... \ @@ -57,9 +54,8 @@ test-integration-managers: generate fmt vet manifests ./pkg/resourcemanager/psql/firewallrule/... \ ./pkg/resourcemanager/appinsights/... \ ./pkg/resourcemanager/vnet/... - 2>&1 | tee testlogs.txt - go-junit-report < testlogs.txt > report-managers.xml - go tool cover -html=coverage-managers.txt -o cover-managers.html + 2>&1 | tee reports/integration-managers-output.txt + go-junit-report < reports/integration-managers-output.txt > reports/integration-managers-report.xml # Run all available tests. Note that Controllers are not unit-testable. test-unit: generate fmt vet manifests @@ -72,6 +68,13 @@ test-unit: generate fmt vet manifests go-junit-report < testlogs.txt > report-unit.xml go tool cover -html=coverage/coverage.txt -o cover-unit.html +# Merge all the available test coverage results and publish a single report +test-process-coverage: + find reports -name "*-coverage-output.txt" -type f -print | xargs gocovmerge > reports/merged-coverage-output.txt + gocov convert reports/merged-coverage-output.txt > reports/merged-coverage-output.json + gocov-xml < reports/merged-coverage-output.json > reports/merged-coverage.xml + go tool cover -html=reports/merged-coverage-output.txt -o reports/merged-coverage.html + # Cleanup resource groups azure created by tests using pattern matching 't-rg-' test-cleanup-azure-resources: # Delete the resource groups that match the pattern @@ -185,9 +188,8 @@ endif install-aad-pod-identity: kubectl apply -f https://raw.githubusercontent.com/Azure/aad-pod-identity/master/deploy/infra/deployment-rbac.yaml -install-test-dependency: - go get -u github.com/jstemmer/go-junit-report \ +install-test-dependencies: + go get github.com/jstemmer/go-junit-report \ && go get github.com/axw/gocov/gocov \ && go get github.com/AlekSi/gocov-xml \ - && go get github.com/onsi/ginkgo/ginkgo \ - && go get golang.org/x/tools/cmd/cover + && go get github.com/wadey/gocovmerge diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 4e4a73b5d28..73a580493d7 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -12,6 +12,8 @@ variables: # Agent VM image name vmImageName: 'ubuntu-latest' + + stages: - stage: Build jobs: @@ -23,19 +25,25 @@ stages: version: '1.13.7' - task: Go@0 - displayName: Get Go dependencies + displayName: Go Get inputs: command: 'get' arguments: '-d' workingDirectory: '$(System.DefaultWorkingDirectory)' - bash: | + set -x + go env os=$(go env GOOS) arch=$(go env GOARCH) - curl -L https://go.kubebuilder.io/dl/2.3.1/${os}/${arch} | tar -xz -C /tmp/ - export KUBEBUILDER_ASSETS=/tmp/kubebuilder_2.3.1_${os}_${arch} + curl -L https://go.kubebuilder.io/dl/2.3.1/${os}/${arch} | tar -xz -C $(go env GOPATH)/bin + mv $(go env GOPATH)/bin/kubebuilder_2.3.1_${os}_${arch} /usr/local/kubebuilder + export PATH=$PATH:/usr/local/kubebuilder/bin - displayName: Install kubebuilder + echo '##vso[task.prependpath]/usr/local/kubebuilder/bin' + + make install-test-dependencies + displayName: Install kubebuilder and test dependencies - bash: make manager displayName: Build manager binary @@ -74,7 +82,7 @@ stages: - stage: Deploy jobs: - job: InstallControllers - displayName: Install Controllers + displayName: Install CRD's steps: - task: Kubernetes@1 displayName: Fetch kubeconfig @@ -114,6 +122,7 @@ stages: - stage: Test jobs: - job: + displayName: Run integration tests steps: - task: Kubernetes@1 displayName: Fetch kubeconfig @@ -125,9 +134,25 @@ stages: checkLatest: true - bash: | + set -x + go env + os=$(go env GOOS) + arch=$(go env GOARCH) + curl -L https://go.kubebuilder.io/dl/2.3.1/${os}/${arch} | tar -xz -C $(go env GOPATH)/bin + mv $(go env GOPATH)/bin/kubebuilder_2.3.1_${os}_${arch} /usr/local/kubebuilder + + export PATH=$PATH:/usr/local/kubebuilder/bin + echo '##vso[task.prependpath]/usr/local/kubebuilder/bin' + + make install-test-dependencies + displayName: Install kubebuilder and test dependencies + + - bash: | + export PATH=$PATH:$(go env GOPATH)/bin + + make install-test-dependencies make test-integration-controllers - make test-integration-managers - displayName: Test controllers & managers + displayName: Test Controllers continueOnError: true env: AZURE_SUBSCRIPTION_ID: $(AZURE_SUBSCRIPTION_ID) @@ -135,8 +160,49 @@ stages: AZURE_CLIENT_ID: $(AZURE_CLIENT_ID) AZURE_OPERATOR_KEYVAULT: $(AZURE_OPERATOR_KEYVAULT) AZURE_TENANT_ID: $(AZURE_TENANT_ID) + BUILD_ID: $(Build.BuildId) + + # The manager tests have been disabled as their addition casues the build pipeline to time out on a single hosted node + # It can be re-enabled once tests are run in parallel or more than 1 hour of execution time is permitted for the suite + # - bash: | + # export PATH=$PATH:$(go env GOPATH)/bin + # make install-test-dependencies + + # make test-integration-managers + # displayName: Test Managers + # continueOnError: true + # env: + # AZURE_SUBSCRIPTION_ID: $(AZURE_SUBSCRIPTION_ID) + # AZURE_CLIENT_SECRET: $(AZURE_SP_SECRET) + # AZURE_CLIENT_ID: $(AZURE_CLIENT_ID) + # AZURE_OPERATOR_KEYVAULT: $(AZURE_OPERATOR_KEYVAULT) + # AZURE_TENANT_ID: $(AZURE_TENANT_ID) + # BUILD_ID: $(Build.BuildId) + + - bash: | + export PATH=$PATH:$(go env GOPATH)/bin + + make test-process-coverage + displayName: Render Coverage Reports + continueOnError: true + workingDirectory: '$(System.DefaultWorkingDirectory)' + + - task: PublishTestResults@2 + inputs: + testRunner: JUnit + testResultsFiles: $(System.DefaultWorkingDirectory)/reports/*-report.xml + + - task: PublishCodeCoverageResults@1 + inputs: + codeCoverageTool: Cobertura + summaryFileLocation: $(System.DefaultWorkingDirectory)/reports/merged-coverage.xml + + - task: PublishBuildArtifacts@1 + inputs: + pathtoPublish: '$(System.DefaultWorkingDirectory)/reports' + artifactName: reports -- stage: Clean +- stage: Cleanup jobs: - job: ResourceCleanup continueOnError: true @@ -146,4 +212,4 @@ stages: azureSubscription: 'tpark-jananiv-azureserviceoperators(4ef44fef-c51d-4d7c-a6ff-8635c02848b1)' scriptType: 'bash' scriptLocation: 'inlineScript' - inlineScript: 'make test-cleanup-azure-resources' \ No newline at end of file + inlineScript: 'make test-cleanup-azure-resources'