From 6dbcf21431e1af6c8e902f18881d1978642d44da Mon Sep 17 00:00:00 2001 From: Andrew Lock Date: Tue, 13 Aug 2024 18:36:16 +0100 Subject: [PATCH] Try fix smoke tests (#5889) ## Summary of changes - Install v1 docker-compose in the hosted runners used for Windows smoke tests - Cleanup the docker networks in the smoke tests ## Reason for change Microsoft [recently](https://learn.microsoft.com/en-us/azure/devops/release-notes/2024/pipelines/sprint-240-update#dockercompose0-uses-docker-compose-v2-in-v1-compatibility-mode) stopped shipping the v1 of docker compose. That broke all our Windows smoke tests. Unfortunately, for various other reasons, we couldn't just convert to the v2 format, due to how we're reusing various bits and pieces. Additionally, we started getting this on our linux smoke tests: ``` Creating network "ddtrace_20240812_56_default" with the default driver could not find an available, non-overlapping IPv4 address pool among the defaults to assign to the network ``` This is basically because we're creating networks, and never cleaning them up, and for efficiency reasons we're not destroying the smoke test VMs after each run. ## Implementation details - Install Docker-compose v1 in the Windows VMs. A bit hacky, but the quickest solution right now - Clean up the docker network after running the snapshot tests ## Test coverage I did some manual tests to make sure this all works as expected. Also killed and restored the smoke tests machines which solved the docker network problem temporarily. ## Other details This is obviously still all kinda horrible. I'd love to replace all this yaml with just testcontainers. One day.... _stares wistfully into the distance_ --- .../steps/install-docker-compose-v1.yml | 24 ++++++++++ .azure-pipelines/steps/run-snapshot-test.yml | 38 ++++++++++------ .azure-pipelines/ultimate-pipeline.yml | 45 ++++++++++++++++--- 3 files changed, 87 insertions(+), 20 deletions(-) create mode 100644 .azure-pipelines/steps/install-docker-compose-v1.yml diff --git a/.azure-pipelines/steps/install-docker-compose-v1.yml b/.azure-pipelines/steps/install-docker-compose-v1.yml new file mode 100644 index 000000000000..3c61eea2e67c --- /dev/null +++ b/.azure-pipelines/steps/install-docker-compose-v1.yml @@ -0,0 +1,24 @@ +parameters: + - name: isLinux + type: boolean + default: false + + - name: 'dockerComposePath' + type: string + default: 'C:/docker-compose/docker-compose.exe' + +steps: +- ${{ if eq(parameters.isLinux, true) }}: + - bash: | + sudo mkdir -p "$(dirname "${{ parameters.dockerComposePath }}")" + sudo curl -SL https://github.com/docker/compose/releases/download/1.29.2/docker-compose-linux-x86_64 -o ${{ parameters.dockerComposePath }} + sudo chmod 755 ${{ parameters.dockerComposePath }} + displayName: Download docker-compose +- ${{ else }}: + - powershell: | + $dir= (Split-Path -parent "${{ parameters.dockerComposePath }}") + mkdir -f -p $dir + # GitHub now requires TLS1.2. In PowerShell, run the following + [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 + Invoke-WebRequest "https://github.com/docker/compose/releases/download/1.29.1/docker-compose-windows-x86_64.exe" -OutFile "${{ parameters.dockerComposePath }}" + displayName: Download docker-compose diff --git a/.azure-pipelines/steps/run-snapshot-test.yml b/.azure-pipelines/steps/run-snapshot-test.yml index c1da627b5fbb..8bc9fc088202 100644 --- a/.azure-pipelines/steps/run-snapshot-test.yml +++ b/.azure-pipelines/steps/run-snapshot-test.yml @@ -12,6 +12,10 @@ parameters: - name: 'apiKey' type: string default: '' + + - name: 'dockerComposePath' + type: string + default: 'docker-compose' steps: - template: ./clean-docker-containers.yml @@ -56,7 +60,7 @@ steps: displayName: Set env-specific variables - bash: | - docker-compose -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) run --rm $(START_TEST_AGENT_TARGET) + ${{ parameters.dockerComposePath }} -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) run --rm $(START_TEST_AGENT_TARGET) env: dockerTag: $(dockerTag) DD_LOGGER_DD_API_KEY: ${{ parameters.apiKey }} @@ -65,27 +69,27 @@ steps: - script: | echo "Starting snapshot session" - docker-compose -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) exec -T $(TEST_AGENT_TARGET) $(CURL_COMMAND) --fail "http://localhost:8126$(START_ENDPOINT)" + ${{ parameters.dockerComposePath }} -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) exec -T $(TEST_AGENT_TARGET) $(CURL_COMMAND) --fail "http://localhost:8126$(START_ENDPOINT)" displayName: start snapshot session env: DD_LOGGER_DD_API_KEY: ${{ parameters.apiKey }} - bash: | - docker-compose -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) run -e dockerTag=$(dockerTag) ${{ parameters.target }} + ${{ parameters.dockerComposePath }} -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) run -e dockerTag=$(dockerTag) ${{ parameters.target }} env: dockerTag: $(dockerTag) DD_LOGGER_DD_API_KEY: ${{ parameters.apiKey }} - displayName: docker-compose run ${{ parameters.target }} + displayName: ${{ parameters.dockerComposePath }} run ${{ parameters.target }} - script: | echo "Dumping traces" - docker-compose -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) exec -T $(TEST_AGENT_TARGET) $(CURL_COMMAND) -o /debug_snapshots/${{ parameters.snapshotPrefix }}_traces.json "http://localhost:8126$(TRACE_DUMP_ENDPOINT)" + ${{ parameters.dockerComposePath }} -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) exec -T $(TEST_AGENT_TARGET) $(CURL_COMMAND) -o /debug_snapshots/${{ parameters.snapshotPrefix }}_traces.json "http://localhost:8126$(TRACE_DUMP_ENDPOINT)" echo "Dumping stats" - docker-compose -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) exec -T $(TEST_AGENT_TARGET) $(CURL_COMMAND) -o /debug_snapshots/${{ parameters.snapshotPrefix }}_stats.json "http://localhost:8126$(STATS_DUMP_ENDPOINT)" + ${{ parameters.dockerComposePath }} -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) exec -T $(TEST_AGENT_TARGET) $(CURL_COMMAND) -o /debug_snapshots/${{ parameters.snapshotPrefix }}_stats.json "http://localhost:8126$(STATS_DUMP_ENDPOINT)" echo "Dumping all requests" - docker-compose -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) exec -T $(TEST_AGENT_TARGET) $(CURL_COMMAND) -o /debug_snapshots/${{ parameters.snapshotPrefix }}_requests.json "http://localhost:8126$(REQUESTS_DUMP_ENDPOINT)" + ${{ parameters.dockerComposePath }} -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) exec -T $(TEST_AGENT_TARGET) $(CURL_COMMAND) -o /debug_snapshots/${{ parameters.snapshotPrefix }}_requests.json "http://localhost:8126$(REQUESTS_DUMP_ENDPOINT)" displayName: dump snapshots env: DD_LOGGER_DD_API_KEY: ${{ parameters.apiKey }} @@ -93,26 +97,26 @@ steps: - ${{ if eq(parameters.isLinux, true) }}: - bash: | echo "Verifying snapshot session (fail on mis-match)" - docker-compose -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) exec -T $(TEST_AGENT_TARGET) $(CURL_COMMAND) --w '\nGetting a 400 means there is a diff in snapshots. You can diff the files with the artifacts generated. You can also run the tests locally. Follow the doc in /docs/development/CI/RunSmokeTestsLocally\n' --fail "http://localhost:8126$(VERIFY_ENDPOINT)" + ${{ parameters.dockerComposePath }} -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) exec -T $(TEST_AGENT_TARGET) $(CURL_COMMAND) --w '\nGetting a 400 means there is a diff in snapshots. You can diff the files with the artifacts generated. You can also run the tests locally. Follow the doc in /docs/development/CI/RunSmokeTestsLocally\n' --fail "http://localhost:8126$(VERIFY_ENDPOINT)" displayName: check snapshots env: DD_LOGGER_DD_API_KEY: ${{ parameters.apiKey }} - ${{ else }}: - bash: | echo "Verifying snapshot session (fail on mis-match)" - docker-compose -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) exec -T $(TEST_AGENT_TARGET) $(CURL_COMMAND) --fail "http://localhost:8126$(VERIFY_ENDPOINT)" + ${{ parameters.dockerComposePath }} -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) exec -T $(TEST_AGENT_TARGET) $(CURL_COMMAND) --fail "http://localhost:8126$(VERIFY_ENDPOINT)" displayName: check snapshots env: DD_LOGGER_DD_API_KEY: ${{ parameters.apiKey }} -- script: docker-compose -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) logs $(TEST_AGENT_TARGET) +- script: ${{ parameters.dockerComposePath }} -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) logs $(TEST_AGENT_TARGET) displayName: dump docker-compose logs for $(TEST_AGENT_TARGET) env: DD_LOGGER_DD_API_KEY: ${{ parameters.apiKey }} condition: succeededOrFailed() continueOnError: true -- script: docker-compose -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) down +- script: ${{ parameters.dockerComposePath }} -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) down displayName: docker-compose stop services env: DD_LOGGER_DD_API_KEY: ${{ parameters.apiKey }} @@ -122,7 +126,7 @@ steps: # Run crash tests - ${{ if eq(parameters.isLinux, true) }}: - bash: | - LOGS=$(docker-compose -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) run -e dockerTag=$(dockerTag) -e CRASH_APP_ON_STARTUP=1 -e COMPlus_DbgEnableMiniDump=0 ${{ parameters.target }}) + LOGS=$(${{ parameters.dockerComposePath }} -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) run -e dockerTag=$(dockerTag) -e CRASH_APP_ON_STARTUP=1 -e COMPlus_DbgEnableMiniDump=0 ${{ parameters.target }}) echo $LOGS # check logs for evidence of crash detection in the output @@ -144,4 +148,12 @@ steps: - script: | sudo chmod -R 644 tracer/build_data/dumps/* || true displayName: Make dumps uploadable to AzDo - condition: succeededOrFailed() \ No newline at end of file + condition: succeededOrFailed() + +- script: | + docker network prune -f + displayName: Clean up docker networks + condition: succeededOrFailed() + continueOnError: true + + \ No newline at end of file diff --git a/.azure-pipelines/ultimate-pipeline.yml b/.azure-pipelines/ultimate-pipeline.yml index 10c1649464b0..97bd1ff546c1 100644 --- a/.azure-pipelines/ultimate-pipeline.yml +++ b/.azure-pipelines/ultimate-pipeline.yml @@ -5454,6 +5454,7 @@ stages: variables: targetShaId: $[ stageDependencies.merge_commit_id.fetch.outputs['set_sha.sha']] targetBranch: $[ stageDependencies.merge_commit_id.fetch.outputs['set_sha.branch']] + dockerComposePath: 'C:/docker-compose/docker-compose.exe' jobs: - template: steps/update-github-status-jobs.yml parameters: @@ -5469,6 +5470,10 @@ stages: vmImage: windows-2022 steps: + - template: steps/install-docker-compose-v1.yml + parameters: + dockerComposePath: $(dockerComposePath) + - template: steps/clone-repo.yml parameters: targetShaId: $(targetShaId) @@ -5492,7 +5497,7 @@ stages: displayName: create test data directories - bash: | - docker-compose -f docker-compose.windows.yml -p $(DockerComposeProjectName) build \ + $(dockerComposePath) -f docker-compose.windows.yml -p $(DockerComposeProjectName) build \ --build-arg DOTNETSDK_VERSION=$(dotnetCoreSdkLatestVersionShort) \ --build-arg RUNTIME_IMAGE=$(runtimeImage) \ --build-arg PUBLISH_FRAMEWORK=$(publishFramework) \ @@ -5509,9 +5514,10 @@ stages: target: 'nuget-smoke-tests.windows' snapshotPrefix: "smoke_test" isLinux: false + dockerComposePath: $(dockerComposePath) - bash: | - docker-compose -f docker-compose.windows.yml -p $(DockerComposeProjectName) build \ + $(dockerComposePath) -f docker-compose.windows.yml -p $(DockerComposeProjectName) build \ --build-arg DOTNETSDK_VERSION=$(dotnetCoreSdkLatestVersionShort) \ --build-arg RUNTIME_IMAGE=$(runtimeImage) \ --build-arg PUBLISH_FRAMEWORK=$(publishFramework) \ @@ -5528,6 +5534,7 @@ stages: target: 'nuget-dddotnet-smoke-tests.windows' snapshotPrefix: "smoke_test" isLinux: false + dockerComposePath: $(dockerComposePath) - publish: tracer/build_data artifact: _$(System.StageName)_$(Agent.JobName)_logs_$(System.JobAttempt) @@ -5544,6 +5551,7 @@ stages: variables: targetShaId: $[ stageDependencies.merge_commit_id.fetch.outputs['set_sha.sha']] targetBranch: $[ stageDependencies.merge_commit_id.fetch.outputs['set_sha.branch']] + dockerComposePath: 'C:/docker-compose/docker-compose.exe' jobs: - template: steps/update-github-status-jobs.yml parameters: @@ -5559,6 +5567,10 @@ stages: vmImage: windows-2022 steps: + - template: steps/install-docker-compose-v1.yml + parameters: + dockerComposePath: $(dockerComposePath) + - template: steps/clone-repo.yml parameters: targetShaId: $(targetShaId) @@ -5579,7 +5591,7 @@ stages: displayName: Create test data directories - bash: | - docker-compose -f docker-compose.windows.yml -p $(DockerComposeProjectName) build \ + $(dockerComposePath) -f docker-compose.windows.yml -p $(DockerComposeProjectName) build \ --build-arg DOTNETSDK_VERSION=$(dotnetCoreSdkLatestVersionShort) \ --build-arg RUNTIME_IMAGE=$(runtimeImage) \ --build-arg PUBLISH_FRAMEWORK=$(publishFramework) \ @@ -5595,6 +5607,7 @@ stages: target: 'dotnet-tool-smoke-tests.windows' snapshotPrefix: "smoke_test" isLinux: false + dockerComposePath: $(dockerComposePath) - publish: tracer/build_data artifact: _$(System.StageName)_$(Agent.JobName)_logs_$(System.JobAttempt) @@ -5611,6 +5624,7 @@ stages: variables: targetShaId: $[ stageDependencies.merge_commit_id.fetch.outputs['set_sha.sha']] targetBranch: $[ stageDependencies.merge_commit_id.fetch.outputs['set_sha.branch']] + dockerComposePath: 'C:/docker-compose/docker-compose.exe' jobs: - template: steps/update-github-status-jobs.yml parameters: @@ -5626,6 +5640,10 @@ stages: vmImage: windows-2022 steps: + - template: steps/install-docker-compose-v1.yml + parameters: + dockerComposePath: $(dockerComposePath) + - template: steps/clone-repo.yml parameters: targetShaId: $(targetShaId) @@ -5646,7 +5664,7 @@ stages: displayName: Create test data directories - bash: | - docker-compose -f docker-compose.windows.yml -p $(DockerComposeProjectName) build \ + $(dockerComposePath) -f docker-compose.windows.yml -p $(DockerComposeProjectName) build \ --build-arg DOTNETSDK_VERSION=$(dotnetCoreSdkLatestVersionShort) \ --build-arg RUNTIME_IMAGE=$(runtimeImage) \ --build-arg PUBLISH_FRAMEWORK=$(publishFramework) \ @@ -5662,6 +5680,7 @@ stages: target: 'smoke-tests.windows' snapshotPrefix: "smoke_test" isLinux: false + dockerComposePath: $(dockerComposePath) - publish: tracer/build_data artifact: _$(System.StageName)_$(Agent.JobName)_logs_$(System.JobAttempt) @@ -5678,6 +5697,7 @@ stages: variables: targetShaId: $[ stageDependencies.merge_commit_id.fetch.outputs['set_sha.sha']] targetBranch: $[ stageDependencies.merge_commit_id.fetch.outputs['set_sha.branch']] + dockerComposePath: 'C:/docker-compose/docker-compose.exe' jobs: - template: steps/update-github-status-jobs.yml parameters: @@ -5694,6 +5714,10 @@ stages: vmImage: windows-2022 steps: + - template: steps/install-docker-compose-v1.yml + parameters: + dockerComposePath: $(dockerComposePath) + - template: steps/clone-repo.yml parameters: targetShaId: $(targetShaId) @@ -5717,7 +5741,7 @@ stages: displayName: Create test data directories - bash: | - docker-compose -f docker-compose.windows.yml -p $(DockerComposeProjectName) build \ + $(dockerComposePath) -f docker-compose.windows.yml -p $(DockerComposeProjectName) build \ --build-arg DOTNETSDK_VERSION=$(dotnetCoreSdkLatestVersionShort) \ --build-arg RUNTIME_IMAGE=$(runtimeImage) \ --build-arg PUBLISH_FRAMEWORK=$(publishFramework) \ @@ -5733,6 +5757,7 @@ stages: target: 'dd-dotnet-smoke-tests.windows' snapshotPrefix: "smoke_test" isLinux: false + dockerComposePath: $(dockerComposePath) - publish: tracer/build_data artifact: _$(System.StageName)_$(Agent.JobName)_logs_$(System.JobAttempt) @@ -5749,6 +5774,7 @@ stages: variables: targetShaId: $[ stageDependencies.merge_commit_id.fetch.outputs['set_sha.sha']] targetBranch: $[ stageDependencies.merge_commit_id.fetch.outputs['set_sha.branch']] + dockerComposePath: 'C:/docker-compose/docker-compose.exe' jobs: - template: steps/update-github-status-jobs.yml parameters: @@ -5761,9 +5787,13 @@ stages: variables: smokeTestAppDir: "$(System.DefaultWorkingDirectory)/tracer/test/test-applications/regression/AspNetCoreSmokeTest" pool: - vmImage: windows-2022 + name: azure-windows-scale-set steps: + - template: steps/install-docker-compose-v1.yml + parameters: + dockerComposePath: $(dockerComposePath) + - template: steps/clone-repo.yml parameters: targetShaId: $(targetShaId) @@ -5782,7 +5812,7 @@ stages: path: $(smokeTestAppDir)/artifacts - bash: | - docker-compose -f docker-compose.windows.yml -p $(DockerComposeProjectName) build \ + $(dockerComposePath) -f docker-compose.windows.yml -p $(DockerComposeProjectName) build \ --build-arg DOTNETSDK_VERSION=$(dotnetCoreSdkLatestVersionShort) \ --build-arg RUNTIME_IMAGE=$(runtimeImage) \ --build-arg PUBLISH_FRAMEWORK=$(publishFramework) \ @@ -5799,6 +5829,7 @@ stages: target: 'tracer-home-smoke-tests.windows' snapshotPrefix: "smoke_test" isLinux: false + dockerComposePath: $(dockerComposePath) - publish: tracer/build_data artifact: _$(System.StageName)_$(Agent.JobName)_logs_$(System.JobAttempt)