Try fix smoke tests (#5889)

## Summary of changes - Install v1 docker-compose in the hosted runners used for Windows smoke tests - Cleanup the docker networks in the smoke tests ## Reason for change Microsoft [recently](https://learn.microsoft.com/en-us/azure/devops/release-notes/2024/pipelines/sprint-240-update#dockercompose0-uses-docker-compose-v2-in-v1-compatibility-mode) stopped shipping the v1 of docker compose. That broke all our Windows smoke tests. Unfortunately, for various other reasons, we couldn't just convert to the v2 format, due to how we're reusing various bits and pieces. Additionally, we started getting this on our linux smoke tests: ``` Creating network "ddtrace_20240812_56_default" with the default driver could not find an available, non-overlapping IPv4 address pool among the defaults to assign to the network ``` This is basically because we're creating networks, and never cleaning them up, and for efficiency reasons we're not destroying the smoke test VMs after each run. ## Implementation details - Install Docker-compose v1 in the Windows VMs. A bit hacky, but the quickest solution right now - Clean up the docker network after running the snapshot tests ## Test coverage I did some manual tests to make sure this all works as expected. Also killed and restored the smoke tests machines which solved the docker network problem temporarily. ## Other details This is obviously still all kinda horrible. I'd love to replace all this yaml with just testcontainers. One day.... _stares wistfully into the distance_
DataDog · Aug 13, 2024 · 6dbcf21 · 6dbcf21
1 parent 47f3013
commit 6dbcf21
Show file tree

Hide file tree

Showing 3 changed files with 87 additions and 20 deletions.
diff --git a/.azure-pipelines/steps/install-docker-compose-v1.yml b/.azure-pipelines/steps/install-docker-compose-v1.yml
@@ -0,0 +1,24 @@
+parameters:
+  - name: isLinux
+    type: boolean
+    default: false
+
+  - name: 'dockerComposePath'
+    type: string
+    default: 'C:/docker-compose/docker-compose.exe'
+
+steps:
+- ${{ if eq(parameters.isLinux, true) }}:
+  - bash: |
+      sudo mkdir -p "$(dirname "${{ parameters.dockerComposePath }}")" 
+      sudo curl -SL https://github.com/docker/compose/releases/download/1.29.2/docker-compose-linux-x86_64 -o ${{ parameters.dockerComposePath }}
+      sudo chmod 755 ${{ parameters.dockerComposePath }}
+    displayName: Download docker-compose
+- ${{ else }}:
+  - powershell: |
+      $dir= (Split-Path -parent "${{ parameters.dockerComposePath }}")
+      mkdir -f -p $dir
+      # GitHub now requires TLS1.2. In PowerShell, run the following
+      [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
+      Invoke-WebRequest "https://github.com/docker/compose/releases/download/1.29.1/docker-compose-windows-x86_64.exe" -OutFile "${{ parameters.dockerComposePath }}"
+    displayName: Download docker-compose
diff --git a/.azure-pipelines/steps/run-snapshot-test.yml b/.azure-pipelines/steps/run-snapshot-test.yml
@@ -12,6 +12,10 @@ parameters:
   - name: 'apiKey'
     type: string
     default: ''
+
+  - name: 'dockerComposePath'
+    type: string
+    default: 'docker-compose'
 
 steps:
 - template: ./clean-docker-containers.yml
@@ -56,7 +60,7 @@ steps:
     displayName: Set env-specific  variables
 
 - bash: |
-    docker-compose -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) run --rm $(START_TEST_AGENT_TARGET)
+    ${{ parameters.dockerComposePath }} -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) run --rm $(START_TEST_AGENT_TARGET)
   env:
     dockerTag: $(dockerTag)
     DD_LOGGER_DD_API_KEY: ${{ parameters.apiKey }}
@@ -65,54 +69,54 @@ steps:
 
 - script: |
     echo "Starting snapshot session"
-    docker-compose -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) exec -T $(TEST_AGENT_TARGET) $(CURL_COMMAND) --fail "http://localhost:8126$(START_ENDPOINT)"
+    ${{ parameters.dockerComposePath }} -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) exec -T $(TEST_AGENT_TARGET) $(CURL_COMMAND) --fail "http://localhost:8126$(START_ENDPOINT)"
   displayName: start snapshot session
   env:
     DD_LOGGER_DD_API_KEY: ${{ parameters.apiKey }}
 
 - bash: |
-    docker-compose -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) run -e dockerTag=$(dockerTag) ${{ parameters.target }}
+    ${{ parameters.dockerComposePath }} -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) run -e dockerTag=$(dockerTag) ${{ parameters.target }}
   env:
     dockerTag: $(dockerTag)
     DD_LOGGER_DD_API_KEY: ${{ parameters.apiKey }}
-  displayName: docker-compose run ${{ parameters.target }}
+  displayName: ${{ parameters.dockerComposePath }} run ${{ parameters.target }}
 
 - script: |
     echo "Dumping traces"
-    docker-compose -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) exec -T $(TEST_AGENT_TARGET) $(CURL_COMMAND) -o /debug_snapshots/${{ parameters.snapshotPrefix }}_traces.json "http://localhost:8126$(TRACE_DUMP_ENDPOINT)"
+    ${{ parameters.dockerComposePath }} -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) exec -T $(TEST_AGENT_TARGET) $(CURL_COMMAND) -o /debug_snapshots/${{ parameters.snapshotPrefix }}_traces.json "http://localhost:8126$(TRACE_DUMP_ENDPOINT)"
     
     echo "Dumping stats"
-    docker-compose -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) exec -T $(TEST_AGENT_TARGET) $(CURL_COMMAND) -o /debug_snapshots/${{ parameters.snapshotPrefix }}_stats.json "http://localhost:8126$(STATS_DUMP_ENDPOINT)"
+    ${{ parameters.dockerComposePath }} -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) exec -T $(TEST_AGENT_TARGET) $(CURL_COMMAND) -o /debug_snapshots/${{ parameters.snapshotPrefix }}_stats.json "http://localhost:8126$(STATS_DUMP_ENDPOINT)"
     
     echo "Dumping all requests"
-    docker-compose -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) exec -T $(TEST_AGENT_TARGET) $(CURL_COMMAND) -o /debug_snapshots/${{ parameters.snapshotPrefix }}_requests.json "http://localhost:8126$(REQUESTS_DUMP_ENDPOINT)"
+    ${{ parameters.dockerComposePath }} -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) exec -T $(TEST_AGENT_TARGET) $(CURL_COMMAND) -o /debug_snapshots/${{ parameters.snapshotPrefix }}_requests.json "http://localhost:8126$(REQUESTS_DUMP_ENDPOINT)"
   displayName: dump snapshots
   env:
     DD_LOGGER_DD_API_KEY: ${{ parameters.apiKey }}
 
 - ${{ if eq(parameters.isLinux, true) }}:
   - bash: |
       echo "Verifying snapshot session (fail on mis-match)"
-      docker-compose -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) exec -T $(TEST_AGENT_TARGET) $(CURL_COMMAND) --w '\nGetting a 400 means there is a diff in snapshots. You can diff the files with the artifacts generated. You can also run the tests locally. Follow the doc in /docs/development/CI/RunSmokeTestsLocally\n' --fail "http://localhost:8126$(VERIFY_ENDPOINT)"
+      ${{ parameters.dockerComposePath }} -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) exec -T $(TEST_AGENT_TARGET) $(CURL_COMMAND) --w '\nGetting a 400 means there is a diff in snapshots. You can diff the files with the artifacts generated. You can also run the tests locally. Follow the doc in /docs/development/CI/RunSmokeTestsLocally\n' --fail "http://localhost:8126$(VERIFY_ENDPOINT)"
     displayName: check snapshots
     env:
       DD_LOGGER_DD_API_KEY: ${{ parameters.apiKey }}
 - ${{ else }}:
   - bash: |
       echo "Verifying snapshot session (fail on mis-match)"
-      docker-compose -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) exec -T $(TEST_AGENT_TARGET) $(CURL_COMMAND) --fail "http://localhost:8126$(VERIFY_ENDPOINT)"
+      ${{ parameters.dockerComposePath }} -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) exec -T $(TEST_AGENT_TARGET) $(CURL_COMMAND) --fail "http://localhost:8126$(VERIFY_ENDPOINT)"
     displayName: check snapshots
     env:
       DD_LOGGER_DD_API_KEY: ${{ parameters.apiKey }}
 
-- script: docker-compose -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) logs $(TEST_AGENT_TARGET)
+- script: ${{ parameters.dockerComposePath }} -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) logs $(TEST_AGENT_TARGET)
   displayName: dump docker-compose logs for $(TEST_AGENT_TARGET)
   env:
     DD_LOGGER_DD_API_KEY: ${{ parameters.apiKey }}
   condition: succeededOrFailed()
   continueOnError: true
 
-- script: docker-compose -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) down
+- script: ${{ parameters.dockerComposePath }} -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) down
   displayName: docker-compose stop services
   env:
     DD_LOGGER_DD_API_KEY: ${{ parameters.apiKey }}
@@ -122,7 +126,7 @@ steps:
 # Run crash tests
 - ${{ if eq(parameters.isLinux, true) }}:
   - bash: |
-      LOGS=$(docker-compose -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) run -e dockerTag=$(dockerTag) -e CRASH_APP_ON_STARTUP=1 -e COMPlus_DbgEnableMiniDump=0 ${{ parameters.target }})
+      LOGS=$(${{ parameters.dockerComposePath }} -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) run -e dockerTag=$(dockerTag) -e CRASH_APP_ON_STARTUP=1 -e COMPlus_DbgEnableMiniDump=0 ${{ parameters.target }})
       echo $LOGS
 
       # check logs for evidence of crash detection in the output
@@ -144,4 +148,12 @@ steps:
     - script: |
         sudo chmod -R 644 tracer/build_data/dumps/* || true
       displayName: Make dumps uploadable to AzDo
-      condition: succeededOrFailed()
+      condition: succeededOrFailed()
+
+- script: |
+    docker network prune -f
+  displayName: Clean up docker networks
+  condition: succeededOrFailed()
+  continueOnError: true
+
+
diff --git a/.azure-pipelines/ultimate-pipeline.yml b/.azure-pipelines/ultimate-pipeline.yml
@@ -5454,6 +5454,7 @@ stages:
   variables:
     targetShaId: $[ stageDependencies.merge_commit_id.fetch.outputs['set_sha.sha']]
     targetBranch: $[ stageDependencies.merge_commit_id.fetch.outputs['set_sha.branch']]
+    dockerComposePath: 'C:/docker-compose/docker-compose.exe'
   jobs:
   - template: steps/update-github-status-jobs.yml
     parameters:
@@ -5469,6 +5470,10 @@ stages:
       vmImage: windows-2022
 
     steps:
+    - template: steps/install-docker-compose-v1.yml
+      parameters:
+        dockerComposePath: $(dockerComposePath)
+
     - template: steps/clone-repo.yml
       parameters:
         targetShaId: $(targetShaId)
@@ -5492,7 +5497,7 @@ stages:
       displayName: create test data directories
 
     - bash: |
-        docker-compose -f docker-compose.windows.yml -p $(DockerComposeProjectName) build \
+        $(dockerComposePath) -f docker-compose.windows.yml -p $(DockerComposeProjectName) build \
           --build-arg DOTNETSDK_VERSION=$(dotnetCoreSdkLatestVersionShort) \
           --build-arg RUNTIME_IMAGE=$(runtimeImage) \
           --build-arg PUBLISH_FRAMEWORK=$(publishFramework) \
@@ -5509,9 +5514,10 @@ stages:
         target: 'nuget-smoke-tests.windows'
         snapshotPrefix: "smoke_test"
         isLinux: false
+        dockerComposePath: $(dockerComposePath)
 
     - bash: |
-        docker-compose -f docker-compose.windows.yml -p $(DockerComposeProjectName) build \
+        $(dockerComposePath) -f docker-compose.windows.yml -p $(DockerComposeProjectName) build \
           --build-arg DOTNETSDK_VERSION=$(dotnetCoreSdkLatestVersionShort) \
           --build-arg RUNTIME_IMAGE=$(runtimeImage) \
           --build-arg PUBLISH_FRAMEWORK=$(publishFramework) \
@@ -5528,6 +5534,7 @@ stages:
         target: 'nuget-dddotnet-smoke-tests.windows'
         snapshotPrefix: "smoke_test"
         isLinux: false
+        dockerComposePath: $(dockerComposePath)
 
     - publish: tracer/build_data
       artifact: _$(System.StageName)_$(Agent.JobName)_logs_$(System.JobAttempt)
@@ -5544,6 +5551,7 @@ stages:
   variables:
     targetShaId: $[ stageDependencies.merge_commit_id.fetch.outputs['set_sha.sha']]
     targetBranch: $[ stageDependencies.merge_commit_id.fetch.outputs['set_sha.branch']]
+    dockerComposePath: 'C:/docker-compose/docker-compose.exe'
   jobs:
   - template: steps/update-github-status-jobs.yml
     parameters:
@@ -5559,6 +5567,10 @@ stages:
       vmImage: windows-2022
 
     steps:
+    - template: steps/install-docker-compose-v1.yml
+      parameters:
+        dockerComposePath: $(dockerComposePath)
+
     - template: steps/clone-repo.yml
       parameters:
         targetShaId: $(targetShaId)
@@ -5579,7 +5591,7 @@ stages:
       displayName: Create test data directories
 
     - bash: |
-        docker-compose -f docker-compose.windows.yml -p $(DockerComposeProjectName) build \
+        $(dockerComposePath) -f docker-compose.windows.yml -p $(DockerComposeProjectName) build \
           --build-arg DOTNETSDK_VERSION=$(dotnetCoreSdkLatestVersionShort) \
           --build-arg RUNTIME_IMAGE=$(runtimeImage) \
           --build-arg PUBLISH_FRAMEWORK=$(publishFramework) \
@@ -5595,6 +5607,7 @@ stages:
         target: 'dotnet-tool-smoke-tests.windows'
         snapshotPrefix: "smoke_test"
         isLinux: false
+        dockerComposePath: $(dockerComposePath)
 
     - publish: tracer/build_data
       artifact: _$(System.StageName)_$(Agent.JobName)_logs_$(System.JobAttempt)
@@ -5611,6 +5624,7 @@ stages:
   variables:
     targetShaId: $[ stageDependencies.merge_commit_id.fetch.outputs['set_sha.sha']]
     targetBranch: $[ stageDependencies.merge_commit_id.fetch.outputs['set_sha.branch']]
+    dockerComposePath: 'C:/docker-compose/docker-compose.exe'
   jobs:
   - template: steps/update-github-status-jobs.yml
     parameters:
@@ -5626,6 +5640,10 @@ stages:
       vmImage: windows-2022
 
     steps:
+    - template: steps/install-docker-compose-v1.yml
+      parameters:
+        dockerComposePath: $(dockerComposePath)
+
     - template: steps/clone-repo.yml
       parameters:
         targetShaId: $(targetShaId)
@@ -5646,7 +5664,7 @@ stages:
       displayName: Create test data directories
 
     - bash: |
-        docker-compose -f docker-compose.windows.yml -p $(DockerComposeProjectName) build \
+        $(dockerComposePath) -f docker-compose.windows.yml -p $(DockerComposeProjectName) build \
           --build-arg DOTNETSDK_VERSION=$(dotnetCoreSdkLatestVersionShort) \
           --build-arg RUNTIME_IMAGE=$(runtimeImage) \
           --build-arg PUBLISH_FRAMEWORK=$(publishFramework) \
@@ -5662,6 +5680,7 @@ stages:
         target: 'smoke-tests.windows'
         snapshotPrefix: "smoke_test"
         isLinux: false
+        dockerComposePath: $(dockerComposePath)
 
     - publish: tracer/build_data
       artifact: _$(System.StageName)_$(Agent.JobName)_logs_$(System.JobAttempt)
@@ -5678,6 +5697,7 @@ stages:
   variables:
     targetShaId: $[ stageDependencies.merge_commit_id.fetch.outputs['set_sha.sha']]
     targetBranch: $[ stageDependencies.merge_commit_id.fetch.outputs['set_sha.branch']]
+    dockerComposePath: 'C:/docker-compose/docker-compose.exe'
   jobs:
     - template: steps/update-github-status-jobs.yml
       parameters:
@@ -5694,6 +5714,10 @@ stages:
         vmImage: windows-2022
 
       steps:
+        - template: steps/install-docker-compose-v1.yml
+          parameters:
+            dockerComposePath: $(dockerComposePath)
+
         - template: steps/clone-repo.yml
           parameters:
             targetShaId: $(targetShaId)
@@ -5717,7 +5741,7 @@ stages:
           displayName: Create test data directories
 
         - bash: |
-            docker-compose -f docker-compose.windows.yml -p $(DockerComposeProjectName) build \
+            $(dockerComposePath) -f docker-compose.windows.yml -p $(DockerComposeProjectName) build \
               --build-arg DOTNETSDK_VERSION=$(dotnetCoreSdkLatestVersionShort) \
               --build-arg RUNTIME_IMAGE=$(runtimeImage) \
               --build-arg PUBLISH_FRAMEWORK=$(publishFramework) \
@@ -5733,6 +5757,7 @@ stages:
             target: 'dd-dotnet-smoke-tests.windows'
             snapshotPrefix: "smoke_test"
             isLinux: false
+            dockerComposePath: $(dockerComposePath)
 
         - publish: tracer/build_data
           artifact: _$(System.StageName)_$(Agent.JobName)_logs_$(System.JobAttempt)
@@ -5749,6 +5774,7 @@ stages:
   variables:
     targetShaId: $[ stageDependencies.merge_commit_id.fetch.outputs['set_sha.sha']]
     targetBranch: $[ stageDependencies.merge_commit_id.fetch.outputs['set_sha.branch']]
+    dockerComposePath: 'C:/docker-compose/docker-compose.exe'
   jobs:
   - template: steps/update-github-status-jobs.yml
     parameters:
@@ -5761,9 +5787,13 @@ stages:
     variables:
       smokeTestAppDir: "$(System.DefaultWorkingDirectory)/tracer/test/test-applications/regression/AspNetCoreSmokeTest"
     pool:
-      vmImage: windows-2022
+      name: azure-windows-scale-set
 
     steps:
+    - template: steps/install-docker-compose-v1.yml
+      parameters:
+        dockerComposePath: $(dockerComposePath)
+
     - template: steps/clone-repo.yml
       parameters:
         targetShaId: $(targetShaId)
@@ -5782,7 +5812,7 @@ stages:
         path: $(smokeTestAppDir)/artifacts
 
     - bash: |
-        docker-compose -f docker-compose.windows.yml -p $(DockerComposeProjectName) build \
+        $(dockerComposePath) -f docker-compose.windows.yml -p $(DockerComposeProjectName) build \
           --build-arg DOTNETSDK_VERSION=$(dotnetCoreSdkLatestVersionShort) \
           --build-arg RUNTIME_IMAGE=$(runtimeImage) \
           --build-arg PUBLISH_FRAMEWORK=$(publishFramework) \
@@ -5799,6 +5829,7 @@ stages:
         target: 'tracer-home-smoke-tests.windows'
         snapshotPrefix: "smoke_test"
         isLinux: false
+        dockerComposePath: $(dockerComposePath)
 
     - publish: tracer/build_data
       artifact: _$(System.StageName)_$(Agent.JobName)_logs_$(System.JobAttempt)