diff --git a/.azure-pipelines/steps/install-docker-compose-v1.yml b/.azure-pipelines/steps/install-docker-compose-v1.yml new file mode 100644 index 000000000000..3c61eea2e67c --- /dev/null +++ b/.azure-pipelines/steps/install-docker-compose-v1.yml @@ -0,0 +1,24 @@ +parameters: + - name: isLinux + type: boolean + default: false + + - name: 'dockerComposePath' + type: string + default: 'C:/docker-compose/docker-compose.exe' + +steps: +- ${{ if eq(parameters.isLinux, true) }}: + - bash: | + sudo mkdir -p "$(dirname "${{ parameters.dockerComposePath }}")" + sudo curl -SL https://github.com/docker/compose/releases/download/1.29.2/docker-compose-linux-x86_64 -o ${{ parameters.dockerComposePath }} + sudo chmod 755 ${{ parameters.dockerComposePath }} + displayName: Download docker-compose +- ${{ else }}: + - powershell: | + $dir= (Split-Path -parent "${{ parameters.dockerComposePath }}") + mkdir -f -p $dir + # GitHub now requires TLS1.2. In PowerShell, run the following + [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 + Invoke-WebRequest "https://github.com/docker/compose/releases/download/1.29.1/docker-compose-windows-x86_64.exe" -OutFile "${{ parameters.dockerComposePath }}" + displayName: Download docker-compose diff --git a/.azure-pipelines/steps/run-snapshot-test.yml b/.azure-pipelines/steps/run-snapshot-test.yml index c1da627b5fbb..8bc9fc088202 100644 --- a/.azure-pipelines/steps/run-snapshot-test.yml +++ b/.azure-pipelines/steps/run-snapshot-test.yml @@ -12,6 +12,10 @@ parameters: - name: 'apiKey' type: string default: '' + + - name: 'dockerComposePath' + type: string + default: 'docker-compose' steps: - template: ./clean-docker-containers.yml @@ -56,7 +60,7 @@ steps: displayName: Set env-specific variables - bash: | - docker-compose -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) run --rm $(START_TEST_AGENT_TARGET) + ${{ parameters.dockerComposePath }} -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) run --rm $(START_TEST_AGENT_TARGET) env: dockerTag: $(dockerTag) DD_LOGGER_DD_API_KEY: ${{ parameters.apiKey }} @@ -65,27 +69,27 @@ steps: - script: | echo "Starting snapshot session" - docker-compose -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) exec -T $(TEST_AGENT_TARGET) $(CURL_COMMAND) --fail "http://localhost:8126$(START_ENDPOINT)" + ${{ parameters.dockerComposePath }} -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) exec -T $(TEST_AGENT_TARGET) $(CURL_COMMAND) --fail "http://localhost:8126$(START_ENDPOINT)" displayName: start snapshot session env: DD_LOGGER_DD_API_KEY: ${{ parameters.apiKey }} - bash: | - docker-compose -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) run -e dockerTag=$(dockerTag) ${{ parameters.target }} + ${{ parameters.dockerComposePath }} -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) run -e dockerTag=$(dockerTag) ${{ parameters.target }} env: dockerTag: $(dockerTag) DD_LOGGER_DD_API_KEY: ${{ parameters.apiKey }} - displayName: docker-compose run ${{ parameters.target }} + displayName: ${{ parameters.dockerComposePath }} run ${{ parameters.target }} - script: | echo "Dumping traces" - docker-compose -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) exec -T $(TEST_AGENT_TARGET) $(CURL_COMMAND) -o /debug_snapshots/${{ parameters.snapshotPrefix }}_traces.json "http://localhost:8126$(TRACE_DUMP_ENDPOINT)" + ${{ parameters.dockerComposePath }} -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) exec -T $(TEST_AGENT_TARGET) $(CURL_COMMAND) -o /debug_snapshots/${{ parameters.snapshotPrefix }}_traces.json "http://localhost:8126$(TRACE_DUMP_ENDPOINT)" echo "Dumping stats" - docker-compose -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) exec -T $(TEST_AGENT_TARGET) $(CURL_COMMAND) -o /debug_snapshots/${{ parameters.snapshotPrefix }}_stats.json "http://localhost:8126$(STATS_DUMP_ENDPOINT)" + ${{ parameters.dockerComposePath }} -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) exec -T $(TEST_AGENT_TARGET) $(CURL_COMMAND) -o /debug_snapshots/${{ parameters.snapshotPrefix }}_stats.json "http://localhost:8126$(STATS_DUMP_ENDPOINT)" echo "Dumping all requests" - docker-compose -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) exec -T $(TEST_AGENT_TARGET) $(CURL_COMMAND) -o /debug_snapshots/${{ parameters.snapshotPrefix }}_requests.json "http://localhost:8126$(REQUESTS_DUMP_ENDPOINT)" + ${{ parameters.dockerComposePath }} -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) exec -T $(TEST_AGENT_TARGET) $(CURL_COMMAND) -o /debug_snapshots/${{ parameters.snapshotPrefix }}_requests.json "http://localhost:8126$(REQUESTS_DUMP_ENDPOINT)" displayName: dump snapshots env: DD_LOGGER_DD_API_KEY: ${{ parameters.apiKey }} @@ -93,26 +97,26 @@ steps: - ${{ if eq(parameters.isLinux, true) }}: - bash: | echo "Verifying snapshot session (fail on mis-match)" - docker-compose -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) exec -T $(TEST_AGENT_TARGET) $(CURL_COMMAND) --w '\nGetting a 400 means there is a diff in snapshots. You can diff the files with the artifacts generated. You can also run the tests locally. Follow the doc in /docs/development/CI/RunSmokeTestsLocally\n' --fail "http://localhost:8126$(VERIFY_ENDPOINT)" + ${{ parameters.dockerComposePath }} -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) exec -T $(TEST_AGENT_TARGET) $(CURL_COMMAND) --w '\nGetting a 400 means there is a diff in snapshots. You can diff the files with the artifacts generated. You can also run the tests locally. Follow the doc in /docs/development/CI/RunSmokeTestsLocally\n' --fail "http://localhost:8126$(VERIFY_ENDPOINT)" displayName: check snapshots env: DD_LOGGER_DD_API_KEY: ${{ parameters.apiKey }} - ${{ else }}: - bash: | echo "Verifying snapshot session (fail on mis-match)" - docker-compose -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) exec -T $(TEST_AGENT_TARGET) $(CURL_COMMAND) --fail "http://localhost:8126$(VERIFY_ENDPOINT)" + ${{ parameters.dockerComposePath }} -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) exec -T $(TEST_AGENT_TARGET) $(CURL_COMMAND) --fail "http://localhost:8126$(VERIFY_ENDPOINT)" displayName: check snapshots env: DD_LOGGER_DD_API_KEY: ${{ parameters.apiKey }} -- script: docker-compose -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) logs $(TEST_AGENT_TARGET) +- script: ${{ parameters.dockerComposePath }} -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) logs $(TEST_AGENT_TARGET) displayName: dump docker-compose logs for $(TEST_AGENT_TARGET) env: DD_LOGGER_DD_API_KEY: ${{ parameters.apiKey }} condition: succeededOrFailed() continueOnError: true -- script: docker-compose -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) down +- script: ${{ parameters.dockerComposePath }} -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) down displayName: docker-compose stop services env: DD_LOGGER_DD_API_KEY: ${{ parameters.apiKey }} @@ -122,7 +126,7 @@ steps: # Run crash tests - ${{ if eq(parameters.isLinux, true) }}: - bash: | - LOGS=$(docker-compose -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) run -e dockerTag=$(dockerTag) -e CRASH_APP_ON_STARTUP=1 -e COMPlus_DbgEnableMiniDump=0 ${{ parameters.target }}) + LOGS=$(${{ parameters.dockerComposePath }} -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) run -e dockerTag=$(dockerTag) -e CRASH_APP_ON_STARTUP=1 -e COMPlus_DbgEnableMiniDump=0 ${{ parameters.target }}) echo $LOGS # check logs for evidence of crash detection in the output @@ -144,4 +148,12 @@ steps: - script: | sudo chmod -R 644 tracer/build_data/dumps/* || true displayName: Make dumps uploadable to AzDo - condition: succeededOrFailed() \ No newline at end of file + condition: succeededOrFailed() + +- script: | + docker network prune -f + displayName: Clean up docker networks + condition: succeededOrFailed() + continueOnError: true + + \ No newline at end of file diff --git a/.azure-pipelines/ultimate-pipeline.yml b/.azure-pipelines/ultimate-pipeline.yml index 10c1649464b0..97bd1ff546c1 100644 --- a/.azure-pipelines/ultimate-pipeline.yml +++ b/.azure-pipelines/ultimate-pipeline.yml @@ -5454,6 +5454,7 @@ stages: variables: targetShaId: $[ stageDependencies.merge_commit_id.fetch.outputs['set_sha.sha']] targetBranch: $[ stageDependencies.merge_commit_id.fetch.outputs['set_sha.branch']] + dockerComposePath: 'C:/docker-compose/docker-compose.exe' jobs: - template: steps/update-github-status-jobs.yml parameters: @@ -5469,6 +5470,10 @@ stages: vmImage: windows-2022 steps: + - template: steps/install-docker-compose-v1.yml + parameters: + dockerComposePath: $(dockerComposePath) + - template: steps/clone-repo.yml parameters: targetShaId: $(targetShaId) @@ -5492,7 +5497,7 @@ stages: displayName: create test data directories - bash: | - docker-compose -f docker-compose.windows.yml -p $(DockerComposeProjectName) build \ + $(dockerComposePath) -f docker-compose.windows.yml -p $(DockerComposeProjectName) build \ --build-arg DOTNETSDK_VERSION=$(dotnetCoreSdkLatestVersionShort) \ --build-arg RUNTIME_IMAGE=$(runtimeImage) \ --build-arg PUBLISH_FRAMEWORK=$(publishFramework) \ @@ -5509,9 +5514,10 @@ stages: target: 'nuget-smoke-tests.windows' snapshotPrefix: "smoke_test" isLinux: false + dockerComposePath: $(dockerComposePath) - bash: | - docker-compose -f docker-compose.windows.yml -p $(DockerComposeProjectName) build \ + $(dockerComposePath) -f docker-compose.windows.yml -p $(DockerComposeProjectName) build \ --build-arg DOTNETSDK_VERSION=$(dotnetCoreSdkLatestVersionShort) \ --build-arg RUNTIME_IMAGE=$(runtimeImage) \ --build-arg PUBLISH_FRAMEWORK=$(publishFramework) \ @@ -5528,6 +5534,7 @@ stages: target: 'nuget-dddotnet-smoke-tests.windows' snapshotPrefix: "smoke_test" isLinux: false + dockerComposePath: $(dockerComposePath) - publish: tracer/build_data artifact: _$(System.StageName)_$(Agent.JobName)_logs_$(System.JobAttempt) @@ -5544,6 +5551,7 @@ stages: variables: targetShaId: $[ stageDependencies.merge_commit_id.fetch.outputs['set_sha.sha']] targetBranch: $[ stageDependencies.merge_commit_id.fetch.outputs['set_sha.branch']] + dockerComposePath: 'C:/docker-compose/docker-compose.exe' jobs: - template: steps/update-github-status-jobs.yml parameters: @@ -5559,6 +5567,10 @@ stages: vmImage: windows-2022 steps: + - template: steps/install-docker-compose-v1.yml + parameters: + dockerComposePath: $(dockerComposePath) + - template: steps/clone-repo.yml parameters: targetShaId: $(targetShaId) @@ -5579,7 +5591,7 @@ stages: displayName: Create test data directories - bash: | - docker-compose -f docker-compose.windows.yml -p $(DockerComposeProjectName) build \ + $(dockerComposePath) -f docker-compose.windows.yml -p $(DockerComposeProjectName) build \ --build-arg DOTNETSDK_VERSION=$(dotnetCoreSdkLatestVersionShort) \ --build-arg RUNTIME_IMAGE=$(runtimeImage) \ --build-arg PUBLISH_FRAMEWORK=$(publishFramework) \ @@ -5595,6 +5607,7 @@ stages: target: 'dotnet-tool-smoke-tests.windows' snapshotPrefix: "smoke_test" isLinux: false + dockerComposePath: $(dockerComposePath) - publish: tracer/build_data artifact: _$(System.StageName)_$(Agent.JobName)_logs_$(System.JobAttempt) @@ -5611,6 +5624,7 @@ stages: variables: targetShaId: $[ stageDependencies.merge_commit_id.fetch.outputs['set_sha.sha']] targetBranch: $[ stageDependencies.merge_commit_id.fetch.outputs['set_sha.branch']] + dockerComposePath: 'C:/docker-compose/docker-compose.exe' jobs: - template: steps/update-github-status-jobs.yml parameters: @@ -5626,6 +5640,10 @@ stages: vmImage: windows-2022 steps: + - template: steps/install-docker-compose-v1.yml + parameters: + dockerComposePath: $(dockerComposePath) + - template: steps/clone-repo.yml parameters: targetShaId: $(targetShaId) @@ -5646,7 +5664,7 @@ stages: displayName: Create test data directories - bash: | - docker-compose -f docker-compose.windows.yml -p $(DockerComposeProjectName) build \ + $(dockerComposePath) -f docker-compose.windows.yml -p $(DockerComposeProjectName) build \ --build-arg DOTNETSDK_VERSION=$(dotnetCoreSdkLatestVersionShort) \ --build-arg RUNTIME_IMAGE=$(runtimeImage) \ --build-arg PUBLISH_FRAMEWORK=$(publishFramework) \ @@ -5662,6 +5680,7 @@ stages: target: 'smoke-tests.windows' snapshotPrefix: "smoke_test" isLinux: false + dockerComposePath: $(dockerComposePath) - publish: tracer/build_data artifact: _$(System.StageName)_$(Agent.JobName)_logs_$(System.JobAttempt) @@ -5678,6 +5697,7 @@ stages: variables: targetShaId: $[ stageDependencies.merge_commit_id.fetch.outputs['set_sha.sha']] targetBranch: $[ stageDependencies.merge_commit_id.fetch.outputs['set_sha.branch']] + dockerComposePath: 'C:/docker-compose/docker-compose.exe' jobs: - template: steps/update-github-status-jobs.yml parameters: @@ -5694,6 +5714,10 @@ stages: vmImage: windows-2022 steps: + - template: steps/install-docker-compose-v1.yml + parameters: + dockerComposePath: $(dockerComposePath) + - template: steps/clone-repo.yml parameters: targetShaId: $(targetShaId) @@ -5717,7 +5741,7 @@ stages: displayName: Create test data directories - bash: | - docker-compose -f docker-compose.windows.yml -p $(DockerComposeProjectName) build \ + $(dockerComposePath) -f docker-compose.windows.yml -p $(DockerComposeProjectName) build \ --build-arg DOTNETSDK_VERSION=$(dotnetCoreSdkLatestVersionShort) \ --build-arg RUNTIME_IMAGE=$(runtimeImage) \ --build-arg PUBLISH_FRAMEWORK=$(publishFramework) \ @@ -5733,6 +5757,7 @@ stages: target: 'dd-dotnet-smoke-tests.windows' snapshotPrefix: "smoke_test" isLinux: false + dockerComposePath: $(dockerComposePath) - publish: tracer/build_data artifact: _$(System.StageName)_$(Agent.JobName)_logs_$(System.JobAttempt) @@ -5749,6 +5774,7 @@ stages: variables: targetShaId: $[ stageDependencies.merge_commit_id.fetch.outputs['set_sha.sha']] targetBranch: $[ stageDependencies.merge_commit_id.fetch.outputs['set_sha.branch']] + dockerComposePath: 'C:/docker-compose/docker-compose.exe' jobs: - template: steps/update-github-status-jobs.yml parameters: @@ -5761,9 +5787,13 @@ stages: variables: smokeTestAppDir: "$(System.DefaultWorkingDirectory)/tracer/test/test-applications/regression/AspNetCoreSmokeTest" pool: - vmImage: windows-2022 + name: azure-windows-scale-set steps: + - template: steps/install-docker-compose-v1.yml + parameters: + dockerComposePath: $(dockerComposePath) + - template: steps/clone-repo.yml parameters: targetShaId: $(targetShaId) @@ -5782,7 +5812,7 @@ stages: path: $(smokeTestAppDir)/artifacts - bash: | - docker-compose -f docker-compose.windows.yml -p $(DockerComposeProjectName) build \ + $(dockerComposePath) -f docker-compose.windows.yml -p $(DockerComposeProjectName) build \ --build-arg DOTNETSDK_VERSION=$(dotnetCoreSdkLatestVersionShort) \ --build-arg RUNTIME_IMAGE=$(runtimeImage) \ --build-arg PUBLISH_FRAMEWORK=$(publishFramework) \ @@ -5799,6 +5829,7 @@ stages: target: 'tracer-home-smoke-tests.windows' snapshotPrefix: "smoke_test" isLinux: false + dockerComposePath: $(dockerComposePath) - publish: tracer/build_data artifact: _$(System.StageName)_$(Agent.JobName)_logs_$(System.JobAttempt)