[Azure Pipelines] Run infrastructure/ tests on Windows 10 (#14300)

This job is only triggered manually until it has been shown to be robust.

.azure-pipelines.yml

@@ -13,6 +13,11 @@
 #  - A scheduled build needs to be set up for the the epochs/daily branch.
 #  - To get results from scheduled builds into wpt.fyi, a service connection
 #    named wpt.fyi with URL https://wpt.fyi is needed.
+#  - Self-hosted agents for Windows 10 are used:
+#    - 'Hosted Windows Client' is the latest Windows 10
+#    - 'Hosted Windows Client Next' is Windows 10 Insider Preview
+#    Documention for the setup of these agents:
+#    https://docs.microsoft.com/en-us/azure/devops/pipelines/agents/v2-windows
 
 jobs:
 # The affected tests jobs are unconditional for speed, as most PRs have one or
@@ -127,6 +132,31 @@ jobs:
     parameters:
       directory: tools/wpt/
 
+- job: infrastructure_win10
+  displayName: 'infrastructure/ tests (Windows 10)'
+  # This job is only triggered manually until it has been shown to be robust.
+  condition: eq(variables['Build.Reason'], 'Manual')
+  pool:
+    name: 'Hosted Windows Client'
+  steps:
+  - template: tools/ci/azure/system_info.yml
+  - template: tools/ci/azure/checkout.yml
+  - template: tools/ci/azure/install_python.yml
+  - template: tools/ci/azure/pip_install.yml
+    parameters:
+      packages: virtualenv
+  - template: tools/ci/azure/install_certs.yml
+  - template: tools/ci/azure/update_hosts.yml
+  - template: tools/ci/azure/update_manifest.yml
+  - script: python ./wpt run --yes --no-manifest-update --install-fonts --manifest MANIFEST.json --metadata infrastructure/metadata/ --log-tbpl $(Build.ArtifactStagingDirectory)/edge.tbpl.log --log-tbpl-level info edge_webdriver infrastructure/
+    displayName: 'Run tests (Edge)'
+  - task: PublishBuildArtifacts@1
+    displayName: 'Publish results'
+    inputs:
+      artifactName: 'infrastructure'
+    condition: succeededOrFailed()
+  - template: tools/ci/azure/cleanup_win10.yml
+
 - job: all_macOS
   displayName: 'all tests (Safari Technology Preview)'
   condition: eq(variables['Build.Reason'], 'Schedule')

infrastructure/metadata/infrastructure/assumptions/allowed-to-play.html.ini

@@ -1,8 +1,10 @@
 [allowed-to-play.html]
   expected:
+    if product == "edge_webdriver": TIMEOUT
     if product == "safari": ERROR # https://bugs.webkit.org/show_bug.cgi?id=190775
 
 
   [<audio> autoplay]
     expected:
+      if product == "edge_webdriver": TIMEOUT
       if product == "safari": FAIL # https://bugs.webkit.org/show_bug.cgi?id=190775

infrastructure/metadata/infrastructure/assumptions/html-elements.html.ini

@@ -1,4 +1,9 @@
 [html-elements.html]
+  [(pre-req for comparison tests) all CSS short-hand supported]
+    expected:
+      if product == "edge_webdriver": FAIL
+
+
   [Compare CSS span definitions (only valid if pre-reqs pass)]
     expected:
       if product == "safari": FAIL # https://webkit.org/show_bug.cgi?id=187052

infrastructure/metadata/infrastructure/reftest/size.html.ini

@@ -0,0 +1,4 @@
+[size.html]
+  type: reftest
+  expected:
+    if product == "edge_webdriver": FAIL # https://github.com/web-platform-tests/wpt/issues/15159

infrastructure/metadata/infrastructure/server/context.any.js.ini

@@ -1,4 +1,5 @@
 [context.any.sharedworker.html]
   [context]
     expected:
+      if product == "edge_webdriver": FAIL
       if product == "safari": FAIL # https://bugs.webkit.org/show_bug.cgi?id=149850

infrastructure/metadata/infrastructure/server/order-of-metas.any.js.ini

@@ -1,4 +1,27 @@
+[order-of-metas.any.worker.html]
+  expected:
+    if product == "edge_webdriver": TIMEOUT
+
+
+[order-of-metas.any.html]
+  [<meta name=timeout> exists]
+    expected:
+      if product == "edge_webdriver": FAIL
+
+
 [order-of-metas.any.sharedworker.html]
+  [foo\n]
+    expected:
+      if product == "edge_webdriver": FAIL
+
+
   [foo]
     expected:
+      if product == "edge_webdriver": FAIL
       if product == "safari": FAIL # https://bugs.webkit.org/show_bug.cgi?id=149850
+
+
+[order-of-metas.window.html]
+  [<meta name=timeout> exists]
+    expected:
+      if product == "edge_webdriver": FAIL

infrastructure/metadata/infrastructure/server/order-of-metas.window.js.ini

@@ -0,0 +1,4 @@
+[order-of-metas.window.html]
+  [<meta name=timeout> exists]
+    expected:
+      if product == "edge_webdriver": FAIL

infrastructure/metadata/infrastructure/server/secure-context.https.any.js.ini

@@ -1,4 +1,5 @@
 [secure-context.https.any.sharedworker.html]
   [secure-context]
     expected:
+      if product == "edge_webdriver": FAIL
       if product == "safari": FAIL # https://bugs.webkit.org/show_bug.cgi?id=149850

infrastructure/metadata/infrastructure/server/title.any.js.ini

@@ -1,4 +1,20 @@
+[title.any.html]
+  [foobar\n]
+    expected:
+      if product == "edge_webdriver": FAIL
+
+
 [title.any.sharedworker.html]
+  [foobar\n]
+    expected:
+      if product == "edge_webdriver": FAIL
+
+
   [foobar]
     expected:
       if product == "safari": FAIL # https://bugs.webkit.org/show_bug.cgi?id=149850
+
+
+[title.any.worker.html]
+  expected:
+    if product == "edge_webdriver": TIMEOUT

infrastructure/metadata/infrastructure/testdriver/file_upload.sub.html.ini

@@ -0,0 +1,3 @@
+[file_upload.sub.html]
+  expected:
+    if product == "edge_webdriver": ERROR

tools/ci/azure/cleanup_win10.yml

@@ -0,0 +1,13 @@
+# This is cleanup for update_hosts.yml and killing all Edge and WebDriver
+# processes to leave the agent in a clean state for the next job. This is
+# necessary because the Windows 10 agents aren't reset between each job.
+steps:
+- powershell: |
+    $hostFile = "$env:systemroot\System32\drivers\etc\hosts"
+    Copy-Item -Path "$hostFile.back" -Destination $hostFile -Force
+    taskkill /f /im MicrosoftEdge*
+    taskkill /f /im MicrosoftWebDriver.exe
+  displayName: 'Restore hosts file and cleanup test machine'
+  condition: always()
+  errorActionPreference: silentlyContinue
+  ignoreLASTEXITCODE: true

tools/ci/azure/install_certs.yml

@@ -2,4 +2,8 @@ steps:
 - script: |
     # https://github.com/web-platform-tests/results-collection/blob/master/src/scripts/trust-root-ca.sh
     sudo security add-trusted-cert -d -r trustRoot -k /Library/Keychains/System.keychain tools/certs/cacert.pem
-  displayName: 'Install web-platform.test certificate'
+  displayName: 'Install web-platform.test certificate (macOS)'
+  condition: and(succeeded(), eq(variables['Agent.OS'], 'Darwin'))
+- script: certutil –addstore -enterprise –f "Root" tools\certs\cacert.pem
+  displayName: 'Install web-platform.test certificate (Windows)'
+  condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'))

tools/ci/azure/install_python.yml

@@ -0,0 +1,15 @@
+steps:
+- powershell: |
+    Start-BitsTransfer -Source https://www.python.org/ftp/python/2.7.15/python-2.7.15.amd64.msi -Destination python.msi
+    $hash = (Get-FileHash python.msi).Hash.ToLower()
+    $expectedHash = "5e85f3c4c209de98480acbf2ba2e71a907fd5567a838ad4b6748c76deb286ad7"
+    if ($hash.Equals($expectedHash)) {
+      Write-Host "python.msi hash verified"
+    } else {
+      Throw "python.msi hash mismatch, got $hash, expected $expectedHash"
+    }
+    msiexec /i python.msi TARGETDIR=C:\Python27 /q
+    Write-Host "##vso[task.prependpath]C:\Python27"
+    Write-Host "##vso[task.prependpath]C:\Python27\Scripts"
+  displayName: 'Install Python (Windows)'
+  condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'))

tools/ci/azure/pip_install.yml

@@ -9,4 +9,11 @@ steps:
     # Integrity Protection, see https://stackoverflow.com/a/33004920.
     pip install --user ${{ parameters.packages }}
     echo "##vso[task.prependpath]$HOME/Library/Python/2.7/bin"
-  displayName: 'Install Python packages'
+  displayName: 'Install Python packages (macOS)'
+  condition: and(succeeded(), eq(variables['Agent.OS'], 'Darwin'))
+  # TODO(foolip): remove --no-cache-dir when the Windows agents are reset
+  # between each job. Currently, jobs can otherwise fail with "Cache entry
+  # deserialization failed, entry ignored".
+- powershell: pip --no-cache-dir --disable-pip-version-check install --upgrade ${{ parameters.packages }}
+  displayName: 'Install Python packages (Windows)'
+  condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'))

tools/ci/azure/system_info.yml

@@ -0,0 +1,4 @@
+steps:
+- script: systeminfo
+  displayName: 'Show system info (Windows)'
+  condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'))

tools/ci/azure/update_hosts.yml

@@ -1,3 +1,10 @@
 steps:
 - script: ./wpt make-hosts-file | sudo tee -a /etc/hosts
-  displayName: 'Update /etc/hosts'
+  displayName: 'Update hosts (macOS)'
+  condition: and(succeeded(), eq(variables['Agent.OS'], 'Darwin'))
+- powershell: |
+    $hostFile = "$env:systemroot\System32\drivers\etc\hosts"
+    Copy-Item -Path $hostFile -Destination "$hostFile.back" -Force
+    python wpt make-hosts-file | Out-File $env:systemroot\System32\drivers\etc\hosts -Encoding ascii -Append
+  displayName: 'Update hosts (Windows)'
+  condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'))

tools/ci/azure/update_manifest.yml

@@ -1,3 +1,4 @@
 steps:
-- script: ./wpt manifest
+# `python wpt` instead of `./wpt` is to make this work on Windows:
+- script: python wpt manifest
   displayName: 'Update manifest'