VM Update September 2022 (#26895)

* Revert Linux to Ubuntu 20.04. * Cherry-pick intel-mkl from #15233 * Update pwsh to 7.2.6. * Temporarily remove ci.baseline.txt skips for ICEs. * Fix ICU version for Ubuntu version change. * Update pools. * Merge apt installs and fix WSL CUDA bindings. * Fix WSL detection. * Remove WSL specific nvidia because it misses nccl and cudnn. * Add python-yaml for duktape. * [shogun] Force use of OpenBLAS as the Lapack implementation. Shogun declares a dependency on blas, which we have set to openblas. Installing Intel MKL on the VMs caused build of Shogun to fail because it tried to use MKL and that build is broken. * Put colmap, graphicsmagick, and mfl skips back, as they are clearly not fixed. * PASSING, REMOVE FROM FAIL LIST: jemalloc:x64-osx (/Users/vagrant/Data/work/2/s/scripts/azure-pipelines/../ci.baseline.txt). * Update pool. * Revert the intel-mkl changes until the BLAS/LAPACK vendoring problem is fixed. * Update pool. * Explicitly add bison, needed by gettext. * Update pools
microsoft · Sep 26, 2022 · 86cab74 · 86cab74
1 parent 5bc9803
commit 86cab74
Show file tree

Hide file tree

Showing 6 changed files with 58 additions and 51 deletions.
diff --git a/scripts/azure-pipelines/azure-pipelines.yml b/scripts/azure-pipelines/azure-pipelines.yml
@@ -2,8 +2,8 @@
 # SPDX-License-Identifier: MIT
 #
 variables:
-  windows-pool: 'PrWin-2022-08-10'
-  linux-pool: 'PrLin-2022-07-14'
+  windows-pool: 'PrWin-2022-09-20'
+  linux-pool: 'PrLin-2022-09-25'
   osx-pool: 'PrOsx-2022-02-04'
 
 parameters:

diff --git a/scripts/azure-pipelines/linux/create-image.ps1 b/scripts/azure-pipelines/linux/create-image.ps1
@@ -88,8 +88,8 @@ $VM = Add-AzVMNetworkInterface -VM $VM -Id $Nic.Id
 $VM = Set-AzVMSourceImage `
   -VM $VM `
   -PublisherName 'Canonical' `
-  -Offer '0001-com-ubuntu-server-jammy' `
-  -Skus '22_04-lts' `
+  -Offer '0001-com-ubuntu-server-focal' `
+  -Skus '20_04-lts' `
   -Version latest
 
 $VM = Set-AzVMBootDiagnostic -VM $VM -Disable

diff --git a/scripts/azure-pipelines/linux/provision-image.sh b/scripts/azure-pipelines/linux/provision-image.sh
@@ -5,91 +5,100 @@
 
 export DEBIAN_FRONTEND=noninteractive
 
+# Add apt repos
+
+## CUDA
+apt-key del 7fa2af80
+wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/cuda-ubuntu2004.pin
+mv cuda-ubuntu2004.pin /etc/apt/preferences.d/cuda-repository-pin-600
+apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/3bf863cc.pub
+add-apt-repository "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/ /"
+
+## PowerShell
+wget -q https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb
+dpkg -i packages-microsoft-prod.deb
+rm -f packages-microsoft-prod.deb
+add-apt-repository universe
+
 apt-get -y update
 apt-get -y dist-upgrade
 
-# Install vcpkg prerequisites
+# Add apt packages
+
+## vcpkg prerequisites
 APT_PACKAGES="git curl zip unzip tar"
 
-# Install common build dependencies
+## common build dependencies
 APT_PACKAGES="$APT_PACKAGES at libxt-dev gperf libxaw7-dev cifs-utils \
   build-essential g++ gfortran libx11-dev libxkbcommon-x11-dev libxi-dev \
   libgl1-mesa-dev libglu1-mesa-dev mesa-common-dev libxinerama-dev libxxf86vm-dev \
   libxcursor-dev yasm libnuma1 libnuma-dev \
-  flex libbison-dev autoconf libudev-dev libncurses5-dev libtool libxrandr-dev \
+  flex bison libbison-dev autoconf libudev-dev libncurses5-dev libtool libxrandr-dev \
   xutils-dev dh-autoreconf autoconf-archive libgles2-mesa-dev ruby-full \
   pkg-config meson nasm cmake ninja-build"
 
-# CUDA tooling
-APT_PACKAGES="$APT_PACKAGES nvidia-cudnn nvidia-cuda-toolkit"
-
-# Additionally required by qt5-base
+## required by qt5-base
 APT_PACKAGES="$APT_PACKAGES libxext-dev libxfixes-dev libxrender-dev \
   libxcb1-dev libx11-xcb-dev libxcb-glx0-dev libxcb-util0-dev"
 
-# Additionally required by qt5-base for qt5-x11extras
+## required by qt5-base for qt5-x11extras
 APT_PACKAGES="$APT_PACKAGES libxkbcommon-dev libxcb-keysyms1-dev \
   libxcb-image0-dev libxcb-shm0-dev libxcb-icccm4-dev libxcb-sync-dev \
   libxcb-xfixes0-dev libxcb-shape0-dev libxcb-randr0-dev \
   libxcb-render-util0-dev libxcb-xinerama0-dev libxcb-xkb-dev libxcb-xinput-dev"
 
-# Additionally required by libhdfs3
+## required by libhdfs3
 APT_PACKAGES="$APT_PACKAGES libkrb5-dev"
 
-# Additionally required by kf5windowsystem
+## required by kf5windowsystem
 APT_PACKAGES="$APT_PACKAGES libxcb-res0-dev"
 
-# Additionally required by mesa
+## required by mesa
 APT_PACKAGES="$APT_PACKAGES python3-setuptools python3-mako"
 
-# Additionally required by some packages to install additional python packages
+## required by some packages to install additional python packages
 APT_PACKAGES="$APT_PACKAGES python3-pip python3-venv"
 
-# Additionally required by qtwebengine
+## required by qtwebengine
 APT_PACKAGES="$APT_PACKAGES nodejs"
 
-# Additionally required by qtwayland
+## required by qtwayland
 APT_PACKAGES="$APT_PACKAGES libwayland-dev"
 
-# Additionally required by all GN projects
+## required by all GN projects
 APT_PACKAGES="$APT_PACKAGES python2 python-is-python3"
 
-# Additionally required by libctl
+## required by libctl
 APT_PACKAGES="$APT_PACKAGES guile-2.2-dev"
 
-# Additionally required by gtk
+## required by gtk
 APT_PACKAGES="$APT_PACKAGES libxdamage-dev"
 
-# Additionally required by gtk3 and at-spi2-atk
+## required by gtk3 and at-spi2-atk
 APT_PACKAGES="$APT_PACKAGES libdbus-1-dev"
 
-# Additionally required by at-spi2-atk
+## required by at-spi2-atk
 APT_PACKAGES="$APT_PACKAGES libxtst-dev"
 
-# Additionally required by bond
+## required by bond
 APT_PACKAGES="$APT_PACKAGES haskell-stack"
 
-# Additionally required/installed by Azure DevOps Scale Set Agents
-APT_PACKAGES="$APT_PACKAGES libkrb5-3 zlib1g libicu70"
+## required by duktape
+APT_PACKAGES="$APT_PACKAGES python-yaml"
 
-apt-get -y install $APT_PACKAGES
+## CUDA
+APT_PACKAGES="$APT_PACKAGES cuda-compiler-11-6 cuda-libraries-dev-11-6 cuda-driver-dev-11-6 \
+  cuda-cudart-dev-11-6 libcublas-11-6 libcurand-dev-11-6 cuda-nvml-dev-11-6 libcudnn8-dev libnccl2 \
+  libnccl-dev"
 
-# Start up cudnn
-update-nvidia-cudnn -d
-update-nvidia-cudnn -u
+## PowerShell
+APT_PACKAGES="$APT_PACKAGES powershell"
 
-# Install nccl
-wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-ubuntu2204.pin
-mv cuda-ubuntu2204.pin /etc/apt/preferences.d/cuda-repository-pin-600
-apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/3bf863cc.pub
-add-apt-repository "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/ /"
-apt-get -y update
-apt-get install --no-install-recommends libnccl2 libnccl-dev
+## Additionally required/installed by Azure DevOps Scale Set Agents, skip on WSL
+if [[ $(grep microsoft /proc/version) ]]; then
+echo "Skipping install of ADO prerequisites on WSL."
+else
+APT_PACKAGES="$APT_PACKAGES libkrb5-3 zlib1g libicu66"
+fi
 
-# Install PowerShell
-wget -q https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb
-dpkg -i packages-microsoft-prod.deb
-rm -f packages-microsoft-prod.deb
-apt-get update
-add-apt-repository universe
-apt-get install -y powershell
+apt-get -y --no-install-recommends install $APT_PACKAGES
diff --git a/scripts/azure-pipelines/windows/deploy-pwsh.ps1 b/scripts/azure-pipelines/windows/deploy-pwsh.ps1
@@ -5,5 +5,5 @@
 
 # REPLACE WITH UTILITY-PREFIX.ps1
 
-$PwshUrl = 'https://github.com/PowerShell/PowerShell/releases/download/v7.2.5/PowerShell-7.2.5-win-x64.msi'
+$PwshUrl = 'https://github.com/PowerShell/PowerShell/releases/download/v7.2.6/PowerShell-7.2.6-win-x64.msi'
 InstallMSI -Url $PwshUrl -Name 'PowerShell Core'
diff --git a/scripts/ci.baseline.txt b/scripts/ci.baseline.txt
@@ -381,8 +381,6 @@ irrlicht:arm-uwp=fail
 irrlicht:x64-uwp=fail
 jemalloc:arm64-windows=fail
 jemalloc:arm-uwp=fail
-jemalloc:x64-osx=fail
-jemalloc:arm64-osx=fail
 jemalloc:x64-uwp=fail
 jinja2cpplight:arm-uwp=fail
 jinja2cpplight:x64-uwp=fail

diff --git a/scripts/vcpkgTools.xml b/scripts/vcpkgTools.xml
@@ -194,11 +194,11 @@
         <archiveName>ninja-freebsd-1.8.2.zip</archiveName>
     </tool>
     <tool name="powershell-core" os="windows">
-        <version>7.2.5</version>
+        <version>7.2.6</version>
         <exeRelativePath>pwsh.exe</exeRelativePath>
-        <url>https://github.com/PowerShell/PowerShell/releases/download/v7.2.5/PowerShell-7.2.5-win-x86.zip</url>
-        <sha512>75b549db2500a5510eae3ec595540b42726c1b049f382c7683b6f0ed26dddd219e2ac67e9902358615f96c7884eda63ec3d0ca558b7b92f4604fdc61322ca4dd</sha512>
-        <archiveName>PowerShell-7.2.5-win-x86.zip</archiveName>
+        <url>https://github.com/PowerShell/PowerShell/releases/download/v7.2.6/PowerShell-7.2.6-win-x86.zip</url>
+        <sha512>4b444e0d5de22380e47701c190cb8ec9464a14a7ca566bcb8a936f723d58571af537d0d872f788dac99e67b27a047ce53de3fe1a986967799c10ea6784dbd6b2</sha512>
+        <archiveName>PowerShell-7.2.6-win-x86.zip</archiveName>
     </tool>
     <tool name="node" os="windows">
         <version>16.15.1</version>