ci: use sysroots on linux prod pipeline to build native modules

build/.cachesalt

@@ -1 +1 @@
-2023-09-06T09:54:45.225Z
+2023-11-24T11:56:03.453Z

build/azure-pipelines/cli/cli-compile-and-publish.yml

@@ -38,7 +38,26 @@ steps:
           ${{ pair.key }}: ${{ pair.value }}
 
   - ${{ else }}:
-    - script: cargo build --release --target ${{ parameters.VSCODE_CLI_TARGET }} --bin=code
+    - script: |
+        set -e
+        if [ -n "$SYSROOT_ARCH" ]; then
+          export VSCODE_SYSROOT_DIR=$(Build.SourcesDirectory)/.build/sysroots
+          node -e '(async () => { const { getVSCodeSysroot } = require("../build/linux/debian/install-sysroot.js"); await getVSCodeSysroot(process.env["SYSROOT_ARCH"]); })()'
+          if [ "$SYSROOT_ARCH" == "arm64" ]; then
+            export CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER="$VSCODE_SYSROOT_DIR/aarch64-linux-gnu/bin/aarch64-linux-gnu-gcc"
+            export CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_RUSTFLAGS="-C link-arg=--sysroot=$VSCODE_SYSROOT_DIR/aarch64-linux-gnu/aarch64-linux-gnu/sysroot"
+            export CC_aarch64_unknown_linux_gnu="$VSCODE_SYSROOT_DIR/aarch64-linux-gnu/bin/aarch64-linux-gnu-gcc --sysroot=$VSCODE_SYSROOT_DIR/aarch64-linux-gnu/aarch64-linux-gnu/sysroot"
+          elif [ "$SYSROOT_ARCH" == "amd64" ]; then
+            export CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER="$VSCODE_SYSROOT_DIR/x86_64-linux-gnu/bin/x86_64-linux-gnu-gcc"
+            export CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUSTFLAGS="-C link-arg=--sysroot=$VSCODE_SYSROOT_DIR/x86_64-linux-gnu/x86_64-linux-gnu/sysroot -C link-arg=-L$VSCODE_SYSROOT_DIR/x86_64-linux-gnu/x86_64-linux-gnu/sysroot/usr/lib/x86_64-linux-gnu"
+            export CC_x86_64_unknown_linux_gnu="$VSCODE_SYSROOT_DIR/x86_64-linux-gnu/bin/x86_64-linux-gnu-gcc --sysroot=$VSCODE_SYSROOT_DIR/x86_64-linux-gnu/x86_64-linux-gnu/sysroot"
+          elif [ "$SYSROOT_ARCH" == "armhf" ]; then
+            export CARGO_TARGET_ARMV7_UNKNOWN_LINUX_GNUEABIHF_LINKER="$VSCODE_SYSROOT_DIR/arm-rpi-linux-gnueabihf/bin/arm-rpi-linux-gnueabihf-gcc"
+            export CARGO_TARGET_ARMV7_UNKNOWN_LINUX_GNUEABIHF_RUSTFLAGS="-C link-arg=--sysroot=$VSCODE_SYSROOT_DIR/arm-rpi-linux-gnueabihf/arm-rpi-linux-gnueabihf/sysroot"
+            export CC_armv7_unknown_linux_gnueabihf="$VSCODE_SYSROOT_DIR/arm-rpi-linux-gnueabihf/bin/arm-rpi-linux-gnueabihf-gcc --sysroot=$VSCODE_SYSROOT_DIR/arm-rpi-linux-gnueabihf/arm-rpi-linux-gnueabihf/sysroot"
+          fi
+        fi
+        cargo build --release --target ${{ parameters.VSCODE_CLI_TARGET }} --bin=code
       displayName: Compile ${{ parameters.VSCODE_CLI_TARGET }}
       workingDirectory: $(Build.SourcesDirectory)/cli
       env:

build/azure-pipelines/linux/cli-build-linux.yml

@@ -39,13 +39,45 @@ steps:
       tar -xvzf $(Build.ArtifactStagingDirectory)/vscode-internal-openssl-prebuilt-0.0.11.tgz --strip-components=1 --directory=$(Build.ArtifactStagingDirectory)/openssl
     displayName: Extract openssl prebuilt
 
-  - ${{ if eq(parameters.VSCODE_BUILD_LINUX_ARMHF, true) }}:
-    - bash: sudo apt-get install -yq gcc-arm-linux-gnueabihf g++-arm-linux-gnueabihf binutils-arm-linux-gnueabihf
-      displayName: Install arm32 toolchains
+  - ${{ if ne(parameters.VSCODE_QUALITY, 'oss') }}:
+    - script: node build/setup-npm-registry.js $NPM_REGISTRY
+      condition: and(succeeded(), ne(variables['NPM_REGISTRY'], 'none'))
+      displayName: Setup NPM Registry
 
-  - ${{ if eq(parameters.VSCODE_BUILD_LINUX_ARM64, true) }}:
-    - bash: sudo apt-get install -yq gcc-aarch64-linux-gnu g++-aarch64-linux-gnu binutils-aarch64-linux-gnu
-      displayName: Install arm64 toolchains
+    - script: |
+        set -e
+        npm config set registry "$NPM_REGISTRY" --location=project
+        # npm >v7 deprecated the `always-auth` config option, refs npm/cli@72a7eeb
+        # following is a workaround for yarn to send authorization header
+        # for GET requests to the registry.
+        echo "always-auth=true" >> .npmrc
+        yarn config set registry "$NPM_REGISTRY"
+      condition: and(succeeded(), ne(variables['NPM_REGISTRY'], 'none'))
+      displayName: Setup NPM & Yarn
+
+    - task: npmAuthenticate@0
+      inputs:
+        workingFile: .npmrc
+      condition: and(succeeded(), ne(variables['NPM_REGISTRY'], 'none'))
+      displayName: Setup NPM Authentication
+
+  - script: |
+      set -e
+
+      for i in {1..5}; do # try 5 times
+        yarn --cwd build --frozen-lockfile --check-files && break
+        if [ $i -eq 3 ]; then
+          echo "Yarn failed too many times" >&2
+          exit 1
+        fi
+        echo "Yarn failed $i, trying again..."
+      done
+    displayName: Install build dependencies
+
+  - script: |
+      set -e
+      mkdir -p $(Build.SourcesDirectory)/.build
+    displayName: Create .build folder for misc dependencies
 
   - template: ../cli/install-rust-posix.yml
     parameters:
@@ -65,9 +97,9 @@ steps:
         VSCODE_CLI_ARTIFACT: vscode_cli_linux_arm64_cli
         VSCODE_CHECK_ONLY: ${{ parameters.VSCODE_CHECK_ONLY }}
         VSCODE_CLI_ENV:
-          CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER: aarch64-linux-gnu-gcc
           OPENSSL_LIB_DIR: $(Build.ArtifactStagingDirectory)/openssl/arm64-linux/lib
           OPENSSL_INCLUDE_DIR: $(Build.ArtifactStagingDirectory)/openssl/arm64-linux/include
+          SYSROOT_ARCH: arm64
 
   - ${{ if eq(parameters.VSCODE_BUILD_LINUX, true) }}:
     - template: ../cli/cli-compile-and-publish.yml
@@ -79,6 +111,7 @@ steps:
         VSCODE_CLI_ENV:
           OPENSSL_LIB_DIR: $(Build.ArtifactStagingDirectory)/openssl/x64-linux/lib
           OPENSSL_INCLUDE_DIR: $(Build.ArtifactStagingDirectory)/openssl/x64-linux/include
+          SYSROOT_ARCH: amd64
 
   - ${{ if eq(parameters.VSCODE_BUILD_LINUX_ARMHF, true) }}:
     - template: ../cli/cli-compile-and-publish.yml
@@ -88,6 +121,6 @@ steps:
         VSCODE_CLI_ARTIFACT: vscode_cli_linux_armhf_cli
         VSCODE_CHECK_ONLY: ${{ parameters.VSCODE_CHECK_ONLY }}
         VSCODE_CLI_ENV:
-          CARGO_TARGET_ARMV7_UNKNOWN_LINUX_GNUEABIHF_LINKER: arm-linux-gnueabihf-gcc
           OPENSSL_LIB_DIR: $(Build.ArtifactStagingDirectory)/openssl/arm-linux/lib
           OPENSSL_INCLUDE_DIR: $(Build.ArtifactStagingDirectory)/openssl/arm-linux/include
+          SYSROOT_ARCH: armhf

build/azure-pipelines/linux/install.sh

@@ -5,7 +5,15 @@ set -e
 # To workaround the issue of yarn not respecting the registry value from .npmrc
 yarn config set registry "$NPM_REGISTRY"
 
-if [ -z "$CC" ] || [ -z "$CXX" ]; then
+SYSROOT_ARCH=$VSCODE_ARCH
+if [ "$SYSROOT_ARCH" == "x64" ]; then
+  SYSROOT_ARCH="amd64"
+fi
+
+export VSCODE_SYSROOT_DIR=$PWD/.build/sysroots
+SYSROOT_ARCH="$SYSROOT_ARCH" node -e '(async () => { const { getVSCodeSysroot } = require("./build/linux/debian/install-sysroot.js"); await getVSCodeSysroot(process.env["SYSROOT_ARCH"]); })()'
+
+if [ "$npm_config_arch" == "x64" ]; then
   # Download clang based on chromium revision used by vscode
   curl -s https://raw.githubusercontent.com/chromium/chromium/114.0.5735.199/tools/clang/scripts/update.py | python - --output-dir=$PWD/.build/CR_Clang --host-os=linux
 
@@ -22,12 +30,27 @@ if [ -z "$CC" ] || [ -z "$CXX" ]; then
   # https://source.chromium.org/chromium/chromium/src/+/refs/tags/114.0.5735.199:build/config/arm.gni
   # https://source.chromium.org/chromium/chromium/src/+/refs/tags/114.0.5735.199:build/config/compiler/BUILD.gn
   # https://source.chromium.org/chromium/chromium/src/+/refs/tags/114.0.5735.199:build/config/c++/BUILD.gn
-  export CC=$PWD/.build/CR_Clang/bin/clang
-  export CXX=$PWD/.build/CR_Clang/bin/clang++
-  export CXXFLAGS="-nostdinc++ -D__NO_INLINE__ -I$PWD/.build/libcxx_headers -isystem$PWD/.build/libcxx_headers/include -isystem$PWD/.build/libcxxabi_headers/include -fPIC -flto=thin -fsplit-lto-unit -D_LIBCPP_ABI_NAMESPACE=Cr"
-  export LDFLAGS="-stdlib=libc++ -fuse-ld=lld -flto=thin -L$PWD/.build/libcxx-objects -lc++abi -Wl,--lto-O0"
-  export VSCODE_REMOTE_CC=$(which gcc)
-  export VSCODE_REMOTE_CXX=$(which g++)
+  export CC="$PWD/.build/CR_Clang/bin/clang --gcc-toolchain=$VSCODE_SYSROOT_DIR/x86_64-linux-gnu"
+  export CXX="$PWD/.build/CR_Clang/bin/clang++ --gcc-toolchain=$VSCODE_SYSROOT_DIR/x86_64-linux-gnu"
+  export CXXFLAGS="-nostdinc++ -D__NO_INLINE__ -I$PWD/.build/libcxx_headers -isystem$PWD/.build/libcxx_headers/include -isystem$PWD/.build/libcxxabi_headers/include -fPIC -flto=thin -fsplit-lto-unit -D_LIBCPP_ABI_NAMESPACE=Cr --sysroot=$VSCODE_SYSROOT_DIR/x86_64-linux-gnu/x86_64-linux-gnu/sysroot"
+  export LDFLAGS="-stdlib=libc++ --sysroot=$VSCODE_SYSROOT_DIR/x86_64-linux-gnu/x86_64-linux-gnu/sysroot -fuse-ld=lld -flto=thin -L$PWD/.build/libcxx-objects -lc++abi -L$VSCODE_SYSROOT_DIR/x86_64-linux-gnu/x86_64-linux-gnu/sysroot/usr/lib/x86_64-linux-gnu -L$VSCODE_SYSROOT_DIR/x86_64-linux-gnu/x86_64-linux-gnu/sysroot/lib/x86_64-linux-gnu -Wl,--lto-O0"
+  # Set compiler toolchain for remote server
+  export VSCODE_REMOTE_CC=$VSCODE_SYSROOT_DIR/x86_64-linux-gnu/bin/x86_64-linux-gnu-gcc
+  export VSCODE_REMOTE_CXX=$VSCODE_SYSROOT_DIR/x86_64-linux-gnu/bin/x86_64-linux-gnu-g++
+  export VSCODE_REMOTE_CXXFLAGS="--sysroot=$VSCODE_SYSROOT_DIR/x86_64-linux-gnu/x86_64-linux-gnu/sysroot"
+  export VSCODE_REMOTE_LDFLAGS="--sysroot=$VSCODE_SYSROOT_DIR/x86_64-linux-gnu/x86_64-linux-gnu/sysroot -L$VSCODE_SYSROOT_DIR/x86_64-linux-gnu/x86_64-linux-gnu/sysroot/usr/lib/x86_64-linux-gnu -L$VSCODE_SYSROOT_DIR/x86_64-linux-gnu/x86_64-linux-gnu/sysroot/lib/x86_64-linux-gnu"
+elif [ "$npm_config_arch" == "arm64" ]; then
+  # Set compiler toolchain for client native modules and remote server
+  export CC=$VSCODE_SYSROOT_DIR/aarch64-linux-gnu/bin/aarch64-linux-gnu-gcc
+  export CXX=$VSCODE_SYSROOT_DIR/aarch64-linux-gnu/bin/aarch64-linux-gnu-g++
+  export CXXFLAGS="--sysroot=$VSCODE_SYSROOT_DIR/aarch64-linux-gnu/aarch64-linux-gnu/sysroot"
+  export LDFLAGS="--sysroot=$VSCODE_SYSROOT_DIR/aarch64-linux-gnu/aarch64-linux-gnu/sysroot -L$VSCODE_SYSROOT_DIR/aarch64-linux-gnu/aarch64-linux-gnu/sysroot/usr/lib/aarch64-linux-gnu -L$VSCODE_SYSROOT_DIR/aarch64-linux-gnu/aarch64-linux-gnu/sysroot/lib/aarch64-linux-gnu"
+elif [ "$npm_config_arch" == "arm" ]; then
+  # Set compiler toolchain for client native modules and remote server
+  export CC=$VSCODE_SYSROOT_DIR/arm-rpi-linux-gnueabihf/bin/arm-rpi-linux-gnueabihf-gcc
+  export CXX=$VSCODE_SYSROOT_DIR/arm-rpi-linux-gnueabihf/bin/arm-rpi-linux-gnueabihf-g++
+  export CXXFLAGS="--sysroot=$VSCODE_SYSROOT_DIR/arm-rpi-linux-gnueabihf/arm-rpi-linux-gnueabihf/sysroot"
+  export LDFLAGS="--sysroot=$VSCODE_SYSROOT_DIR/arm-rpi-linux-gnueabihf/arm-rpi-linux-gnueabihf/sysroot -L$VSCODE_SYSROOT_DIR/arm-rpi-linux-gnueabihf/arm-rpi-linux-gnueabihf/sysroot/usr/lib/arm-linux-gnueabihf -L$VSCODE_SYSROOT_DIR/arm-rpi-linux-gnueabihf/arm-rpi-linux-gnueabihf/sysroot/lib/arm-linux-gnueabihf"
 fi
 
 for i in {1..5}; do # try 5 times

build/azure-pipelines/linux/product-build-linux.yml

@@ -48,13 +48,13 @@ steps:
       # Start X server
       sudo apt-get update
       sudo apt-get install -y pkg-config \
-        libxss1 \
         dbus \
         xvfb \
         libgtk-3-0 \
-        libgbm1 \
         libxkbfile-dev \
-        libkrb5-dev
+        libkrb5-dev \
+        libgbm1 \
+        rpm
       sudo cp build/azure-pipelines/linux/xvfb.init /etc/init.d/xvfb
       sudo chmod +x /etc/init.d/xvfb
       sudo update-rc.d xvfb defaults
@@ -100,17 +100,6 @@ steps:
     condition: and(succeeded(), ne(variables.NODE_MODULES_RESTORED, 'true'), ne(variables['NPM_REGISTRY'], 'none'))
     displayName: Setup NPM Authentication
 
-  - ${{ if and(ne(parameters.VSCODE_QUALITY, 'oss'), or(eq(parameters.VSCODE_ARCH, 'x64'), eq(parameters.VSCODE_ARCH, 'arm64'))) }}:
-    - task: Docker@1
-      displayName: "Pull Docker image"
-      inputs:
-        azureSubscriptionEndpoint: "vscode-builds-subscription"
-        azureContainerRegistry: vscodehub.azurecr.io
-        command: "Run an image"
-        imageName: vscode-linux-build-agent:centos7-devtoolset8-$(VSCODE_ARCH)
-        containerCommand: uname
-      condition: and(succeeded(), ne(variables.NODE_MODULES_RESTORED, 'true'))
-
   - ${{ if ne(parameters.VSCODE_QUALITY, 'oss') }}:
     - script: |
         set -e
@@ -124,30 +113,62 @@ steps:
           echo "Yarn failed $i, trying again..."
         done
 
-        docker run -e GITHUB_TOKEN -e npm_config_arch -e NPM_REGISTRY \
-        -e VSCODE_REMOTE_DEPENDENCIES_CONTAINER_NAME -e VSCODE_HOST_MOUNT \
-        -e ELECTRON_SKIP_BINARY_DOWNLOAD -e PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD \
-        -e VSCODE_SKIP_NODE_VERSION_CHECK \
-        -v /mnt/vss/_work/1/s:/home/builduser/vscode -v /mnt/vss/_work/1/s/.build/.netrc:/home/builduser/.netrc \
-        -v /var/run/docker.sock:/var/run/docker.sock \
-        -u 1000:1000 \
-        -w /home/builduser/vscode vscodehub.azurecr.io/vscode-linux-build-agent:bionic-$(VSCODE_ARCH) \
-        /bin/bash -c "./build/azure-pipelines/linux/install.sh"
-
-        sudo chown -R $USER:$USER /mnt/vss/_work/1/s
+        ./build/azure-pipelines/linux/install.sh
       env:
         npm_config_arch: $(NPM_ARCH)
+        VSCODE_ARCH: $(VSCODE_ARCH)
         NPM_REGISTRY: "$(NPM_REGISTRY)"
         ELECTRON_SKIP_BINARY_DOWNLOAD: 1
         PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1
-        VSCODE_SKIP_NODE_VERSION_CHECK: 1
         GITHUB_TOKEN: "$(github-distro-mixin-password)"
-        VSCODE_HOST_MOUNT: "/mnt/vss/_work/1/s"
-        ${{ if or(eq(parameters.VSCODE_ARCH, 'x64'), eq(parameters.VSCODE_ARCH, 'arm64')) }}:
-          VSCODE_REMOTE_DEPENDENCIES_CONTAINER_NAME: vscodehub.azurecr.io/vscode-linux-build-agent:centos7-devtoolset8-$(VSCODE_ARCH)
       displayName: Install dependencies (non-OSS)
       condition: and(succeeded(), ne(variables.NODE_MODULES_RESTORED, 'true'))
 
+    - script: |
+        set -e
+
+        TRIPLE="x86_64-linux-gnu"
+        if [ "$VSCODE_ARCH" == "arm64" ]; then
+          TRIPLE="aarch64-linux-gnu"
+        elif [ "$VSCODE_ARCH" == "armhf" ]; then
+          TRIPLE="arm-rpi-linux-gnueabihf"
+        fi
+
+        # Get all files with .node extension from remote/node_modules folder
+        files=$(find remote/node_modules -name "*.node")
+
+        # Check if any file has dependency of GLIBC > 2.28 or GLIBCXX > 3.4.25
+        for file in $files; do
+          glibc_version=""
+          glibcxx_version=""
+          while IFS= read -r line; do
+            if [[ $line == *"GLIBC_"* ]]; then
+              version=$(echo "$line" | awk '{print $5}' | tr -d '()')
+              version=${version#*_}
+              if [[ -z $glibc_version || $(printf "%s\n%s" "$version" "$glibc_version" | sort -V | tail -n1) == "$version" ]]; then
+                glibc_version=$version
+              fi
+            elif [[ $line == *"GLIBCXX_"* ]]; then
+              version=$(echo "$line" | awk '{print $5}' | tr -d '()')
+              version=${version#*_}
+              if [[ -z $glibcxx_version || $(printf "%s\n%s" "$version" "$glibcxx_version" | sort -V | tail -n1) == "$version" ]]; then
+                glibcxx_version=$version
+              fi
+            fi
+          done < <("$PWD/.build/sysroots/$TRIPLE/$TRIPLE/bin/objdump" -T "$file")
+
+          if [[ "$glibcxx_version" > "3.4.25" ]]; then
+            echo "Error: File $file has dependency on GLIBC > 2.28"
+            exit 1
+          fi
+          if [[ "$glibcxx_version" == "3.4.25" ]]; then
+            echo "Error: File $file has dependency on GLIBCXX > 3.4.25"
+            exit 1
+          fi
+        done
+      condition: and(succeeded(), ne(variables.NODE_MODULES_RESTORED, 'true'))
+      displayName: Check GLIBC and GLIBCXX dependencies in remote/node_modules
+
     - script: node build/azure-pipelines/distro/mixin-npm
       condition: and(succeeded(), ne(variables.NODE_MODULES_RESTORED, 'true'))
       displayName: Mixin distro node modules
@@ -266,11 +287,6 @@ steps:
   - ${{ if and(ne(parameters.VSCODE_CIBUILD, true), ne(parameters.VSCODE_QUALITY, 'oss')) }}:
     - script:  |
         set -e
-        docker run -v /mnt/vss/_work/1/s:/home/builduser/vscode \
-        -v /mnt/vss/_work/1/s/.build/.netrc:/home/builduser/.netrc \
-        -v /mnt/vss/_work/1/VSCode-linux-$(VSCODE_ARCH):/home/builduser/VSCode-linux-$(VSCODE_ARCH) \
-        -u 1000:1000 \
-        -w /home/builduser/vscode vscodehub.azurecr.io/vscode-linux-build-agent:bionic-$(VSCODE_ARCH) \
         yarn gulp "vscode-linux-$(VSCODE_ARCH)-prepare-deb"
       displayName: Prepare deb package
 
@@ -282,11 +298,6 @@ steps:
 
     - script:  |
         set -e
-        docker run -v /mnt/vss/_work/1/s:/home/builduser/vscode \
-        -v /mnt/vss/_work/1/s/.build/.netrc:/home/builduser/.netrc \
-        -v /mnt/vss/_work/1/VSCode-linux-$(VSCODE_ARCH):/home/builduser/VSCode-linux-$(VSCODE_ARCH) \
-        -u 1000:1000 \
-        -w /home/builduser/vscode vscodehub.azurecr.io/vscode-linux-build-agent:bionic-$(VSCODE_ARCH) \
         yarn gulp "vscode-linux-$(VSCODE_ARCH)-prepare-rpm"
       displayName: Prepare rpm package
 

build/checksums/vscode-sysroot.txt

@@ -0,0 +1,3 @@
+a2d51dc505ed544c52757f90bcdab44920132295fc7a67166eff86b6e0e24aa8  aarch64-linux-gnu.tar.gz
+032cf16bf8b965e1351305f10f3dedabf4f9868027ac6d0e8f52321ca0b70d4a  arm-rpi-linux-gnueabihf.tar.gz
+360475a764d0faf4d3743aa866347eff78072639d20660def83e1a03eadf534c  x86_64-linux-gnu.tar.gz