Merge branch 'main' into opentelemetry-stream-fix

Signed-off-by: ashish b <108897222+ashishb-solo@users.noreply.github.com>

.azure-pipelines/bazel.yml

@@ -12,27 +12,30 @@ parameters:
 - name: cacheKeyDocker
   type: string
   default: ".devcontainer/Dockerfile"
-- name: cacheKeyDockerVersion
+- name: cacheKeyVersion
   type: string
-  default: $(cacheKeyBuildImage)
-- name: cacheKeyDockerName
+  default: $(cacheKeyVersion)
+- name: pathCacheTemp
   type: string
-  default: envoy_build_image
-- name: cacheKeyDockerPath
-  type: string
-  default: /mnt/docker
-- name: cacheKeyDockerTmpDir
+  default: $(pathCacheTemp)
+
+- name: tmpfsCacheDisabled
   type: string
-  default: /mnt/docker_cache
-- name: cacheKeyDockerNoTmpfs
+  default: ''
+- name: tmpfsDockerDisabled
   type: string
   default: ''
-- name: cacheKey
+
+- name: cacheKeyBazel
   type: string
-  default: $(cacheKeyBazelFiles)
+  default: $(cacheKeyBazel)
 - name: cacheVersion
   type: string
-  default: $(cacheKeyBazel)
+  default: $(cacheKeyVersion)
+
+- name: pathDockerBind
+  type: string
+  default: $(pathDockerBind)
 
 - name: rbe
   displayName: "Enable RBE"
@@ -72,6 +75,10 @@ parameters:
   type: string
   default: true
 
+- name: diskspaceHack
+  type: boolean
+  default: false
+
 - name: stepsPre
   type: stepList
   default: []
@@ -88,6 +95,10 @@ steps:
   fetchDepth: ${{ parameters.repoFetchDepth }}
   fetchTags: ${{ parameters.repoFetchTags }}
 
+- bash: ./.azure-pipelines/cleanup.sh
+  displayName: "Free disk space"
+  condition: and(succeeded(), eq('${{ parameters.diskspaceHack }}', true))
+
 # Set up tmpfs directories for self-hosted agents which have a surplus of mem.
 #
 # NB: Do not add any directory that grow larger than spare memory capacity!
@@ -119,7 +130,7 @@ steps:
     done
     sudo chown -R azure-pipelines:azure-pipelines $(Build.StagingDirectory)/bazel_root/
   displayName: "Mount/tmpfs bazel directories"
-  condition: and(succeeded(), eq('${{ parameters.managedAgent }}', false))
+  condition: and(succeeded(), eq('${{ parameters.managedAgent }}', false), ne('${{ parameters.tmpfsDockerDisabled }}', true))
 
 - bash: |
     set -e
@@ -132,30 +143,18 @@ steps:
     sudo chown -R vsts:vsts "${CACHE_DIRS[@]}" $(Build.StagingDirectory)/bazel_root/
     echo "Created bazel cache directories: "${CACHE_DIRS[*]}""
   displayName: "Create bazel directories"
-  condition: and(succeeded(), eq('${{ parameters.managedAgent }}', true))
+  condition: and(succeeded(), eq('${{ parameters.managedAgent }}', true), eq('${{ parameters.tmpfsDockerDisabled }}', true))
 
 # Caching
-- task: Cache@2
-  inputs:
-    key: '"${{ parameters.ciTarget }}" | "${{ parameters.cacheVersion }}" | "${{ parameters.artifactSuffix }}" | ${{ parameters.cacheKey }}'
-    path: $(Build.StagingDirectory)/bazel
-    cacheHitVar: BAZEL_CACHE_RESTORED
-  continueOnError: true
-- script: |
-    set -e
-    sudo tar xf $(Build.StagingDirectory)/bazel/cache.tar.zst -C $(Build.StagingDirectory) --warning=no-timestamp
-    sudo rm -rf $(Build.StagingDirectory)/bazel/*
-  displayName: "Cache/restore (${{ parameters.ciTarget }})"
-  condition: and(not(canceled()), eq(variables.BAZEL_CACHE_RESTORED, 'true'))
 - template: cached.yml
   parameters:
-    key: "${{ parameters.cacheKeyDocker }}"
-    version: "${{ parameters.cacheKeyDockerVersion }}"
-    name: "${{ parameters.cacheKeyDockerName }}"
-    path: "${{ parameters.cacheKeyDockerPath }}"
-    tmpDirectory: "${{ parameters.cacheKeyDockerTmpDir }}"
-    tmpNoTmpfs: "${{ parameters.cacheKeyDockerNoTmpfs }}"
+    keyBazel: "${{ parameters.cacheKeyBazel }}"
+    keyDocker: "${{ parameters.cacheKeyDocker }}"
+    pathDockerBind: "${{ parameters.pathDockerBind }}"
     arch: "${{ parameters.artifactSuffix }}"
+    pathTemp: "${{ parameters.pathCacheTemp }}"
+    tmpfsDisabled: "${{ parameters.tmpfsCacheDisabled }}"
+    tmpfsDockerDisabled: "${{ parameters.tmpfsDockerDisabled }}"
 
 - ${{ each step in parameters.stepsPre }}:
   - ${{ each pair in step }}:
@@ -195,9 +194,7 @@ steps:
     ${{ if parameters.rbe }}:
       GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey)
       ENVOY_RBE: "1"
-      BAZEL_BUILD_EXTRA_OPTIONS: "--config=remote-ci --jobs=$(RbeJobs) ${{ parameters.bazelBuildExtraOptions }}"
-      BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com
-      BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance
+      BAZEL_BUILD_EXTRA_OPTIONS: "--config=remote-ci --config=rbe-google --jobs=$(RbeJobs) ${{ parameters.bazelBuildExtraOptions }}"
     ${{ if eq(parameters.rbe, false) }}:
       BAZEL_BUILD_EXTRA_OPTIONS: "--config=ci ${{ parameters.bazelBuildExtraOptions }}"
       BAZEL_REMOTE_CACHE: $(LocalBuildCache)
@@ -221,22 +218,9 @@ steps:
   - ${{ each pair in step }}:
       ${{ pair.key }}: ${{ pair.value }}
 
-- script: |
-    set -e
-    CACHE_DIRS=(
-        ".cache"
-        "bazel_root/install"
-        "repository_cache/"
-        "bazel_root/base/external")
-    mkdir -p $(Build.StagingDirectory)/bazel/
-    sudo tar cf - -C $(Build.StagingDirectory) "${CACHE_DIRS[@]}" \
-        | zstd - -T0 -o $(Build.StagingDirectory)/bazel/cache.tar.zst
-    echo "Created tarball ($(Build.StagingDirectory)/bazel/cache.tar.zst): ${CACHE_DIRS[@]}"
-  displayName: "Cache/save (${{ parameters.ciTarget }})"
-  condition: and(not(canceled()), ne(variables.BAZEL_CACHE_RESTORED, 'true'))
-
 - task: PublishTestResults@2
   inputs:
+    publishRunAttachments: false
     testResultsFiles: "**/bazel-out/**/testlogs/**/test.xml"
     testRunTitle: "${{ parameters.ciTarget }}"
     searchFolder: $(Build.StagingDirectory)/bazel_root

.azure-pipelines/cached.yml

@@ -2,25 +2,36 @@
 parameters:
 - name: name
   type: string
-  default: envoy_build_image
-- name: version
-  type: string
-  default: ""
+  default: $(cacheKeyName)
 - name: arch
   type: string
   default: ""
-- name: key
+- name: version
+  type: string
+  default: $(cacheKeyVersion)
+
+- name: keyDocker
+  type: string
+  default: $(cacheKeyDocker)
+- name: keyBazel
   type: string
-  default: ".devcontainer/Dockerfile"
-- name: tmpDirectory
+  default: $(cacheKeyBazel)
+
+- name: pathTemp
+  type: string
+  default: $(pathCacheTemp)
+
+- name: tmpfsDisabled
   type: string
-  default: /mnt/docker_cache
-- name: tmpNoTmpfs
+  default:
+- name: tmpfsDockerDisabled
   type: string
   default:
-- name: path
+
+- name: pathDockerBind
   type: string
-  default: /mnt/docker
+  default: $(pathDockerBind)
+
 - name: cacheTimeoutWorkaround
   type: number
   default: 5
@@ -30,24 +41,37 @@ parameters:
 
 
 steps:
-- script: sudo .azure-pipelines/docker/prepare_cache.sh "${{ parameters.tmpDirectory }}" "${{ parameters.tmpNoTmpfs }}"
-  displayName: "Cache/prepare (${{ parameters.name }})"
+- script: sudo .azure-pipelines/docker/prepare_cache.sh "${{ parameters.pathTemp }}" "${{ parameters.tmpfsDisabled }}"
+  displayName: "Cache/prepare"
+
+- task: Cache@2
+  env:
+    VSO_DEDUP_REDIRECT_TIMEOUT_IN_SEC: "${{ parameters.cacheTimeoutWorkaround }}"
+  displayName: "Cache (Docker)"
+  inputs:
+    key: '${{ parameters.name }} | "${{ parameters.version }}" | "${{ parameters.arch }}" | ${{ parameters.keyDocker }}'
+    path: "${{ parameters.pathTemp }}/docker"
+    cacheHitVar: DOCKER_CACHE_RESTORED
+
 - task: Cache@2
   env:
     VSO_DEDUP_REDIRECT_TIMEOUT_IN_SEC: "${{ parameters.cacheTimeoutWorkaround }}"
-  displayName: "Cache/fetch (${{ parameters.name }})"
+  displayName: "Cache (Bazel)"
   inputs:
-    key: '${{ parameters.name }} | "${{ parameters.version }}" | "${{ parameters.arch }}" | ${{ parameters.key }}'
-    path: "${{ parameters.tmpDirectory }}"
-    cacheHitVar: CACHE_RESTORED
+    key: '${{ parameters.name }} | bazel | "${{ parameters.version }}" | "${{ parameters.arch }}" | ${{ parameters.keyBazel }}'
+    path: "${{ parameters.pathTemp }}/bazel"
+    cacheHitVar: BAZEL_CACHE_RESTORED
 
-# Prime the cache for all jobs
-- script: sudo .azure-pipelines/docker/prime_cache.sh "${{ parameters.tmpDirectory }}" "${{ parameters.arch }}"
+# Prime the caches for all jobs
+- script: .azure-pipelines/docker/prime_cache.sh "$(Build.StagingDirectory)" "${{ parameters.pathTemp }}" "${{ parameters.arch }}"
+  env:
+    DOCKER_RESTORED: $(DOCKER_CACHE_RESTORED)
+    BAZEL_RESTORED: $(BAZEL_CACHE_RESTORED)
   displayName: "Cache/prime (${{ parameters.name }})"
   # TODO(phlax): figure if there is a way to test cache without downloading it
-  condition: and(not(canceled()), eq(${{ parameters.prime }}, true), ne(variables.CACHE_RESTORED, 'true'))
+  condition: and(not(canceled()), eq(${{ parameters.prime }}, true), or(ne(variables.DOCKER_CACHE_RESTORED, 'true'), ne(variables.BAZEL_CACHE_RESTORED, 'true')))
 
-# Load the cache for a job
-- script: sudo .azure-pipelines/docker/load_cache.sh "${{ parameters.tmpDirectory }}" "${{ parameters.path }}"
-  displayName: "Cache/restore (${{ parameters.name }})"
+# Load the caches for a job
+- script: sudo .azure-pipelines/docker/load_caches.sh "$(Build.StagingDirectory)" "${{ parameters.pathTemp }}" "${{ parameters.pathDockerBind }}" "${{ parameters.tmpfsDockerDisabled }}"
+  displayName: "Cache/restore"
   condition: and(not(canceled()), eq(${{ parameters.prime }}, false))

.azure-pipelines/docker/clean_docker.sh

@@ -0,0 +1,11 @@
+#!/bin/bash -e
+
+set -o pipefail
+
+echo "Stopping Docker ..."
+systemctl stop docker
+
+echo "Restarting Docker with empty /var/lib/docker ..."
+mv /var/lib/docker/ /var/lib/docker.old
+mkdir /var/lib/docker
+systemctl start docker

.azure-pipelines/docker/create_cache.sh

@@ -0,0 +1,22 @@
+#!/bin/bash -e
+
+set -o pipefail
+
+CACHE_TARBALL="${1}"
+shift
+
+echo "Exporting ${*} -> ${CACHE_TARBALL}"
+
+CACHE_PATH="$(dirname "$CACHE_TARBALL")"
+mkdir -p "$CACHE_PATH"
+
+CACHE_ARGS=()
+for path in "$@"; do
+    total="$(du -sh "$path" | cut -f1)"
+    echo "Adding cache dir (${path}): ${total}"
+    CACHE_ARGS+=(-C "$path" .)
+done
+
+tar cf - "${CACHE_ARGS[@]}" | zstd - -q -T0 -o "$CACHE_TARBALL"
+echo "Cache tarball created: ${CACHE_TARBALL}"
+ls -lh "$CACHE_TARBALL"

.azure-pipelines/docker/load_cache.sh → .azure-pipelines/docker/load_caches.sh

@@ -1,16 +1,22 @@
 #!/bin/bash -e
 
-DOCKER_CACHE_PATH="$1"
-DOCKER_BIND_PATH="$2"
+ENVOY_DOCKER_BUILD_DIR="$1"
+CACHE_PATH="$2"
+DOCKER_BIND_PATH="$3"
+DOCKER_NO_TMPFS="$4"
 
-if [[ -z "$DOCKER_CACHE_PATH" ]]; then
-    echo "load_docker_cache called without path arg" >&2
+
+if [[ -z "$CACHE_PATH" ]]; then
+    echo "load_caches called without path arg" >&2
     exit 1
 fi
 
-
+DOCKER_CACHE_PATH="${CACHE_PATH}/docker"
 DOCKER_CACHE_TARBALL="${DOCKER_CACHE_PATH}/docker.tar.zst"
 
+BAZEL_CACHE_PATH="${CACHE_PATH}/bazel"
+BAZEL_CACHE_TARBALL="${BAZEL_CACHE_PATH}/bazel.tar.zst"
+
 echo "Stopping Docker daemon ..."
 systemctl stop docker docker.socket
 
@@ -22,22 +28,22 @@ if id -u vsts &> /dev/null && [[ -n "$DOCKER_BIND_PATH" ]]; then
     echo "Binding docker directory ${DOCKER_BIND_PATH} -> /var/lib/docker ..."
     mkdir -p "$DOCKER_BIND_PATH"
     mount -o bind "$DOCKER_BIND_PATH" /var/lib/docker
-elif ! id -u vsts &> /dev/null; then
+elif ! id -u vsts &> /dev/null && [[ -z "$DOCKER_NO_TMPFS" ]]; then
     echo "Mounting tmpfs directory -> /var/lib/docker ..."
     # Use a ramdisk to load docker (avoids Docker slow start on big disk)
     mount -t tmpfs none /var/lib/docker
 else
-    # If we are on a managed host but the bind path is not set then we need to remove
+    # If we are on a managed/resource-constrained host but the bind path is not set then we need to remove
     # the old /var/lib/docker to free some space (maybe)
     DOCKER_REMOVE_EXISTING=1
 fi
 
 if [[ -e "${DOCKER_CACHE_TARBALL}" ]]; then
     echo "Extracting docker cache ${DOCKER_CACHE_TARBALL} -> /var/lib/docker ..."
-    zstd --stdout -d "$DOCKER_CACHE_TARBALL" | tar -xf - -C /var/lib/docker
+    zstd --stdout -d "$DOCKER_CACHE_TARBALL" | tar --warning=no-timestamp -xf - -C /var/lib/docker
     touch /tmp/DOCKER_CACHE_RESTORED
 else
-    echo "No cache to restore, starting Docker with no data"
+    echo "No Docker cache to restore, starting Docker with no data"
 fi
 
 echo "Starting Docker daemon ..."
@@ -51,9 +57,27 @@ else
     rm -rf "${DOCKER_CACHE_PATH}"
 fi
 docker images
-df -h
+
+mkdir -p "${ENVOY_DOCKER_BUILD_DIR}"
+
+if [[ -e "${BAZEL_CACHE_TARBALL}" ]]; then
+    echo "Extracting bazel cache ${BAZEL_CACHE_TARBALL} -> ${ENVOY_DOCKER_BUILD_DIR} ..."
+    zstd --stdout -d "$BAZEL_CACHE_TARBALL" | tar --warning=no-timestamp -xf - -C "${ENVOY_DOCKER_BUILD_DIR}"
+else
+    echo "No bazel cache to restore, starting bazel with no data"
+fi
+
+if mountpoint -q "${CACHE_PATH}"; then
+    echo "Unmount cache tmp ${CACHE_PATH} ..."
+    umount "${CACHE_PATH}"
+else
+    echo "Remove cache tmp ${CACHE_PATH} ..."
+    rm -rf "${CACHE_PATH}"
+fi
 
 # this takes time but may be desirable in some situations
 if [[ -n "$DOCKER_REMOVE_EXISTING" ]]; then
     rm -rf /var/lib/docker.old
 fi
+
+df -h

.azure-pipelines/docker/prepare_cache.sh

@@ -3,7 +3,7 @@
 DOCKER_CACHE_PATH="$1"
 NO_MOUNT_TMPFS="${2:-}"
 DOCKER_CACHE_OWNERSHIP="vsts:vsts"
-
+TMPFS_SIZE=5G
 
 if [[ -z "$DOCKER_CACHE_PATH" ]]; then
     echo "prepare_docker_cache called without path arg" >&2
@@ -18,6 +18,8 @@ echo "Creating cache directory (${DOCKER_CACHE_PATH}) ..."
 mkdir -p "${DOCKER_CACHE_PATH}"
 if [[ -z "$NO_MOUNT_TMPFS" ]]; then
     echo "Mount tmpfs directory: ${DOCKER_CACHE_PATH}"
-    mount -t tmpfs none "$DOCKER_CACHE_PATH"
+    mount -o size="$TMPFS_SIZE" -t tmpfs none "$DOCKER_CACHE_PATH"
 fi
+mkdir -p "${DOCKER_CACHE_PATH}/docker"
+mkdir -p "${DOCKER_CACHE_PATH}/bazel"
 chown -R "$DOCKER_CACHE_OWNERSHIP" "${DOCKER_CACHE_PATH}"