[Build][202012] Support Debian snapshot mirror to improve build stabi…

…lity (#14558)

#### Why I did it
Cherry-pick commits from master to support the snapshot based mirror, and fix the code conflicts. And add the last commit to fix the build broken issue according to the mirror change.

ad162ae [Build] Optimize the version control for Debian packages (#14557)
38c5d7f [Build] Support j2 template for debian sources for docker ptf (#13198)
5e4826e  [Ci] Support to use the same snapshot for all platform builds (#13913)
8206925 [Build] Change the default mirror version config file (#13786)
5e4a866 [Build] Support Debian snapshot mirror to improve build stability (#13097)
ac5d89c  [Build] Support j2 template for debian sources (#12557)

.azure-pipelines/azure-pipelines-UpgrateVersion.yml

@@ -40,12 +40,32 @@ parameters:
     - marvell-armhf
 
 stages:
+- stage: Prepare
+  jobs:
+  - job: Prepare
+    steps:
+    - script: |
+        DEFAULT_MIRROR_URL_PREFIX=http://packages.trafficmanager.net
+        DEBIAN_TIMESTAMP=$(curl $DEFAULT_MIRROR_URL_PREFIX/snapshot/debian/latest/timestamp)
+        DEBIAN_SECURITY_TIMESTAMP=$(curl $DEFAULT_MIRROR_URL_PREFIX/snapshot/debian-security/latest/timestamp)
+        echo "DEBIAN_TIMESTAMP=$DEBIAN_TIMESTAMP, DEBIAN_SECURITY_TIMESTAMP=$DEBIAN_SECURITY_TIMESTAMP"
+        echo "##vso[task.setvariable variable=DEBIAN_TIMESTAMP;isOutput=true]$DEBIAN_TIMESTAMP"
+        echo "##vso[task.setvariable variable=DEBIAN_SECURITY_TIMESTAMP;isOutput=true]$DEBIAN_SECURITY_TIMESTAMP"
+      name: SetVersions
+      displayName: 'Set snapshot versions'
 - stage: Build
+  dependsOn: Prepare
   variables:
     - name: CACHE_MODE
       value: none
     - name: VERSION_CONTROL_OPTIONS
       value: 'SONIC_VERSION_CONTROL_COMPONENTS='
+    - name: SKIP_CHECKOUT
+      value: true
+    - name: DEBIAN_TIMESTAMP
+      value: $[ stageDependencies.Prepare.Prepare.outputs['SetVersions.DEBIAN_TIMESTAMP'] ]
+    - name: DEBIAN_SECURITY_TIMESTAMP
+      value: $[ stageDependencies.Prepare.Prepare.outputs['SetVersions.DEBIAN_SECURITY_TIMESTAMP'] ]
     - template: .azure-pipelines/template-variables.yml@buildimage
   jobs:
   - template: azure-pipelines-build.yml
@@ -54,6 +74,21 @@ stages:
       buildOptions: '${{ variables.VERSION_CONTROL_OPTIONS }} ENABLE_DOCKER_BASE_PULL=n SONIC_BUILD_JOBS=$(nproc) ENABLE_IMAGE_SIGNATURE=y'
       preSteps:
       - template: .azure-pipelines/template-clean-sonic-slave.yml@buildimage
+      - checkout: self
+        submodules: recursive
+        fetchDepth: 0
+        path: s
+        displayName: 'Checkout code'
+      - script: |
+          echo "DEBIAN_TIMESTAMP=$DEBIAN_TIMESTAMP, DEBIAN_SECURITY_TIMESTAMP=$DEBIAN_SECURITY_TIMESTAMP"
+          if [ "$MIRROR_SNAPSHOT" == y ]; then
+            mkdir -p target/versions/default/
+            echo "debian==$DEBIAN_TIMESTAMP" > target/versions/default/versions-mirror
+            echo "debian-security==$DEBIAN_SECURITY_TIMESTAMP" >> target/versions/default/versions-mirror
+            cat target/versions/default/versions-mirror
+          fi
+        displayName: 'Set snapshot versions'
+
 - stage: UpgradeVersions
   jobs:
   - job: UpgradeVersions

.gitignore

@@ -77,3 +77,7 @@ files/build/tmp
 dockers/**/buildinfo
 platform/**/buildinfo
 sonic-slave*/**/buildinfo
+
+# Debian mirror Sources
+sources.list.*
+!sources.list*.j2

Makefile.work

@@ -135,12 +135,13 @@ SLAVE_IMAGE = $(SLAVE_BASE_IMAGE)-$(USER_LC)
 # Generate the version control build info
 $(shell SONIC_VERSION_CONTROL_COMPONENTS=$(SONIC_VERSION_CONTROL_COMPONENTS) \
     TRUSTED_GPG_URLS=$(TRUSTED_GPG_URLS) PACKAGE_URL_PREFIX=$(PACKAGE_URL_PREFIX) \
+    MIRROR_SNAPSHOT=$(MIRROR_SNAPSHOT) \
     scripts/generate_buildinfo_config.sh)
 
 # Generate the slave Dockerfile, and prepare build info for it
 $(shell CONFIGURED_ARCH=$(CONFIGURED_ARCH) MULTIARCH_QEMU_ENVIRON=$(MULTIARCH_QEMU_ENVIRON) DOCKER_EXTRA_OPTS=$(DOCKER_EXTRA_OPTS) DEFAULT_CONTAINER_REGISTRY=$(DEFAULT_CONTAINER_REGISTRY) j2 $(SLAVE_DIR)/Dockerfile.j2 > $(SLAVE_DIR)/Dockerfile)
 $(shell CONFIGURED_ARCH=$(CONFIGURED_ARCH) MULTIARCH_QEMU_ENVIRON=$(MULTIARCH_QEMU_ENVIRON) j2 $(SLAVE_DIR)/Dockerfile.user.j2 > $(SLAVE_DIR)/Dockerfile.user)
-$(shell BUILD_SLAVE=y DEFAULT_CONTAINER_REGISTRY=$(DEFAULT_CONTAINER_REGISTRY) scripts/prepare_docker_buildinfo.sh $(SLAVE_BASE_IMAGE) $(SLAVE_DIR)/Dockerfile $(CONFIGURED_ARCH) "" $(BLDENV))
+$(shell BUILD_SLAVE=y DEFAULT_CONTAINER_REGISTRY=$(DEFAULT_CONTAINER_REGISTRY) MIRROR_SNAPSHOT=$(MIRROR_SNAPSHOT) scripts/prepare_docker_buildinfo.sh $(SLAVE_BASE_IMAGE) $(SLAVE_DIR)/Dockerfile $(CONFIGURED_ARCH) "" $(BLDENV))
 
 # Add the versions in the tag, if the version change, need to rebuild the slave
 SLAVE_BASE_TAG = $(shell cat $(SLAVE_DIR)/Dockerfile $(SLAVE_DIR)/buildinfo/versions/versions-* src/sonic-build-hooks/hooks/* | sha1sum | awk '{print substr($$1,0,11);}')
@@ -280,13 +281,20 @@ SONIC_BUILD_INSTRUCTION :=  make \
                            SONIC_ENABLE_IMAGE_SIGNATURE=$(ENABLE_IMAGE_SIGNATURE) \
                            SONIC_DEFAULT_CONTAINER_REGISTRY=$(DEFAULT_CONTAINER_REGISTRY) \
                            SLAVE_DIR=$(SLAVE_DIR) \
+                           MIRROR_URLS=$(MIRROR_URLS) \
+                           MIRROR_SECURITY_URLS=$(MIRROR_SECURITY_URLS) \
+                           MIRROR_SNAPSHOT=$(MIRROR_SNAPSHOT) \
                            $(SONIC_OVERRIDE_BUILD_VARS)
 
 .PHONY: sonic-slave-build sonic-slave-bash init reset
 
 .DEFAULT_GOAL :=  all
 
-%::
+export MIRROR_URLS
+export MIRROR_SECURITY_URLS
+export SONIC_VERSION_CONTROL_COMPONENTS
+
+%:: | sonic-build-hooks
 ifeq ($(MULTIARCH_QEMU_ENVIRON), y)
 	@$(DOCKER_MULTIARCH_CHECK)
 ifneq ($(BLDENV), )
@@ -296,8 +304,6 @@ endif
 endif
 	@$(OVERLAY_MODULE_CHECK)
 
-	@pushd src/sonic-build-hooks; TRUSTED_GPG_URLS=$(TRUSTED_GPG_URLS) make all; popd
-	@cp src/sonic-build-hooks/buildinfo/sonic-build-hooks* $(SLAVE_DIR)/buildinfo
 	@docker inspect --type image $(SLAVE_BASE_IMAGE):$(SLAVE_BASE_TAG) &> /dev/null || \
 	    { [ $(ENABLE_DOCKER_BASE_PULL) == y ] && { echo Image $(SLAVE_BASE_IMAGE):$(SLAVE_BASE_TAG) not found. Pulling...; } && \
 	      $(DOCKER_BASE_PULL) && \
@@ -322,6 +328,8 @@ endif
 sonic-build-hooks:
 	@pushd src/sonic-build-hooks; TRUSTED_GPG_URLS=$(TRUSTED_GPG_URLS) make all; popd
 	@cp src/sonic-build-hooks/buildinfo/sonic-build-hooks* $(SLAVE_DIR)/buildinfo
+	@[ "$(MULTIARCH_QEMU_ENVIRON)" == y ] && scripts/build_mirror_config.sh $(SLAVE_DIR) amd64 $(BLDENV)
+	@scripts/build_mirror_config.sh $(SLAVE_DIR) $(CONFIGURED_ARCH) $(BLDENV)
 
 sonic-slave-base-build : sonic-build-hooks
 	@$(OVERLAY_MODULE_CHECK)

build_debian.sh

@@ -107,6 +107,7 @@ sudo LANG=C chroot $FILESYSTEM_ROOT mount
 [ -d $TRUSTED_GPG_DIR ] && [ ! -z "$(ls $TRUSTED_GPG_DIR)" ] && sudo cp $TRUSTED_GPG_DIR/* ${FILESYSTEM_ROOT}/etc/apt/trusted.gpg.d/
 
 ## Pointing apt to public apt mirrors and getting latest packages, needed for latest security updates
+scripts/build_mirror_config.sh files/apt $CONFIGURED_ARCH $IMAGE_DISTRO 
 sudo cp files/apt/sources.list.$CONFIGURED_ARCH $FILESYSTEM_ROOT/etc/apt/sources.list
 sudo cp files/apt/apt.conf.d/{81norecommends,apt-{clean,gzip-indexes,no-languages},no-check-valid-until} $FILESYSTEM_ROOT/etc/apt/apt.conf.d/
 

dockers/docker-base-buster/Dockerfile.j2

@@ -27,13 +27,7 @@ ENV DEBIAN_FRONTEND=noninteractive
 
 # Configure data sources for apt/dpkg
 COPY ["dpkg_01_drop", "/etc/dpkg/dpkg.cfg.d/01_drop"]
-{% if CONFIGURED_ARCH == "armhf" %}
-COPY ["sources.list.armhf", "/etc/apt/sources.list"]
-{% elif CONFIGURED_ARCH == "arm64" %}
-COPY ["sources.list.arm64", "/etc/apt/sources.list"]
-{% else %}
-COPY ["sources.list", "/etc/apt/sources.list"]
-{% endif %}
+COPY ["sources.list.{{ CONFIGURED_ARCH }}", "/etc/apt/sources.list"]
 COPY ["no_install_recommend_suggest", "/etc/apt/apt.conf.d"]
 COPY ["no-check-valid-until", "/etc/apt/apt.conf.d"]
 

dockers/docker-base-stretch/Dockerfile.j2

@@ -27,13 +27,7 @@ ENV DEBIAN_FRONTEND=noninteractive
 
 # Configure data sources for apt/dpkg
 COPY ["dpkg_01_drop", "/etc/dpkg/dpkg.cfg.d/01_drop"]
-{% if CONFIGURED_ARCH == "armhf" %}
-COPY ["sources.list.armhf", "/etc/apt/sources.list"]
-{% elif CONFIGURED_ARCH == "arm64" %}
-COPY ["sources.list.arm64", "/etc/apt/sources.list"]
-{% else %}
-COPY ["sources.list", "/etc/apt/sources.list"]
-{% endif %}
+COPY ["sources.list.{{ CONFIGURED_ARCH }}", "/etc/apt/sources.list"]
 COPY ["no_install_recommend_suggest", "/etc/apt/apt.conf.d"]
 COPY ["no-check-valid-until", "/etc/apt/apt.conf.d"]
 

dockers/docker-ptf/Dockerfile.j2

@@ -9,23 +9,21 @@ FROM {{ prefix }}debian:stretch
 
 MAINTAINER Pavel Shirshov
 
-## Remove retired jessie-updates repo
-RUN sed -i '/deb http:\/\/deb.debian.org\/debian jessie-updates main/d' /etc/apt/sources.list
-
 ## Copy dependencies
 COPY \
 {% for deb in docker_ptf_debs.split(' ') -%}
 debs/{{ deb }}{{' '}}
 {%- endfor -%}
 debs/
 
+COPY ["sources.list.{{ CONFIGURED_ARCH }}", "/etc/apt/sources.list"]
+
 ## Make apt-get non-interactive
 ENV DEBIAN_FRONTEND=noninteractive
 
 ## Set the apt source, update package cache and install necessary packages
 ## TODO: Clean up this step
-RUN sed --in-place 's/httpredir.debian.org/debian-archive.trafficmanager.net/' /etc/apt/sources.list \
-    && apt-get update          \
+RUN apt-get update          \
     && apt-get upgrade -y   \
     && apt-get dist-upgrade -y  \
     && apt-get install -y   \

files/apt/sources.list.j2

@@ -0,0 +1,20 @@
+# The configuration is generated by template
+# Please add additional sources in /etc/apt/sources.list.d
+
+{% for mirror_url in MIRROR_URLS.split(',') %}
+deb [arch={{ ARCHITECTURE }}] {{ mirror_url }} {{ DISTRIBUTION }} main contrib non-free
+deb-src [arch={{ ARCHITECTURE }}] {{ mirror_url }} {{ DISTRIBUTION }} main contrib non-free
+deb [arch={{ ARCHITECTURE }}] {{ mirror_url }} {{ DISTRIBUTION }}-updates main contrib non-free
+deb-src [arch={{ ARCHITECTURE }}] {{ mirror_url }} {{ DISTRIBUTION }}-updates main contrib non-free
+deb [arch={{ ARCHITECTURE }}] {{ mirror_url }} {{ DISTRIBUTION }}-backports main contrib non-free
+{% endfor %}
+{% for mirror_url in MIRROR_SECURITY_URLS.split(',') %}
+{% set dist_separator='/' %}{% if 'packages.trafficmanager.net/debian' in mirror_url %}{% set dist_separator='_' %}{% endif %}
+{% if DISTRIBUTION == 'stretch' or DISTRIBUTION == 'buster' %}
+deb [arch={{ ARCHITECTURE }}] {{ mirror_url }} {{ DISTRIBUTION }}{{ dist_separator }}updates main contrib non-free
+deb-src [arch={{ ARCHITECTURE }}] {{ mirror_url }} {{ DISTRIBUTION }}{{ dist_separator }}updates main contrib non-free
+{% else %}
+deb [arch={{ ARCHITECTURE }}] {{ mirror_url }} {{ DISTRIBUTION }}-security main contrib non-free
+deb-src [arch={{ ARCHITECTURE }}] {{ mirror_url }} {{ DISTRIBUTION }}-security main contrib non-free
+{% endif %}
+{% endfor %}