From 5298dff41944b90d7f5440074d66d88dc183c4b0 Mon Sep 17 00:00:00 2001 From: shilongliu Date: Thu, 28 Jan 2021 08:24:27 +0000 Subject: [PATCH 01/36] add azure pipeline for nightly build Signed-off-by: shilongliu --- pipelines/official_build.yml | 40 ++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 pipelines/official_build.yml diff --git a/pipelines/official_build.yml b/pipelines/official_build.yml new file mode 100644 index 000000000000..7997935b6a63 --- /dev/null +++ b/pipelines/official_build.yml @@ -0,0 +1,40 @@ +# Starter pipeline +# Start with a minimal pipeline that you can customize to build and deploy your code. +# Add steps that build, run tests, deploy, and more: +# https://aka.ms/yaml + +schedules: +- cron: "0 0 * * *" + displayName: Daily midnight build + branches: + include: + - master + always: true + +stages: +- stage: Build + pool: sonicbld + jobs: + - job: + timeoutInMinutes: 720 + steps: + - checkout: self + displayName: 'checkout code' + - bash: | + make init + make configure PLATFORM=$(PLATFORM) + displayName: 'make init & config' + - bash: | + make SONIC_CONFIG_BUILD_JOBS=1 INSTALL_DEBUG_TOOLS=y target/sonic-broadcom.bin + mv target/sonic-broadcom.bin target/sonic-broadcom-dbg.bin + displayName: "make debug bin file" + - bash: | + make SONIC_CONFIG_BUILD_JOBS=1 target/sonic-broadcom.bin + displayName: "make bin file" + - bash: | + make SONIC_CONFIG_BUILD_JOBS=1 target/sonic-aboot-broadcom.swi + displayName: "make swi file" + - publish: $(Build.SourcesDirectory)/target/ + artifact: target + displayName: "archive target" + From 2c43a47882b15d993b78ad3ca828ffd7ff8629bc Mon Sep 17 00:00:00 2001 From: shilongliu Date: Thu, 28 Jan 2021 08:29:02 +0000 Subject: [PATCH 02/36] try cron Signed-off-by: shilongliu --- pipelines/official_build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipelines/official_build.yml b/pipelines/official_build.yml index 7997935b6a63..acfd850f724c 100644 --- a/pipelines/official_build.yml +++ b/pipelines/official_build.yml @@ -4,7 +4,7 @@ # https://aka.ms/yaml schedules: -- cron: "0 0 * * *" +- cron: "5 0 * * *" displayName: Daily midnight build branches: include: From b810f8e6afd65bb68413d62fa3c8f4d6a58e1af5 Mon Sep 17 00:00:00 2001 From: shilongliu Date: Thu, 28 Jan 2021 09:02:49 +0000 Subject: [PATCH 03/36] update azure-pipelines.yml Signed-off-by: shilongliu --- pipelines/official_build.yml | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/pipelines/official_build.yml b/pipelines/official_build.yml index acfd850f724c..ca99324444b7 100644 --- a/pipelines/official_build.yml +++ b/pipelines/official_build.yml @@ -13,6 +13,9 @@ schedules: stages: - stage: Build + variables: + - name: WCACHE + value: "SONIC_DPKG_CACHE_METHOD=wcache SONIC_DPKG_CACHE_SOURCE=/nfs/dpkg_cache/$(PLATFORM)" pool: sonicbld jobs: - job: @@ -25,15 +28,18 @@ stages: make configure PLATFORM=$(PLATFORM) displayName: 'make init & config' - bash: | - make SONIC_CONFIG_BUILD_JOBS=1 INSTALL_DEBUG_TOOLS=y target/sonic-broadcom.bin + make SONIC_CONFIG_BUILD_JOBS=1 $(WCACHE) INSTALL_DEBUG_TOOLS=y target/sonic-broadcom.bin mv target/sonic-broadcom.bin target/sonic-broadcom-dbg.bin displayName: "make debug bin file" - bash: | - make SONIC_CONFIG_BUILD_JOBS=1 target/sonic-broadcom.bin + make SONIC_CONFIG_BUILD_JOBS=1 $(WCACHE) target/sonic-broadcom.bin displayName: "make bin file" - bash: | - make SONIC_CONFIG_BUILD_JOBS=1 target/sonic-aboot-broadcom.swi + make SONIC_CONFIG_BUILD_JOBS=1 $(WCACHE) target/sonic-aboot-broadcom.swi displayName: "make swi file" + - bash: | + make SONIC_CONFIG_BUILD_JOBS=1 $(WCACHE) target/sonic-broadcom.raw + displayName: "make raw file" - publish: $(Build.SourcesDirectory)/target/ artifact: target displayName: "archive target" From de2310fde9fdf990367f15c9bb89988ff3d6deb0 Mon Sep 17 00:00:00 2001 From: shilongliu Date: Thu, 28 Jan 2021 10:49:01 +0000 Subject: [PATCH 04/36] fix script Signed-off-by: shilongliu --- .../official_build.yml => azure-pipelines/official.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) rename pipelines/official_build.yml => azure-pipelines/official.yml (86%) diff --git a/pipelines/official_build.yml b/azure-pipelines/official.yml similarity index 86% rename from pipelines/official_build.yml rename to azure-pipelines/official.yml index ca99324444b7..f41c3831f73a 100644 --- a/pipelines/official_build.yml +++ b/azure-pipelines/official.yml @@ -20,6 +20,7 @@ stages: jobs: - job: timeoutInMinutes: 720 + displayName: Build steps: - checkout: self displayName: 'checkout code' @@ -28,19 +29,23 @@ stages: make configure PLATFORM=$(PLATFORM) displayName: 'make init & config' - bash: | + trap "sudo rm -rf fsroot" EXIT make SONIC_CONFIG_BUILD_JOBS=1 $(WCACHE) INSTALL_DEBUG_TOOLS=y target/sonic-broadcom.bin mv target/sonic-broadcom.bin target/sonic-broadcom-dbg.bin displayName: "make debug bin file" - bash: | + trap "sudo rm -rf fsroot" EXIT make SONIC_CONFIG_BUILD_JOBS=1 $(WCACHE) target/sonic-broadcom.bin displayName: "make bin file" - bash: | + trap "sudo rm -rf fsroot" EXIT make SONIC_CONFIG_BUILD_JOBS=1 $(WCACHE) target/sonic-aboot-broadcom.swi displayName: "make swi file" - bash: | + trap "sudo rm -rf fsroot" EXIT make SONIC_CONFIG_BUILD_JOBS=1 $(WCACHE) target/sonic-broadcom.raw displayName: "make raw file" - publish: $(Build.SourcesDirectory)/target/ - artifact: target + artifact: buildimage.$(PLATFORM) displayName: "archive target" From 60144f2f8379c267afe1813e0700a26bd2a45770 Mon Sep 17 00:00:00 2001 From: Shilong Liu Date: Thu, 28 Jan 2021 19:07:29 +0800 Subject: [PATCH 05/36] change totally Signed-off-by: Shilong Liu --- azure-pipelines/official.yml | 52 +++++++++++++----------------------- 1 file changed, 19 insertions(+), 33 deletions(-) diff --git a/azure-pipelines/official.yml b/azure-pipelines/official.yml index f41c3831f73a..50b47cbd5a3a 100644 --- a/azure-pipelines/official.yml +++ b/azure-pipelines/official.yml @@ -13,39 +13,25 @@ schedules: stages: - stage: Build - variables: - - name: WCACHE - value: "SONIC_DPKG_CACHE_METHOD=wcache SONIC_DPKG_CACHE_SOURCE=/nfs/dpkg_cache/$(PLATFORM)" pool: sonicbld + jobs: - - job: - timeoutInMinutes: 720 - displayName: Build - steps: - - checkout: self - displayName: 'checkout code' - - bash: | - make init - make configure PLATFORM=$(PLATFORM) - displayName: 'make init & config' - - bash: | - trap "sudo rm -rf fsroot" EXIT - make SONIC_CONFIG_BUILD_JOBS=1 $(WCACHE) INSTALL_DEBUG_TOOLS=y target/sonic-broadcom.bin - mv target/sonic-broadcom.bin target/sonic-broadcom-dbg.bin - displayName: "make debug bin file" - - bash: | - trap "sudo rm -rf fsroot" EXIT - make SONIC_CONFIG_BUILD_JOBS=1 $(WCACHE) target/sonic-broadcom.bin - displayName: "make bin file" - - bash: | - trap "sudo rm -rf fsroot" EXIT - make SONIC_CONFIG_BUILD_JOBS=1 $(WCACHE) target/sonic-aboot-broadcom.swi - displayName: "make swi file" - - bash: | - trap "sudo rm -rf fsroot" EXIT - make SONIC_CONFIG_BUILD_JOBS=1 $(WCACHE) target/sonic-broadcom.raw - displayName: "make raw file" - - publish: $(Build.SourcesDirectory)/target/ - artifact: buildimage.$(PLATFORM) - displayName: "archive target" + - job: + displayName: "broadcom" + timeoutInMinutes: 600 + steps: + - checkout: self + submodules: recursive + displayName: 'Checkout code' + - script: | + sudo modprobe overlay + CACHE_OPTIONS="SONIC_DPKG_CACHE_METHOD=wcache SONIC_DPKG_CACHE_SOURCE=/nfs/dpkg_cache/broadcom" + ENABLE_DOCKER_BASE_PULL=y make configure PLATFORM=broadcom + trap "sudo rm -rf fsroot" EXIT + make USERNAME=admin SONIC_BUILD_JOBS=$(nproc) $CACHE_OPTIONS target/sonic-broadcom.bin && \ + ENABLE_SYNCD_RPC=y make USERNAME=admin SONIC_BUILD_JOBS=$(nproc) $CACHE_OPTIONS target/docker-syncd-brcm-rpc.gz + displayName: 'Build sonic image' + - publish: $(System.DefaultWorkingDirectory)/ + artifact: sonic-buildimage.broadcom + displayName: "Archive sonic image" From 32fb3aac2ed7ecbda87fcf78a682aa7d68b37e31 Mon Sep 17 00:00:00 2001 From: Shilong Liu Date: Mon, 1 Feb 2021 16:31:03 +0800 Subject: [PATCH 06/36] change totally Signed-off-by: Shilong Liu --- azure-pipelines/buildjob.yaml | 79 +++++++++++++++++++++++++++++++++++ azure-pipelines/official.yml | 21 +--------- 2 files changed, 80 insertions(+), 20 deletions(-) create mode 100644 azure-pipelines/buildjob.yaml diff --git a/azure-pipelines/buildjob.yaml b/azure-pipelines/buildjob.yaml new file mode 100644 index 000000000000..aeb8bb5034b4 --- /dev/null +++ b/azure-pipelines/buildjob.yaml @@ -0,0 +1,79 @@ +jobs: +- job: + displayName: "broadcom" + timeoutInMinutes: 600 + steps: + - checkout: self + clean: true + submodules: recursive + displayName: 'Checkout code' + - script: | + git submodule foreach --recursive git clean -xfdf + git submodule foreach --recursive git reset --hard + git submodule update --init --recursive + displayName: 'reset submodules' + - script: | + sudo modprobe overlay + CACHE_OPTIONS="SONIC_DPKG_CACHE_METHOD=$(cache_mode) SONIC_DPKG_CACHE_SOURCE=/nfs/dpkg_cache/broadcom" + ENABLE_DOCKER_BASE_PULL=y make configure PLATFORM=broadcom + trap "sudo rm -rf fsroot" EXIT + echo "nproc: $(nproc), CACHE_OPTIONS: $(CACHE_OPTIONS)" + make USERNAME=admin SONIC_BUILD_JOBS=$(nproc) $CACHE_OPTIONS target/sonic-broadcom.bin && \ + ENABLE_SYNCD_RPC=y make USERNAME=admin SONIC_BUILD_JOBS=$(nproc) $CACHE_OPTIONS target/docker-syncd-brcm-rpc.gz + displayName: 'Build sonic image' + - publish: $(System.DefaultWorkingDirectory)/ + artifact: sonic-buildimage.broadcom + displayName: "Archive sonic image" + +- job: + displayName: "mellanox" + timeoutInMinutes: 600 + steps: + - checkout: self + clean: true + submodules: recursive + displayName: 'Checkout code' + + - script: | + git submodule foreach --recursive git clean -xfdf + git submodule foreach --recursive git reset --hard + git submodule update --init --recursive + displayName: 'reset submodules' + - script: | + sudo modprobe overlay + CACHE_OPTIONS="SONIC_DPKG_CACHE_METHOD=$(cache_mode) SONIC_DPKG_CACHE_SOURCE=/nfs/dpkg_cache/mellanox" + ENABLE_DOCKER_BASE_PULL=y make configure PLATFORM=mellanox + trap "sudo rm -rf fsroot" EXIT + make USERNAME=admin SONIC_BUILD_JOBS=$(nproc) $CACHE_OPTIONS target/sonic-mellanox.bin && \ + ENABLE_SYNCD_RPC=y make USERNAME=admin SONIC_BUILD_JOBS=$(nproc) $CACHE_OPTIONS target/docker-syncd-mlnx-rpc.gz + displayName: 'Build sonic image' + - publish: $(System.DefaultWorkingDirectory)/ + artifact: sonic-buildimage.mellanox + displayName: "Archive sonic image" + +- job: + displayName: "kvm" + timeoutInMinutes: 600 + steps: + - checkout: self + clean: true + submodules: recursive + displayName: 'Checkout code' + + - script: | + git submodule foreach --recursive git clean -xfdf + git submodule foreach --recursive git reset --hard + git submodule update --init --recursive + displayName: 'reset submodules' + - script: | + echo $(Build.BuildNumber) + sudo modprobe overlay + CACHE_OPTIONS="SONIC_DPKG_CACHE_METHOD=$(cache_mode) SONIC_DPKG_CACHE_SOURCE=/nfs/dpkg_cache/vs" + ENABLE_DOCKER_BASE_PULL=y make configure PLATFORM=vs + trap "sudo rm -rf fsroot" EXIT + make USERNAME=admin SONIC_BUILD_JOBS=$(nproc) $CACHE_OPTIONS \ + target/docker-sonic-vs.gz target/sonic-vs.img.gz target/docker-ptf.gz + displayName: 'Build sonic image' + - publish: $(System.DefaultWorkingDirectory)/ + artifact: sonic-buildimage.kvm + displayName: "Archive sonic image" \ No newline at end of file diff --git a/azure-pipelines/official.yml b/azure-pipelines/official.yml index 50b47cbd5a3a..ff6fa6ed96f4 100644 --- a/azure-pipelines/official.yml +++ b/azure-pipelines/official.yml @@ -15,23 +15,4 @@ stages: - stage: Build pool: sonicbld - jobs: - - job: - displayName: "broadcom" - timeoutInMinutes: 600 - steps: - - checkout: self - submodules: recursive - displayName: 'Checkout code' - - - script: | - sudo modprobe overlay - CACHE_OPTIONS="SONIC_DPKG_CACHE_METHOD=wcache SONIC_DPKG_CACHE_SOURCE=/nfs/dpkg_cache/broadcom" - ENABLE_DOCKER_BASE_PULL=y make configure PLATFORM=broadcom - trap "sudo rm -rf fsroot" EXIT - make USERNAME=admin SONIC_BUILD_JOBS=$(nproc) $CACHE_OPTIONS target/sonic-broadcom.bin && \ - ENABLE_SYNCD_RPC=y make USERNAME=admin SONIC_BUILD_JOBS=$(nproc) $CACHE_OPTIONS target/docker-syncd-brcm-rpc.gz - displayName: 'Build sonic image' - - publish: $(System.DefaultWorkingDirectory)/ - artifact: sonic-buildimage.broadcom - displayName: "Archive sonic image" + template: buildjob.yaml \ No newline at end of file From 5faa2adda7e9929a6b5a39f1e1e4c6395b4a193c Mon Sep 17 00:00:00 2001 From: liushilongbuaa <584370777@qq.com> Date: Mon, 1 Feb 2021 16:34:41 +0800 Subject: [PATCH 07/36] Update official.yml for Azure Pipelines --- azure-pipelines/official.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines/official.yml b/azure-pipelines/official.yml index ff6fa6ed96f4..81b99048236f 100644 --- a/azure-pipelines/official.yml +++ b/azure-pipelines/official.yml @@ -8,7 +8,7 @@ schedules: displayName: Daily midnight build branches: include: - - master + - testmaster always: true stages: From 1cce65da9df8fd6aefb02184cb4fbeda5317083c Mon Sep 17 00:00:00 2001 From: liushilongbuaa <584370777@qq.com> Date: Mon, 1 Feb 2021 16:36:14 +0800 Subject: [PATCH 08/36] Update official.yml for Azure Pipelines --- azure-pipelines/official.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/azure-pipelines/official.yml b/azure-pipelines/official.yml index 81b99048236f..da17360eb9ee 100644 --- a/azure-pipelines/official.yml +++ b/azure-pipelines/official.yml @@ -15,4 +15,5 @@ stages: - stage: Build pool: sonicbld - template: buildjob.yaml \ No newline at end of file + jobs: + - template: buildjob.yaml \ No newline at end of file From 607a100a40c23373ed1215ac4e277ec946bc4130 Mon Sep 17 00:00:00 2001 From: liushilongbuaa <584370777@qq.com> Date: Mon, 1 Feb 2021 16:40:05 +0800 Subject: [PATCH 09/36] Update official.yml for Azure Pipelines --- azure-pipelines/official.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/azure-pipelines/official.yml b/azure-pipelines/official.yml index da17360eb9ee..625024bac7aa 100644 --- a/azure-pipelines/official.yml +++ b/azure-pipelines/official.yml @@ -11,6 +11,10 @@ schedules: - testmaster always: true +variables: + - name: cache_mode + value: rcache + stages: - stage: Build pool: sonicbld From a40428f0321789a85d0b067b5411288c52b1d0b7 Mon Sep 17 00:00:00 2001 From: Shilong Liu Date: Mon, 1 Feb 2021 19:18:36 +0800 Subject: [PATCH 10/36] update Signed-off-by: Shilong Liu --- azure-pipelines/buildjob.yaml | 9 ++++----- azure-pipelines/official.yml | 3 +-- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/azure-pipelines/buildjob.yaml b/azure-pipelines/buildjob.yaml index aeb8bb5034b4..f2c7d7e838bc 100644 --- a/azure-pipelines/buildjob.yaml +++ b/azure-pipelines/buildjob.yaml @@ -1,3 +1,6 @@ +variables: + - name: CACHE_OPTIONS + value: "SONIC_DPKG_CACHE_METHOD=$(cache_mode) SONIC_DPKG_CACHE_SOURCE=/nfs/dpkg_cache/broadcom" jobs: - job: displayName: "broadcom" @@ -14,10 +17,8 @@ jobs: displayName: 'reset submodules' - script: | sudo modprobe overlay - CACHE_OPTIONS="SONIC_DPKG_CACHE_METHOD=$(cache_mode) SONIC_DPKG_CACHE_SOURCE=/nfs/dpkg_cache/broadcom" ENABLE_DOCKER_BASE_PULL=y make configure PLATFORM=broadcom trap "sudo rm -rf fsroot" EXIT - echo "nproc: $(nproc), CACHE_OPTIONS: $(CACHE_OPTIONS)" make USERNAME=admin SONIC_BUILD_JOBS=$(nproc) $CACHE_OPTIONS target/sonic-broadcom.bin && \ ENABLE_SYNCD_RPC=y make USERNAME=admin SONIC_BUILD_JOBS=$(nproc) $CACHE_OPTIONS target/docker-syncd-brcm-rpc.gz displayName: 'Build sonic image' @@ -41,7 +42,6 @@ jobs: displayName: 'reset submodules' - script: | sudo modprobe overlay - CACHE_OPTIONS="SONIC_DPKG_CACHE_METHOD=$(cache_mode) SONIC_DPKG_CACHE_SOURCE=/nfs/dpkg_cache/mellanox" ENABLE_DOCKER_BASE_PULL=y make configure PLATFORM=mellanox trap "sudo rm -rf fsroot" EXIT make USERNAME=admin SONIC_BUILD_JOBS=$(nproc) $CACHE_OPTIONS target/sonic-mellanox.bin && \ @@ -68,7 +68,6 @@ jobs: - script: | echo $(Build.BuildNumber) sudo modprobe overlay - CACHE_OPTIONS="SONIC_DPKG_CACHE_METHOD=$(cache_mode) SONIC_DPKG_CACHE_SOURCE=/nfs/dpkg_cache/vs" ENABLE_DOCKER_BASE_PULL=y make configure PLATFORM=vs trap "sudo rm -rf fsroot" EXIT make USERNAME=admin SONIC_BUILD_JOBS=$(nproc) $CACHE_OPTIONS \ @@ -76,4 +75,4 @@ jobs: displayName: 'Build sonic image' - publish: $(System.DefaultWorkingDirectory)/ artifact: sonic-buildimage.kvm - displayName: "Archive sonic image" \ No newline at end of file + displayName: "Archive sonic image" diff --git a/azure-pipelines/official.yml b/azure-pipelines/official.yml index 625024bac7aa..4e01ce9319ef 100644 --- a/azure-pipelines/official.yml +++ b/azure-pipelines/official.yml @@ -19,5 +19,4 @@ stages: - stage: Build pool: sonicbld - jobs: - - template: buildjob.yaml \ No newline at end of file + template: buildjob.yaml \ No newline at end of file From b9d1f9f281541abe8c5b679747d952030d5fc9ba Mon Sep 17 00:00:00 2001 From: Shilong Liu Date: Mon, 1 Feb 2021 19:23:35 +0800 Subject: [PATCH 11/36] update Signed-off-by: Shilong Liu --- azure-pipelines/official.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/azure-pipelines/official.yml b/azure-pipelines/official.yml index 4e01ce9319ef..24c29ed10c18 100644 --- a/azure-pipelines/official.yml +++ b/azure-pipelines/official.yml @@ -19,4 +19,5 @@ stages: - stage: Build pool: sonicbld +extends: template: buildjob.yaml \ No newline at end of file From d76d87660025ce7d53a685b00397ebe6bee8c467 Mon Sep 17 00:00:00 2001 From: Shilong Liu Date: Mon, 1 Feb 2021 19:27:32 +0800 Subject: [PATCH 12/36] update Signed-off-by: Shilong Liu --- azure-pipelines/buildjob.yaml | 12 +++++++++--- azure-pipelines/official.yml | 11 +++++------ 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/azure-pipelines/buildjob.yaml b/azure-pipelines/buildjob.yaml index f2c7d7e838bc..e3e32a7bb949 100644 --- a/azure-pipelines/buildjob.yaml +++ b/azure-pipelines/buildjob.yaml @@ -1,10 +1,10 @@ -variables: - - name: CACHE_OPTIONS - value: "SONIC_DPKG_CACHE_METHOD=$(cache_mode) SONIC_DPKG_CACHE_SOURCE=/nfs/dpkg_cache/broadcom" jobs: - job: displayName: "broadcom" timeoutInMinutes: 600 + variables: + - name: CACHE_OPTIONS + value: "SONIC_DPKG_CACHE_METHOD=$(cache_mode) SONIC_DPKG_CACHE_SOURCE=/nfs/dpkg_cache/broadcom" steps: - checkout: self clean: true @@ -29,6 +29,9 @@ jobs: - job: displayName: "mellanox" timeoutInMinutes: 600 + variables: + - name: CACHE_OPTIONS + value: "SONIC_DPKG_CACHE_METHOD=$(cache_mode) SONIC_DPKG_CACHE_SOURCE=/nfs/dpkg_cache/broadcom" steps: - checkout: self clean: true @@ -54,6 +57,9 @@ jobs: - job: displayName: "kvm" timeoutInMinutes: 600 + variables: + - name: CACHE_OPTIONS + value: "SONIC_DPKG_CACHE_METHOD=$(cache_mode) SONIC_DPKG_CACHE_SOURCE=/nfs/dpkg_cache/broadcom" steps: - checkout: self clean: true diff --git a/azure-pipelines/official.yml b/azure-pipelines/official.yml index 24c29ed10c18..8e9511a9015a 100644 --- a/azure-pipelines/official.yml +++ b/azure-pipelines/official.yml @@ -11,13 +11,12 @@ schedules: - testmaster always: true -variables: - - name: cache_mode - value: rcache - stages: - stage: Build pool: sonicbld + variables: + - name: cache_mode + value: rcache -extends: - template: buildjob.yaml \ No newline at end of file + jobs: + - template: buildjob.yaml \ No newline at end of file From 486c8dacfffd76fea09b9eae4c268ca28aca4def Mon Sep 17 00:00:00 2001 From: Shilong Liu Date: Mon, 1 Feb 2021 19:41:05 +0800 Subject: [PATCH 13/36] update Signed-off-by: Shilong Liu --- azure-pipelines/official.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure-pipelines/official.yml b/azure-pipelines/official.yml index 8e9511a9015a..912828a173e4 100644 --- a/azure-pipelines/official.yml +++ b/azure-pipelines/official.yml @@ -16,7 +16,7 @@ stages: pool: sonicbld variables: - name: cache_mode - value: rcache + value: wcache jobs: - - template: buildjob.yaml \ No newline at end of file + - template: buildjob.yaml From 48d27338bc11845d07a3a620284ea7681dd75076 Mon Sep 17 00:00:00 2001 From: Shilong Liu Date: Mon, 1 Feb 2021 20:00:57 +0800 Subject: [PATCH 14/36] update Signed-off-by: Shilong Liu --- azure-pipelines/buildjob.yaml | 28 ---------------------------- 1 file changed, 28 deletions(-) diff --git a/azure-pipelines/buildjob.yaml b/azure-pipelines/buildjob.yaml index e3e32a7bb949..af91d1c4aa58 100644 --- a/azure-pipelines/buildjob.yaml +++ b/azure-pipelines/buildjob.yaml @@ -54,31 +54,3 @@ jobs: artifact: sonic-buildimage.mellanox displayName: "Archive sonic image" -- job: - displayName: "kvm" - timeoutInMinutes: 600 - variables: - - name: CACHE_OPTIONS - value: "SONIC_DPKG_CACHE_METHOD=$(cache_mode) SONIC_DPKG_CACHE_SOURCE=/nfs/dpkg_cache/broadcom" - steps: - - checkout: self - clean: true - submodules: recursive - displayName: 'Checkout code' - - - script: | - git submodule foreach --recursive git clean -xfdf - git submodule foreach --recursive git reset --hard - git submodule update --init --recursive - displayName: 'reset submodules' - - script: | - echo $(Build.BuildNumber) - sudo modprobe overlay - ENABLE_DOCKER_BASE_PULL=y make configure PLATFORM=vs - trap "sudo rm -rf fsroot" EXIT - make USERNAME=admin SONIC_BUILD_JOBS=$(nproc) $CACHE_OPTIONS \ - target/docker-sonic-vs.gz target/sonic-vs.img.gz target/docker-ptf.gz - displayName: 'Build sonic image' - - publish: $(System.DefaultWorkingDirectory)/ - artifact: sonic-buildimage.kvm - displayName: "Archive sonic image" From 5f33ba942c036822854641e56da8b0cea2bba391 Mon Sep 17 00:00:00 2001 From: Shilong Liu Date: Tue, 2 Feb 2021 14:35:33 +0800 Subject: [PATCH 15/36] change directory Signed-off-by: Shilong Liu --- .../build-template.yml | 40 ++++++++++++++++--- .../official.yml | 9 ++--- slave.mk | 1 + 3 files changed, 39 insertions(+), 11 deletions(-) rename azure-pipelines/buildjob.yaml => .azure-pipelines/build-template.yml (59%) rename {azure-pipelines => .azure-pipelines}/official.yml (76%) diff --git a/azure-pipelines/buildjob.yaml b/.azure-pipelines/build-template.yml similarity index 59% rename from azure-pipelines/buildjob.yaml rename to .azure-pipelines/build-template.yml index af91d1c4aa58..c97684b0c13e 100644 --- a/azure-pipelines/buildjob.yaml +++ b/.azure-pipelines/build-template.yml @@ -1,10 +1,13 @@ +parameters: +- name: cache_mode + type: string + values: + - wcache + jobs: - job: displayName: "broadcom" timeoutInMinutes: 600 - variables: - - name: CACHE_OPTIONS - value: "SONIC_DPKG_CACHE_METHOD=$(cache_mode) SONIC_DPKG_CACHE_SOURCE=/nfs/dpkg_cache/broadcom" steps: - checkout: self clean: true @@ -17,6 +20,7 @@ jobs: displayName: 'reset submodules' - script: | sudo modprobe overlay + CACHE_OPTIONS="SONIC_DPKG_CACHE_METHOD=${{ cache_mode }} SONIC_DPKG_CACHE_SOURCE=/nfs/dpkg_cache/broadcom" ENABLE_DOCKER_BASE_PULL=y make configure PLATFORM=broadcom trap "sudo rm -rf fsroot" EXIT make USERNAME=admin SONIC_BUILD_JOBS=$(nproc) $CACHE_OPTIONS target/sonic-broadcom.bin && \ @@ -29,9 +33,6 @@ jobs: - job: displayName: "mellanox" timeoutInMinutes: 600 - variables: - - name: CACHE_OPTIONS - value: "SONIC_DPKG_CACHE_METHOD=$(cache_mode) SONIC_DPKG_CACHE_SOURCE=/nfs/dpkg_cache/broadcom" steps: - checkout: self clean: true @@ -45,6 +46,7 @@ jobs: displayName: 'reset submodules' - script: | sudo modprobe overlay + CACHE_OPTIONS="SONIC_DPKG_CACHE_METHOD=${{ cache_mode }} SONIC_DPKG_CACHE_SOURCE=/nfs/dpkg_cache/mellanox" ENABLE_DOCKER_BASE_PULL=y make configure PLATFORM=mellanox trap "sudo rm -rf fsroot" EXIT make USERNAME=admin SONIC_BUILD_JOBS=$(nproc) $CACHE_OPTIONS target/sonic-mellanox.bin && \ @@ -54,3 +56,29 @@ jobs: artifact: sonic-buildimage.mellanox displayName: "Archive sonic image" +- job: + displayName: "kvm" + timeoutInMinutes: 600 + steps: + - checkout: self + clean: true + submodules: recursive + displayName: 'Checkout code' + + - script: | + git submodule foreach --recursive git clean -xfdf + git submodule foreach --recursive git reset --hard + git submodule update --init --recursive + displayName: 'reset submodules' + - script: | + echo $(Build.BuildNumber) + sudo modprobe overlay + CACHE_OPTIONS="SONIC_DPKG_CACHE_METHOD=${{ cache_mode }} SONIC_DPKG_CACHE_SOURCE=/nfs/dpkg_cache/vs" + ENABLE_DOCKER_BASE_PULL=y make configure PLATFORM=vs + trap "sudo rm -rf fsroot" EXIT + make USERNAME=admin SONIC_BUILD_JOBS=$(nproc) $CACHE_OPTIONS \ + target/docker-sonic-vs.gz target/sonic-vs.img.gz target/docker-ptf.gz + displayName: 'Build sonic image' + - publish: $(System.DefaultWorkingDirectory)/ + artifact: sonic-buildimage.kvm + displayName: "Archive sonic image" \ No newline at end of file diff --git a/azure-pipelines/official.yml b/.azure-pipelines/official.yml similarity index 76% rename from azure-pipelines/official.yml rename to .azure-pipelines/official.yml index 912828a173e4..4cb0c3f2ab90 100644 --- a/azure-pipelines/official.yml +++ b/.azure-pipelines/official.yml @@ -8,15 +8,14 @@ schedules: displayName: Daily midnight build branches: include: - - testmaster + - master always: true stages: - stage: Build pool: sonicbld - variables: - - name: cache_mode - value: wcache jobs: - - template: buildjob.yaml + - template: build-template.yaml + parameters: + cache_mode: wcache diff --git a/slave.mk b/slave.mk index d228efbc3369..3408d6fdea55 100644 --- a/slave.mk +++ b/slave.mk @@ -254,6 +254,7 @@ $(info "INCLUDE_NAT" : "$(INCLUDE_NAT)") $(info "INCLUDE_KUBERNETES" : "$(INCLUDE_KUBERNETES)") $(info "TELEMETRY_WRITABLE" : "$(TELEMETRY_WRITABLE)") $(info "PDDF_SUPPORT" : "$(PDDF_SUPPORT)") +$(info "SONIC_DPKG_CACHE_METHOD" : "$(SONIC_DPKG_CACHE_METHOD)") $(info ) else $(info SONiC Build System for $(CONFIGURED_PLATFORM):$(CONFIGURED_ARCH)) From c564ca5b736b38a3bb6d82e141b17e6ecca5af80 Mon Sep 17 00:00:00 2001 From: Shilong Liu Date: Tue, 2 Feb 2021 14:37:34 +0800 Subject: [PATCH 16/36] update Signed-off-by: Shilong Liu --- .azure-pipelines/official.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/official.yml b/.azure-pipelines/official.yml index 4cb0c3f2ab90..42f3dfd4135b 100644 --- a/.azure-pipelines/official.yml +++ b/.azure-pipelines/official.yml @@ -16,6 +16,6 @@ stages: pool: sonicbld jobs: - - template: build-template.yaml + - template: build-template.yml parameters: cache_mode: wcache From a47df6ee715c14b20a844012fbb3277de4f836ed Mon Sep 17 00:00:00 2001 From: Shilong Liu Date: Tue, 2 Feb 2021 14:39:47 +0800 Subject: [PATCH 17/36] update Signed-off-by: Shilong Liu --- .azure-pipelines/build-template.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.azure-pipelines/build-template.yml b/.azure-pipelines/build-template.yml index c97684b0c13e..e984289396e8 100644 --- a/.azure-pipelines/build-template.yml +++ b/.azure-pipelines/build-template.yml @@ -20,7 +20,7 @@ jobs: displayName: 'reset submodules' - script: | sudo modprobe overlay - CACHE_OPTIONS="SONIC_DPKG_CACHE_METHOD=${{ cache_mode }} SONIC_DPKG_CACHE_SOURCE=/nfs/dpkg_cache/broadcom" + CACHE_OPTIONS="SONIC_DPKG_CACHE_METHOD=${{ parameters.cache_mode }} SONIC_DPKG_CACHE_SOURCE=/nfs/dpkg_cache/broadcom" ENABLE_DOCKER_BASE_PULL=y make configure PLATFORM=broadcom trap "sudo rm -rf fsroot" EXIT make USERNAME=admin SONIC_BUILD_JOBS=$(nproc) $CACHE_OPTIONS target/sonic-broadcom.bin && \ @@ -46,7 +46,7 @@ jobs: displayName: 'reset submodules' - script: | sudo modprobe overlay - CACHE_OPTIONS="SONIC_DPKG_CACHE_METHOD=${{ cache_mode }} SONIC_DPKG_CACHE_SOURCE=/nfs/dpkg_cache/mellanox" + CACHE_OPTIONS="SONIC_DPKG_CACHE_METHOD=${{ parameters.cache_mode }} SONIC_DPKG_CACHE_SOURCE=/nfs/dpkg_cache/mellanox" ENABLE_DOCKER_BASE_PULL=y make configure PLATFORM=mellanox trap "sudo rm -rf fsroot" EXIT make USERNAME=admin SONIC_BUILD_JOBS=$(nproc) $CACHE_OPTIONS target/sonic-mellanox.bin && \ @@ -73,7 +73,7 @@ jobs: - script: | echo $(Build.BuildNumber) sudo modprobe overlay - CACHE_OPTIONS="SONIC_DPKG_CACHE_METHOD=${{ cache_mode }} SONIC_DPKG_CACHE_SOURCE=/nfs/dpkg_cache/vs" + CACHE_OPTIONS="SONIC_DPKG_CACHE_METHOD=${{ parameters.cache_mode }} SONIC_DPKG_CACHE_SOURCE=/nfs/dpkg_cache/vs" ENABLE_DOCKER_BASE_PULL=y make configure PLATFORM=vs trap "sudo rm -rf fsroot" EXIT make USERNAME=admin SONIC_BUILD_JOBS=$(nproc) $CACHE_OPTIONS \ From 2a21fc1f3b176e69fda2957200989156681824b7 Mon Sep 17 00:00:00 2001 From: Shilong Liu Date: Tue, 2 Feb 2021 15:11:29 +0800 Subject: [PATCH 18/36] update Signed-off-by: Shilong Liu --- .azure-pipelines/build-template.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.azure-pipelines/build-template.yml b/.azure-pipelines/build-template.yml index e984289396e8..9326c3b313ab 100644 --- a/.azure-pipelines/build-template.yml +++ b/.azure-pipelines/build-template.yml @@ -81,4 +81,5 @@ jobs: displayName: 'Build sonic image' - publish: $(System.DefaultWorkingDirectory)/ artifact: sonic-buildimage.kvm - displayName: "Archive sonic image" \ No newline at end of file + displayName: "Archive sonic image" + From 8ffa54fd8f574f840cc8c592fb4c7d1a1a52ef96 Mon Sep 17 00:00:00 2001 From: Shilong Liu Date: Tue, 2 Feb 2021 15:30:10 +0800 Subject: [PATCH 19/36] add post cleanup Signed-off-by: Shilong Liu --- .azure-pipelines/build-template.yml | 34 ++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/.azure-pipelines/build-template.yml b/.azure-pipelines/build-template.yml index 9326c3b313ab..9cdb91b3e388 100644 --- a/.azure-pipelines/build-template.yml +++ b/.azure-pipelines/build-template.yml @@ -29,6 +29,17 @@ jobs: - publish: $(System.DefaultWorkingDirectory)/ artifact: sonic-buildimage.broadcom displayName: "Archive sonic image" + - script: | + sudo kill -9 `sudo cat /var/run/march/docker.pid` || true + sudo rm -f /var/run/march/docker.pid || true + sudo rm -rf fsroot || true + git clean -xfdf; + git reset --hard; + git submodule foreach --recursive 'git clean -xfdf || true'; + git submodule foreach --recursive 'git reset --hard || true'; + git submodule foreach --recursive 'git remote update || true'; + git submodule update --init --recursive; + displayName: "Clean Workspace" - job: displayName: "mellanox" @@ -55,6 +66,17 @@ jobs: - publish: $(System.DefaultWorkingDirectory)/ artifact: sonic-buildimage.mellanox displayName: "Archive sonic image" + - script: | + sudo kill -9 `sudo cat /var/run/march/docker.pid` || true + sudo rm -f /var/run/march/docker.pid || true + sudo rm -rf fsroot || true + git clean -xfdf; + git reset --hard; + git submodule foreach --recursive 'git clean -xfdf || true'; + git submodule foreach --recursive 'git reset --hard || true'; + git submodule foreach --recursive 'git remote update || true'; + git submodule update --init --recursive; + displayName: "Clean Workspace" - job: displayName: "kvm" @@ -82,4 +104,14 @@ jobs: - publish: $(System.DefaultWorkingDirectory)/ artifact: sonic-buildimage.kvm displayName: "Archive sonic image" - + - script: | + sudo kill -9 `sudo cat /var/run/march/docker.pid` || true + sudo rm -f /var/run/march/docker.pid || true + sudo rm -rf fsroot || true + git clean -xfdf; + git reset --hard; + git submodule foreach --recursive 'git clean -xfdf || true'; + git submodule foreach --recursive 'git reset --hard || true'; + git submodule foreach --recursive 'git remote update || true'; + git submodule update --init --recursive; + displayName: "Clean Workspace" From e615a71dadd6ce81e33cf16bc262216cec355a4d Mon Sep 17 00:00:00 2001 From: Shilong Liu Date: Tue, 2 Feb 2021 15:39:11 +0800 Subject: [PATCH 20/36] add post cleanup Signed-off-by: Shilong Liu --- .azure-pipelines/build-template.yml | 39 +++++------------------------ .azure-pipelines/cleanup.yml | 12 +++++++++ 2 files changed, 18 insertions(+), 33 deletions(-) create mode 100644 .azure-pipelines/cleanup.yml diff --git a/.azure-pipelines/build-template.yml b/.azure-pipelines/build-template.yml index 9cdb91b3e388..1c14cd0e9f75 100644 --- a/.azure-pipelines/build-template.yml +++ b/.azure-pipelines/build-template.yml @@ -9,6 +9,7 @@ jobs: displayName: "broadcom" timeoutInMinutes: 600 steps: + - template: cleanup.yml - checkout: self clean: true submodules: recursive @@ -29,22 +30,13 @@ jobs: - publish: $(System.DefaultWorkingDirectory)/ artifact: sonic-buildimage.broadcom displayName: "Archive sonic image" - - script: | - sudo kill -9 `sudo cat /var/run/march/docker.pid` || true - sudo rm -f /var/run/march/docker.pid || true - sudo rm -rf fsroot || true - git clean -xfdf; - git reset --hard; - git submodule foreach --recursive 'git clean -xfdf || true'; - git submodule foreach --recursive 'git reset --hard || true'; - git submodule foreach --recursive 'git remote update || true'; - git submodule update --init --recursive; - displayName: "Clean Workspace" + - template: cleanup.yml - job: displayName: "mellanox" timeoutInMinutes: 600 steps: + - template: cleanup.yml - checkout: self clean: true submodules: recursive @@ -66,22 +58,13 @@ jobs: - publish: $(System.DefaultWorkingDirectory)/ artifact: sonic-buildimage.mellanox displayName: "Archive sonic image" - - script: | - sudo kill -9 `sudo cat /var/run/march/docker.pid` || true - sudo rm -f /var/run/march/docker.pid || true - sudo rm -rf fsroot || true - git clean -xfdf; - git reset --hard; - git submodule foreach --recursive 'git clean -xfdf || true'; - git submodule foreach --recursive 'git reset --hard || true'; - git submodule foreach --recursive 'git remote update || true'; - git submodule update --init --recursive; - displayName: "Clean Workspace" + - template: cleanup.yml - job: displayName: "kvm" timeoutInMinutes: 600 steps: + - template: cleanup.yml - checkout: self clean: true submodules: recursive @@ -104,14 +87,4 @@ jobs: - publish: $(System.DefaultWorkingDirectory)/ artifact: sonic-buildimage.kvm displayName: "Archive sonic image" - - script: | - sudo kill -9 `sudo cat /var/run/march/docker.pid` || true - sudo rm -f /var/run/march/docker.pid || true - sudo rm -rf fsroot || true - git clean -xfdf; - git reset --hard; - git submodule foreach --recursive 'git clean -xfdf || true'; - git submodule foreach --recursive 'git reset --hard || true'; - git submodule foreach --recursive 'git remote update || true'; - git submodule update --init --recursive; - displayName: "Clean Workspace" + - template: cleanup.yml diff --git a/.azure-pipelines/cleanup.yml b/.azure-pipelines/cleanup.yml new file mode 100644 index 000000000000..90c994c66edb --- /dev/null +++ b/.azure-pipelines/cleanup.yml @@ -0,0 +1,12 @@ +steps: +- script: | + sudo kill -9 `sudo cat /var/run/march/docker.pid` || true + sudo rm -f /var/run/march/docker.pid || true + sudo rm -rf fsroot || true + git clean -xfdf; + git reset --hard; + git submodule foreach --recursive 'git clean -xfdf || true'; + git submodule foreach --recursive 'git reset --hard || true'; + git submodule foreach --recursive 'git remote update || true'; + git submodule update --init --recursive; + displayName: "Clean Workspace" \ No newline at end of file From 4e7541d96ed3980e77ee8870168a4288aa57745f Mon Sep 17 00:00:00 2001 From: Shilong Liu Date: Tue, 2 Feb 2021 15:41:07 +0800 Subject: [PATCH 21/36] add post cleanup Signed-off-by: Shilong Liu --- .azure-pipelines/cleanup.yml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/.azure-pipelines/cleanup.yml b/.azure-pipelines/cleanup.yml index 90c994c66edb..46e436227941 100644 --- a/.azure-pipelines/cleanup.yml +++ b/.azure-pipelines/cleanup.yml @@ -2,11 +2,5 @@ steps: - script: | sudo kill -9 `sudo cat /var/run/march/docker.pid` || true sudo rm -f /var/run/march/docker.pid || true - sudo rm -rf fsroot || true - git clean -xfdf; - git reset --hard; - git submodule foreach --recursive 'git clean -xfdf || true'; - git submodule foreach --recursive 'git reset --hard || true'; - git submodule foreach --recursive 'git remote update || true'; - git submodule update --init --recursive; + sudo rm -rf sonic-buildimage displayName: "Clean Workspace" \ No newline at end of file From 1c8dd8a2c9bc7fef043b5359d9b96c838b3a34a0 Mon Sep 17 00:00:00 2001 From: Shilong Liu Date: Tue, 2 Feb 2021 15:47:11 +0800 Subject: [PATCH 22/36] add post cleanup Signed-off-by: Shilong Liu --- .azure-pipelines/cleanup.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/cleanup.yml b/.azure-pipelines/cleanup.yml index 46e436227941..57b998744ea3 100644 --- a/.azure-pipelines/cleanup.yml +++ b/.azure-pipelines/cleanup.yml @@ -2,5 +2,5 @@ steps: - script: | sudo kill -9 `sudo cat /var/run/march/docker.pid` || true sudo rm -f /var/run/march/docker.pid || true - sudo rm -rf sonic-buildimage + sudo rm -rf sonic-buildimage || sudo rm -rf ../sonic-buildimage || true displayName: "Clean Workspace" \ No newline at end of file From 0423ebf2fef1f81de739d1c27f1c5977d1b2306b Mon Sep 17 00:00:00 2001 From: Shilong Liu Date: Tue, 2 Feb 2021 15:49:16 +0800 Subject: [PATCH 23/36] update Signed-off-by: Shilong Liu --- slave.mk | 1 - 1 file changed, 1 deletion(-) diff --git a/slave.mk b/slave.mk index 3408d6fdea55..d228efbc3369 100644 --- a/slave.mk +++ b/slave.mk @@ -254,7 +254,6 @@ $(info "INCLUDE_NAT" : "$(INCLUDE_NAT)") $(info "INCLUDE_KUBERNETES" : "$(INCLUDE_KUBERNETES)") $(info "TELEMETRY_WRITABLE" : "$(TELEMETRY_WRITABLE)") $(info "PDDF_SUPPORT" : "$(PDDF_SUPPORT)") -$(info "SONIC_DPKG_CACHE_METHOD" : "$(SONIC_DPKG_CACHE_METHOD)") $(info ) else $(info SONiC Build System for $(CONFIGURED_PLATFORM):$(CONFIGURED_ARCH)) From 6a47703187e0f4ce061a385e9a2a090f59fa432c Mon Sep 17 00:00:00 2001 From: Shilong Liu Date: Tue, 2 Feb 2021 19:23:02 +0800 Subject: [PATCH 24/36] update Signed-off-by: Shilong Liu --- .azure-pipelines/build-template.yml | 102 +++++++++++----------------- .azure-pipelines/official.yml | 9 ++- 2 files changed, 47 insertions(+), 64 deletions(-) diff --git a/.azure-pipelines/build-template.yml b/.azure-pipelines/build-template.yml index 1c14cd0e9f75..719d500dec14 100644 --- a/.azure-pipelines/build-template.yml +++ b/.azure-pipelines/build-template.yml @@ -1,90 +1,66 @@ parameters: +- name: platform + type: string + values: + - broadcom + - mellanox +- name: platform_short + type: string + values: + - brcm + - mlnx + - name: cache_mode type: string values: - wcache -jobs: -- job: - displayName: "broadcom" - timeoutInMinutes: 600 - steps: - - template: cleanup.yml - - checkout: self - clean: true - submodules: recursive - displayName: 'Checkout code' - - script: | - git submodule foreach --recursive git clean -xfdf - git submodule foreach --recursive git reset --hard - git submodule update --init --recursive - displayName: 'reset submodules' - - script: | - sudo modprobe overlay - CACHE_OPTIONS="SONIC_DPKG_CACHE_METHOD=${{ parameters.cache_mode }} SONIC_DPKG_CACHE_SOURCE=/nfs/dpkg_cache/broadcom" - ENABLE_DOCKER_BASE_PULL=y make configure PLATFORM=broadcom - trap "sudo rm -rf fsroot" EXIT - make USERNAME=admin SONIC_BUILD_JOBS=$(nproc) $CACHE_OPTIONS target/sonic-broadcom.bin && \ - ENABLE_SYNCD_RPC=y make USERNAME=admin SONIC_BUILD_JOBS=$(nproc) $CACHE_OPTIONS target/docker-syncd-brcm-rpc.gz - displayName: 'Build sonic image' - - publish: $(System.DefaultWorkingDirectory)/ - artifact: sonic-buildimage.broadcom - displayName: "Archive sonic image" - - template: cleanup.yml +- name: dbg_image + type: boolean -- job: - displayName: "mellanox" - timeoutInMinutes: 600 - steps: - - template: cleanup.yml - - checkout: self - clean: true - submodules: recursive - displayName: 'Checkout code' +- name: swi_image + type: boolean - - script: | - git submodule foreach --recursive git clean -xfdf - git submodule foreach --recursive git reset --hard - git submodule update --init --recursive - displayName: 'reset submodules' - - script: | - sudo modprobe overlay - CACHE_OPTIONS="SONIC_DPKG_CACHE_METHOD=${{ parameters.cache_mode }} SONIC_DPKG_CACHE_SOURCE=/nfs/dpkg_cache/mellanox" - ENABLE_DOCKER_BASE_PULL=y make configure PLATFORM=mellanox - trap "sudo rm -rf fsroot" EXIT - make USERNAME=admin SONIC_BUILD_JOBS=$(nproc) $CACHE_OPTIONS target/sonic-mellanox.bin && \ - ENABLE_SYNCD_RPC=y make USERNAME=admin SONIC_BUILD_JOBS=$(nproc) $CACHE_OPTIONS target/docker-syncd-mlnx-rpc.gz - displayName: 'Build sonic image' - - publish: $(System.DefaultWorkingDirectory)/ - artifact: sonic-buildimage.mellanox - displayName: "Archive sonic image" - - template: cleanup.yml +- name: raw_image + type: boolean + +- name: sync_rpc_image + type: boolean +jobs: - job: - displayName: "kvm" + displayName: ${{ parameters.platform }} timeoutInMinutes: 600 steps: - template: cleanup.yml - checkout: self - clean: true submodules: recursive displayName: 'Checkout code' - - script: | git submodule foreach --recursive git clean -xfdf git submodule foreach --recursive git reset --hard git submodule update --init --recursive displayName: 'reset submodules' - script: | - echo $(Build.BuildNumber) sudo modprobe overlay - CACHE_OPTIONS="SONIC_DPKG_CACHE_METHOD=${{ parameters.cache_mode }} SONIC_DPKG_CACHE_SOURCE=/nfs/dpkg_cache/vs" - ENABLE_DOCKER_BASE_PULL=y make configure PLATFORM=vs - trap "sudo rm -rf fsroot" EXIT - make USERNAME=admin SONIC_BUILD_JOBS=$(nproc) $CACHE_OPTIONS \ - target/docker-sonic-vs.gz target/sonic-vs.img.gz target/docker-ptf.gz + CACHE_OPTIONS="SONIC_DPKG_CACHE_METHOD=${{ parameters.cache_mode }} SONIC_DPKG_CACHE_SOURCE=/nfs/dpkg_cache/${{ parameters.platform }}" + ENABLE_DOCKER_BASE_PULL=y make configure PLATFORM=${{ parameters.platform }} + if [ ${{ parameters.dbg_image }} ];then + make USERNAME=admin $CACHE_OPTIONS SONIC_BUILD_JOBS=$(nproc) INSTALL_DEBUG_TOOLS=y target/sonic-${{ parameters.platform }}.bin && \ + mv target/sonic-${{ parameters.platform }}.bin target/sonic-${{ parameters.platform }}-dbg.bin + fi + if [ ${{ parameters.swi_image }} ];then + make USERNAME=admin $CACHE_OPTIONS SONIC_BUILD_JOBS=$(nproc) ENABLE_IMAGE_SIGNATURE=y target/sonic-aboot-${{ parameters.platform }}.swi + fi + if [ ${{ parameters.raw_image }} ];then + make USERNAME=admin $CACHE_OPTIONS SONIC_BUILD_JOBS=$(nproc) target/sonic-${{ parameters.platform }}.raw + fi + if [ ${{ parameters.sync_rpc_image }} ];then + make USERNAME=admin $CACHE_OPTIONS SONIC_BUILD_JOBS=$(nproc) ENABLE_SYNCD_RPC=y target/docker-syncd-${{ parameters.platform_short }}-rpc.gz + fi + make USERNAME=admin $CACHE_OPTIONS SONIC_BUILD_JOBS=$(nproc) target/sonic-${{ parameters.platform }}.bin displayName: 'Build sonic image' - publish: $(System.DefaultWorkingDirectory)/ - artifact: sonic-buildimage.kvm + artifact: sonic-buildimage.${{ parameters.platform }} displayName: "Archive sonic image" - template: cleanup.yml diff --git a/.azure-pipelines/official.yml b/.azure-pipelines/official.yml index 42f3dfd4135b..f106cb44c4a0 100644 --- a/.azure-pipelines/official.yml +++ b/.azure-pipelines/official.yml @@ -4,7 +4,7 @@ # https://aka.ms/yaml schedules: -- cron: "5 0 * * *" +- cron: "0 8 * * *" displayName: Daily midnight build branches: include: @@ -18,4 +18,11 @@ stages: jobs: - template: build-template.yml parameters: + + platform: broadcom + platform_short: brcm cache_mode: wcache + dbg_image: true + swi_image: true + raw_image: true + sync_rpc_image: true \ No newline at end of file From 243cc3f82c2d69b7abb2628289caa08cd4420af1 Mon Sep 17 00:00:00 2001 From: liushilongbuaa <584370777@qq.com> Date: Tue, 2 Feb 2021 17:46:41 +0800 Subject: [PATCH 25/36] Update cleanup.yml --- .azure-pipelines/cleanup.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.azure-pipelines/cleanup.yml b/.azure-pipelines/cleanup.yml index 57b998744ea3..688e49a77f4d 100644 --- a/.azure-pipelines/cleanup.yml +++ b/.azure-pipelines/cleanup.yml @@ -3,4 +3,5 @@ steps: sudo kill -9 `sudo cat /var/run/march/docker.pid` || true sudo rm -f /var/run/march/docker.pid || true sudo rm -rf sonic-buildimage || sudo rm -rf ../sonic-buildimage || true - displayName: "Clean Workspace" \ No newline at end of file + displayName: "Clean Workspace" + From 1d7790dbaa55562bfba19f5b6a5541e662f20590 Mon Sep 17 00:00:00 2001 From: Shilong Liu Date: Tue, 2 Feb 2021 19:26:54 +0800 Subject: [PATCH 26/36] update Signed-off-by: Shilong Liu --- .azure-pipelines/cleanup.yml | 1 - .azure-pipelines/official.yml | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/.azure-pipelines/cleanup.yml b/.azure-pipelines/cleanup.yml index 688e49a77f4d..8cc2593b879b 100644 --- a/.azure-pipelines/cleanup.yml +++ b/.azure-pipelines/cleanup.yml @@ -4,4 +4,3 @@ steps: sudo rm -f /var/run/march/docker.pid || true sudo rm -rf sonic-buildimage || sudo rm -rf ../sonic-buildimage || true displayName: "Clean Workspace" - diff --git a/.azure-pipelines/official.yml b/.azure-pipelines/official.yml index f106cb44c4a0..d55c3a1e9fe3 100644 --- a/.azure-pipelines/official.yml +++ b/.azure-pipelines/official.yml @@ -18,11 +18,10 @@ stages: jobs: - template: build-template.yml parameters: - platform: broadcom platform_short: brcm cache_mode: wcache dbg_image: true swi_image: true raw_image: true - sync_rpc_image: true \ No newline at end of file + sync_rpc_image: true From cb26972d355868ecb2154a5818517dae49f9f6b4 Mon Sep 17 00:00:00 2001 From: liushilongbuaa <584370777@qq.com> Date: Wed, 3 Feb 2021 13:05:45 +0800 Subject: [PATCH 27/36] Update official.yml for Azure Pipelines --- .azure-pipelines/official.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/official.yml b/.azure-pipelines/official.yml index d55c3a1e9fe3..482f96f5d2aa 100644 --- a/.azure-pipelines/official.yml +++ b/.azure-pipelines/official.yml @@ -20,7 +20,7 @@ stages: parameters: platform: broadcom platform_short: brcm - cache_mode: wcache + cache_mode: rcache dbg_image: true swi_image: true raw_image: true From d9fa028a11d375fd4e9aeea18869ca48bc548407 Mon Sep 17 00:00:00 2001 From: Shilong Liu Date: Wed, 3 Feb 2021 13:08:06 +0800 Subject: [PATCH 28/36] update Signed-off-by: Shilong Liu --- .azure-pipelines/build-template.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.azure-pipelines/build-template.yml b/.azure-pipelines/build-template.yml index 719d500dec14..edd73e4fd20e 100644 --- a/.azure-pipelines/build-template.yml +++ b/.azure-pipelines/build-template.yml @@ -14,6 +14,7 @@ parameters: type: string values: - wcache + - rcache - name: dbg_image type: boolean From eea9b3a41646e1d1fe3fbd0e857f313fb8fb37ce Mon Sep 17 00:00:00 2001 From: Shilong Liu Date: Wed, 3 Feb 2021 15:57:35 +0800 Subject: [PATCH 29/36] update Signed-off-by: Shilong Liu --- .azure-pipelines/build-template.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.azure-pipelines/build-template.yml b/.azure-pipelines/build-template.yml index edd73e4fd20e..ab25421f7a52 100644 --- a/.azure-pipelines/build-template.yml +++ b/.azure-pipelines/build-template.yml @@ -60,6 +60,7 @@ jobs: make USERNAME=admin $CACHE_OPTIONS SONIC_BUILD_JOBS=$(nproc) ENABLE_SYNCD_RPC=y target/docker-syncd-${{ parameters.platform_short }}-rpc.gz fi make USERNAME=admin $CACHE_OPTIONS SONIC_BUILD_JOBS=$(nproc) target/sonic-${{ parameters.platform }}.bin + sudo rm -rf fsroot || true displayName: 'Build sonic image' - publish: $(System.DefaultWorkingDirectory)/ artifact: sonic-buildimage.${{ parameters.platform }} From 965d50a9d9eed9ff6c1463738e38d5c2a2e02963 Mon Sep 17 00:00:00 2001 From: Shilong Liu Date: Wed, 3 Feb 2021 22:20:28 +0800 Subject: [PATCH 30/36] update Signed-off-by: Shilong Liu --- .azure-pipelines/build-template.yml | 1 + .azure-pipelines/cleanup.yml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.azure-pipelines/build-template.yml b/.azure-pipelines/build-template.yml index ab25421f7a52..20bf2674d8a3 100644 --- a/.azure-pipelines/build-template.yml +++ b/.azure-pipelines/build-template.yml @@ -43,6 +43,7 @@ jobs: git submodule update --init --recursive displayName: 'reset submodules' - script: | + set -e sudo modprobe overlay CACHE_OPTIONS="SONIC_DPKG_CACHE_METHOD=${{ parameters.cache_mode }} SONIC_DPKG_CACHE_SOURCE=/nfs/dpkg_cache/${{ parameters.platform }}" ENABLE_DOCKER_BASE_PULL=y make configure PLATFORM=${{ parameters.platform }} diff --git a/.azure-pipelines/cleanup.yml b/.azure-pipelines/cleanup.yml index 8cc2593b879b..fdd9cd568167 100644 --- a/.azure-pipelines/cleanup.yml +++ b/.azure-pipelines/cleanup.yml @@ -2,5 +2,5 @@ steps: - script: | sudo kill -9 `sudo cat /var/run/march/docker.pid` || true sudo rm -f /var/run/march/docker.pid || true - sudo rm -rf sonic-buildimage || sudo rm -rf ../sonic-buildimage || true + sudo rm -rf target || true displayName: "Clean Workspace" From 307c88871b67348d784ae977368fdc00058ba0c3 Mon Sep 17 00:00:00 2001 From: Shilong Liu Date: Thu, 4 Feb 2021 17:07:57 +0800 Subject: [PATCH 31/36] update Signed-off-by: Shilong Liu --- .azure-pipelines/build-template.yml | 1 + .azure-pipelines/official.yml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.azure-pipelines/build-template.yml b/.azure-pipelines/build-template.yml index 20bf2674d8a3..4ca7ceece6eb 100644 --- a/.azure-pipelines/build-template.yml +++ b/.azure-pipelines/build-template.yml @@ -15,6 +15,7 @@ parameters: values: - wcache - rcache + - cache - name: dbg_image type: boolean diff --git a/.azure-pipelines/official.yml b/.azure-pipelines/official.yml index 482f96f5d2aa..4738e7df4989 100644 --- a/.azure-pipelines/official.yml +++ b/.azure-pipelines/official.yml @@ -20,7 +20,7 @@ stages: parameters: platform: broadcom platform_short: brcm - cache_mode: rcache + cache_mode: cache dbg_image: true swi_image: true raw_image: true From 8fdb2585bb10ff2080a37e0a71a5f567830d5dc8 Mon Sep 17 00:00:00 2001 From: Shilong Liu Date: Thu, 4 Feb 2021 19:34:47 +0800 Subject: [PATCH 32/36] update Signed-off-by: Shilong Liu --- .azure-pipelines/build-template.yml | 7 +------ .azure-pipelines/cleanup.yml | 8 +++++++- .azure-pipelines/{official.yml => official-broadcom.yml} | 0 3 files changed, 8 insertions(+), 7 deletions(-) rename .azure-pipelines/{official.yml => official-broadcom.yml} (100%) diff --git a/.azure-pipelines/build-template.yml b/.azure-pipelines/build-template.yml index 4ca7ceece6eb..df1c12b290ed 100644 --- a/.azure-pipelines/build-template.yml +++ b/.azure-pipelines/build-template.yml @@ -38,16 +38,12 @@ jobs: - checkout: self submodules: recursive displayName: 'Checkout code' - - script: | - git submodule foreach --recursive git clean -xfdf - git submodule foreach --recursive git reset --hard - git submodule update --init --recursive - displayName: 'reset submodules' - script: | set -e sudo modprobe overlay CACHE_OPTIONS="SONIC_DPKG_CACHE_METHOD=${{ parameters.cache_mode }} SONIC_DPKG_CACHE_SOURCE=/nfs/dpkg_cache/${{ parameters.platform }}" ENABLE_DOCKER_BASE_PULL=y make configure PLATFORM=${{ parameters.platform }} + trap "sudo rm -rf fsroot" EXIT if [ ${{ parameters.dbg_image }} ];then make USERNAME=admin $CACHE_OPTIONS SONIC_BUILD_JOBS=$(nproc) INSTALL_DEBUG_TOOLS=y target/sonic-${{ parameters.platform }}.bin && \ mv target/sonic-${{ parameters.platform }}.bin target/sonic-${{ parameters.platform }}-dbg.bin @@ -62,7 +58,6 @@ jobs: make USERNAME=admin $CACHE_OPTIONS SONIC_BUILD_JOBS=$(nproc) ENABLE_SYNCD_RPC=y target/docker-syncd-${{ parameters.platform_short }}-rpc.gz fi make USERNAME=admin $CACHE_OPTIONS SONIC_BUILD_JOBS=$(nproc) target/sonic-${{ parameters.platform }}.bin - sudo rm -rf fsroot || true displayName: 'Build sonic image' - publish: $(System.DefaultWorkingDirectory)/ artifact: sonic-buildimage.${{ parameters.platform }} diff --git a/.azure-pipelines/cleanup.yml b/.azure-pipelines/cleanup.yml index fdd9cd568167..55a447eae2b2 100644 --- a/.azure-pipelines/cleanup.yml +++ b/.azure-pipelines/cleanup.yml @@ -2,5 +2,11 @@ steps: - script: | sudo kill -9 `sudo cat /var/run/march/docker.pid` || true sudo rm -f /var/run/march/docker.pid || true - sudo rm -rf target || true + sudo rm -rf fsroot || true + git clean -xfdf + git reset --hard + git submodule foreach --recursive 'git clean -xfdf || true' + git submodule foreach --recursive 'git reset --hard || true' + git submodule foreach --recursive 'git remote update || true' + git submodule update --init --recursive displayName: "Clean Workspace" diff --git a/.azure-pipelines/official.yml b/.azure-pipelines/official-broadcom.yml similarity index 100% rename from .azure-pipelines/official.yml rename to .azure-pipelines/official-broadcom.yml From 8e11f3f0f5b82da94d8ee34bb0d34024b89d459b Mon Sep 17 00:00:00 2001 From: Shilong Liu Date: Fri, 5 Feb 2021 11:33:04 +0800 Subject: [PATCH 33/36] update Signed-off-by: Shilong Liu --- .azure-pipelines/build-template.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.azure-pipelines/build-template.yml b/.azure-pipelines/build-template.yml index df1c12b290ed..fdc700a1486a 100644 --- a/.azure-pipelines/build-template.yml +++ b/.azure-pipelines/build-template.yml @@ -4,6 +4,7 @@ parameters: values: - broadcom - mellanox + - name: platform_short type: string values: From 93dfbcaa92ecf2d3739f0cb2283dcdd7f70cd0e1 Mon Sep 17 00:00:00 2001 From: Shilong Liu Date: Fri, 5 Feb 2021 14:29:43 +0800 Subject: [PATCH 34/36] update Signed-off-by: Shilong Liu --- .azure-pipelines/cleanup.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.azure-pipelines/cleanup.yml b/.azure-pipelines/cleanup.yml index 55a447eae2b2..9723e9b931b8 100644 --- a/.azure-pipelines/cleanup.yml +++ b/.azure-pipelines/cleanup.yml @@ -3,10 +3,10 @@ steps: sudo kill -9 `sudo cat /var/run/march/docker.pid` || true sudo rm -f /var/run/march/docker.pid || true sudo rm -rf fsroot || true - git clean -xfdf - git reset --hard - git submodule foreach --recursive 'git clean -xfdf || true' - git submodule foreach --recursive 'git reset --hard || true' - git submodule foreach --recursive 'git remote update || true' - git submodule update --init --recursive + git clean -xfdf || true + git reset --hard || true + git submodule foreach --recursive 'git clean -xfdf || true' || true + git submodule foreach --recursive 'git reset --hard || true' || true + git submodule foreach --recursive 'git remote update || true' || true + git submodule update --init --recursive || true displayName: "Clean Workspace" From 162958e3f43a592d3bc4a9a95664b93a743e6ca7 Mon Sep 17 00:00:00 2001 From: Shilong Liu Date: Fri, 5 Feb 2021 15:27:24 +0800 Subject: [PATCH 35/36] update Signed-off-by: Shilong Liu --- .azure-pipelines/build-template.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/build-template.yml b/.azure-pipelines/build-template.yml index fdc700a1486a..083302c5e521 100644 --- a/.azure-pipelines/build-template.yml +++ b/.azure-pipelines/build-template.yml @@ -35,10 +35,10 @@ jobs: displayName: ${{ parameters.platform }} timeoutInMinutes: 600 steps: - - template: cleanup.yml - checkout: self submodules: recursive displayName: 'Checkout code' + - template: cleanup.yml - script: | set -e sudo modprobe overlay From e799a2c73c926b32b702635f95a47b6cee5f54fc Mon Sep 17 00:00:00 2001 From: Shilong Liu Date: Fri, 5 Feb 2021 15:31:17 +0800 Subject: [PATCH 36/36] update Signed-off-by: Shilong Liu --- .azure-pipelines/official-broadcom.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/official-broadcom.yml b/.azure-pipelines/official-broadcom.yml index 4738e7df4989..d55c3a1e9fe3 100644 --- a/.azure-pipelines/official-broadcom.yml +++ b/.azure-pipelines/official-broadcom.yml @@ -20,7 +20,7 @@ stages: parameters: platform: broadcom platform_short: brcm - cache_mode: cache + cache_mode: wcache dbg_image: true swi_image: true raw_image: true