[Bf2] SONiC Bringup on BF2 (#2)

- First installation for development phase: boot and run on BlueField-2 using BFB (Bluefield Boot Stream) ​
- Support Grub​
- SONiC to SONiC installer must be working.​
- Integrate the bare minimum drivers/packages into the Kernel​
- Define SKU's to reflect BlueField-2 as host, no data ports yet​
- Docker deemed necessary for SDN appliance have to be running (except swss, syncd & pmon). No exceptions on host services.
- Platform is named nvidia-bluefield. Individual variants are identified by OPN root. Eg: MBF2M516A

.gitmodules

@@ -1,15 +1,19 @@
 [submodule "sonic-swss-common"]
 	path = src/sonic-swss-common
-	url = https://github.com/Azure/sonic-swss-common
+	url = ssh://git@github.com/nvidia-sonic/sonic-swss-common
+	branch = bf2
 [submodule "sonic-linux-kernel"]
 	path = src/sonic-linux-kernel
-	url = https://github.com/Azure/sonic-linux-kernel
+	url = ssh://git@github.com/nvidia-sonic/sonic-linux-kernel
+	branch = bf2
 [submodule "sonic-sairedis"]
 	path = src/sonic-sairedis
-	url = https://github.com/Azure/sonic-sairedis
+	url = ssh://git@github.com/nvidia-sonic/sonic-sairedis
+	branch = bf2
 [submodule "sonic-swss"]
 	path = src/sonic-swss
-	url = https://github.com/Azure/sonic-swss
+	url = ssh://git@github.com/nvidia-sonic/sonic-swss
+	branch = bf2
 [submodule "src/p4c-bm/p4c-bm"]
 	path = platform/p4/p4c-bm/p4c-bm
 	url = https://github.com/krambn/p4c-bm
@@ -30,7 +34,8 @@
 	url = https://github.com/p4lang/ptf.git
 [submodule "src/sonic-utilities"]
 	path = src/sonic-utilities
-	url = https://github.com/Azure/sonic-utilities
+	url = ssh://git@github.com/nvidia-sonic/sonic-utilities
+	branch = bf2
 [submodule "platform/broadcom/sonic-platform-modules-arista"]
 	path = platform/broadcom/sonic-platform-modules-arista
 	url = https://github.com/aristanetworks/sonic
@@ -97,3 +102,6 @@
 [submodule "src/linkmgrd"]
 	path = src/linkmgrd
 	url = https://github.com/Azure/sonic-linkmgrd.git
+[submodule "platform/nvidia-bluefield/bfscripts"]
+	path = platform/nvidia-bluefield/bfscripts
+	url = https://github.com/Mellanox/bfscripts

Makefile.work

@@ -194,7 +194,8 @@ ifeq ($(DOCKER_DATA_ROOT_FOR_MULTIARCH),)
 endif
     # Multiarch docker cannot start dockerd service due to iptables cannot run over different arch kernel
     SONIC_SERVICE_DOCKERD_FOR_MULTIARCH=y
-    SONIC_NATIVE_DOCKERD_FOR_MUTLIARCH := dockerd  --experimental=true --storage-driver=vfs \
+	# Workaround until a fix is seen for https://github.com/Azure/sonic-buildimage/issues/8898
+    SONIC_NATIVE_DOCKERD_FOR_MUTLIARCH := dockerd --config-file= --experimental=true --storage-driver=vfs \
         --data-root=$(DOCKER_DATA_ROOT_FOR_MULTIARCH) --exec-root=/var/run/march/docker/ \
         -H unix:///var/run/march/docker.sock -p /var/run/march/docker.pid
     DOCKER_RUN += -v /var/run/march/docker.sock:/var/run/docker.sock
@@ -216,7 +217,7 @@ endif
     # Also dockerd does mkdir on the provided softlink, so used two level path "d/d"
     D_ROOT=/tmp/d/d
     SONIC_NATIVE_DOCKERD_FOR_DOCKERFS := rm -fr $(PWD)/dockerfs; mkdir -p $(PWD)/dockerfs; sudo rm -fr /tmp/d; mkdir -p /tmp/d; ln -s -f $(PWD)/dockerfs $(D_ROOT); \
-					sudo dockerd --storage-driver=overlay2 --iptables=false \
+					sudo dockerd --config-file= --storage-driver=overlay2 --iptables=false \
 					--data-root $(D_ROOT)/var/lib/docker/ --exec-root=$(D_ROOT)/var/run/docker/ \
 					-H unix://$(D_ROOT)/var/run/docker.sock -p $(D_ROOT)/var/run/docker.pid &
     SONIC_USERFACL_DOCKERD_FOR_DOCKERFS := setfacl -m user:$(USER):rw $(D_ROOT)/var/run/docker.sock

bfb-mk-demo.sh

@@ -0,0 +1,51 @@
+#!/bin/bash
+
+REPO="mellanox/bluefield"
+TAG="bfb_builder_denian10-5.4-2.4.1.3-3.7.1.11866-1"
+
+CDIR="/root/sonic/conf"
+SDIR="/root/sonic"
+DDIR="/sonic/"
+TARGET_MACHINE=nvidia-bluefield
+
+fail_with_cmd()
+{
+    echo $1
+    exit 1
+}
+
+# Check if the image already exists 
+docker image inspect $REPO:$TAG &> /dev/null
+if [ $? -ne 0 ]; then
+    echo "Image doesn't Exist, fetching $TAG from $REPO"
+    docker pull $REPO:$TAG || fail_with_cmd "docker pull failed"
+fi
+
+. onie-image-arm64.conf
+
+SONIC_VERSION=$(cat $FILESYSTEM_ROOT/etc/sonic/sonic_version.yml | grep build_version | cut -f2 -d" ")
+
+echo "SONIC_VERSION: $SONIC_VERSION"
+
+if [ ! -f "$ONIE_INSTALLER_PAYLOAD" ]; then
+    echo "SONIC Related Payload '$ONIE_INSTALLER_PAYLOAD' is not found at $(pwd) exiting"
+    exit 1
+fi
+
+docker run --privileged --init -e container=docker\
+        -e "CDIR=$CDIR" -e "SDIR=$SDIR" -e "DDIR=$DDIR" \
+        --mount type=bind,source="$(pwd)",target=/sonic \
+        $REPO:$TAG \
+        bash -c "mkdir -p $SDIR &&  
+                 rm -rf $SDIR/* && 
+                 mkdir -p $CDIR &&
+                 cp /sonic/$ONIE_INSTALLER_PAYLOAD $CDIR &&
+                 cp /sonic/onie-image-arm64.conf $CDIR &&
+                 cp /sonic/installer/bluefield/create_sonic_bfb $SDIR &&
+                 cp /sonic/installer/bluefield/install.sh $SDIR &&
+                 cp /root/workspace/install.bfb $SDIR &&
+                 ls $SDIR &&
+                 /root/sonic/create_sonic_bfb -s $SONIC_VERSION" || fail_with_cmd "docker run failed"
+
+echo "Building BFB Finished"
+exit 0

build_debian.sh

@@ -29,6 +29,7 @@
 set -x -e
 
 CONFIGURED_ARCH=$([ -f .arch ] && cat .arch || echo amd64)
+CONFIGURED_PLATFORM=$([ -f .platform ] && cat .platform || echo vs)
 
 ## docker engine version (with platform)
 DOCKER_VERSION=5:20.10.7~3-0~debian-$IMAGE_DISTRO
@@ -43,7 +44,6 @@ DEFAULT_USERINFO="Default admin user,,,"
 BUILD_TOOL_PATH=src/sonic-build-hooks/buildinfo
 TRUSTED_GPG_DIR=$BUILD_TOOL_PATH/trusted.gpg.d
 
-## Read ONIE image related config file
 . ./onie-image.conf
 [ -n "$ONIE_IMAGE_PART_SIZE" ] || {
     echo "Error: Invalid ONIE_IMAGE_PART_SIZE in onie image config file"
@@ -179,7 +179,7 @@ sudo chmod +x $FILESYSTEM_ROOT/etc/initramfs-tools/scripts/init-bottom/varlog
 sudo cp files/initramfs-tools/union-fsck $FILESYSTEM_ROOT/etc/initramfs-tools/hooks/union-fsck
 sudo chmod +x $FILESYSTEM_ROOT/etc/initramfs-tools/hooks/union-fsck
 pushd $FILESYSTEM_ROOT/usr/share/initramfs-tools/scripts/init-bottom && sudo patch -p1 < $OLDPWD/files/initramfs-tools/udev.patch; popd
-if [[ $CONFIGURED_ARCH == armhf || $CONFIGURED_ARCH == arm64 ]]; then
+if [[ $CONFIGURED_PLATFORM != nvidia-bluefield && ( $CONFIGURED_ARCH == armhf || $CONFIGURED_ARCH == arm64 ) ]]; then
     sudo cp files/initramfs-tools/uboot-utils $FILESYSTEM_ROOT/etc/initramfs-tools/hooks/uboot-utils
     sudo chmod +x $FILESYSTEM_ROOT/etc/initramfs-tools/hooks/uboot-utils
     cat files/initramfs-tools/modules.arm | sudo tee -a $FILESYSTEM_ROOT/etc/initramfs-tools/modules > /dev/null
@@ -324,6 +324,7 @@ sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get -y in
     haveged                 \
     fdisk                   \
     gpg                     \
+    dmidecode               \
     jq
 
 if [[ $CONFIGURED_ARCH == amd64 ]]; then
@@ -536,7 +537,7 @@ fi
 ## Update initramfs
 sudo chroot $FILESYSTEM_ROOT update-initramfs -u
 ## Convert initrd image to u-boot format
-if [[ $CONFIGURED_ARCH == armhf || $CONFIGURED_ARCH == arm64 ]]; then
+if [[ $CONFIGURED_PLATFORM != nvidia-bluefield && ( $CONFIGURED_ARCH == armhf || $CONFIGURED_ARCH == arm64 ) ]]; then
     INITRD_FILE=initrd.img-${LINUX_KERNEL_VERSION}-${CONFIGURED_ARCH}
     if [[ $CONFIGURED_ARCH == armhf ]]; then
         INITRD_FILE=initrd.img-${LINUX_KERNEL_VERSION}-armmp

build_image.sh

@@ -7,6 +7,7 @@ set -x -e
 ## Read ONIE image related config file
 
 CONFIGURED_ARCH=$([ -f .arch ] && cat .arch || echo amd64)
+CONFIGURED_PLATFORM=$([ -f .platform ] && cat .platform || echo vs)
 
 if [[ $CONFIGURED_ARCH == armhf || $CONFIGURED_ARCH == arm64 ]]; then
     . ./onie-image-${CONFIGURED_ARCH}.conf
@@ -28,13 +29,13 @@ IMAGE_VERSION="${SONIC_IMAGE_VERSION}"
 generate_kvm_image()
 {
     NUM_ASIC=$1
-    if [ $NUM_ASIC == 4 ]; then 
+    if [ $NUM_ASIC == 4 ]; then
          KVM_IMAGE=$OUTPUT_KVM_4ASIC_IMAGE
          RECOVERY_ISO=$onie_recovery_kvm_4asic_image
     elif [ $NUM_ASIC == 6 ]; then
          KVM_IMAGE=$OUTPUT_KVM_6ASIC_IMAGE
          RECOVERY_ISO=$onie_recovery_kvm_6asic_image
-    else 
+    else
          KVM_IMAGE=$OUTPUT_KVM_IMAGE
          RECOVERY_ISO=$onie_recovery_image
          NUM_ASIC=1
@@ -104,6 +105,87 @@ generate_device_list()
     done
 }
 
+clean_dir()
+{
+    rm -rf $1
+    exit $2
+}
+
+generate_s2s_installer_image_bluefield()
+{
+    if  [ ! -d installer ] || [ ! -r installer/sharch_body.sh ] ; then
+        echo "Error: Invalid installer script directory: installer"
+        exit 1
+    fi
+
+    if  [ ! -d installer/bluefield ] || [ ! -r installer/bluefield/sonic_to_sonic.sh ] ; then
+        echo "Error: Invalid arch installer directory: installer/bluefield"
+        exit 1
+    fi
+
+    # Copy platform-specific installer config files
+    rm -rf ./installer/bluefield/platforms/
+    mkdir -p ./installer/bluefield/platforms/
+    for PLATFORM in `ls ./device/nvidia-bluefield`; do
+        if [ -f ./device/nvidia-bluefield/$PLATFORM/installer.conf ]; then
+            cp ./device/nvidia-bluefield/$PLATFORM/installer.conf ./installer/bluefield/platforms/$PLATFORM
+        fi
+    done
+
+    echo -n "Building self-extracting install image ."
+    tmp_dir=$(mktemp --directory)
+    tmp_installdir="$tmp_dir/installer"
+    mkdir $tmp_installdir || clean_dir $tmp_dir 1
+
+    # Copy the bluefield specific sonic_to_sonic installer script and rename
+    cp installer/bluefield/sonic_to_sonic.sh $tmp_installdir/install.sh
+
+    # Replace the build time flags inside the script
+    sed -i -e "s/%%IMAGE_VERSION%%/$IMAGE_VERSION/g" \
+           -e "s/%%INSTALLER_PAYLOAD%%/$ONIE_INSTALLER_PAYLOAD/g" \
+           -e "s/%%FILESYSTEM_DOCKERFS%%/$FILESYSTEM_DOCKERFS/g" \
+           -e "s/%%DOCKERFS_DIR%%/$DOCKERFS_DIR/g" \
+           -e "s/%%FILESYSTEM_SQUASHFS%%/$FILESYSTEM_SQUASHFS/g" $tmp_installdir/install.sh || clean_dir $tmp_dir 1
+    chmod 0755 $tmp_installdir/install.sh
+
+    # Copy the payload file
+    cp $ONIE_INSTALLER_PAYLOAD $tmp_installdir || clean_dir  $tmp_dir 1
+
+    # Create machine.conf file
+    echo "machine=nvidia-bluefield" > $tmp_installdir/machine.conf
+    echo "platform=arm64" >> $tmp_installdir/machine.conf
+
+    sharch="$tmp_dir/sharch.tar"
+    tar -C $tmp_dir -cf $sharch installer || {
+        echo "Error: Problems creating $sharch archive"
+        clean_dir $tmp_dir 1
+    }
+
+    [ -f "$sharch" ] || {
+        echo "Error: $sharch not found"
+        clean_dir $tmp_dir 1
+    }
+
+    sha1=$(cat $sharch | sha1sum | awk '{print $1}')
+    echo -n "."
+
+    cp installer/sharch_body.sh $OUTPUT_ONIE_IMAGE || {
+        echo "Error: Problems copying sharch_body.sh"
+        clean_dir 1
+    }
+
+    # Replace variables in the sharch template
+    sed -i -e "s/%%IMAGE_SHA1%%/$sha1/" $OUTPUT_ONIE_IMAGE
+    echo -n "."
+    cat $sharch >> $OUTPUT_ONIE_IMAGE
+    rm -rf $tmp_dir
+    echo " Done."
+
+    echo "Success:  Demo install image is ready in ${OUTPUT_ONIE_IMAGE}:"
+
+    clean_dir $tmp_dir  0
+}
+
 if [ "$IMAGE_TYPE" = "onie" ]; then
     echo "Build ONIE installer"
     mkdir -p `dirname $OUTPUT_ONIE_IMAGE`
@@ -112,7 +194,6 @@ if [ "$IMAGE_TYPE" = "onie" ]; then
     generate_device_list "./installer/$TARGET_PLATFORM/platforms_asic"
 
     generate_onie_installer_image
-
 ## Build a raw partition dump image using the ONIE installer that can be
 ## used to dd' in-lieu of using the onie-nos-installer. Used while migrating
 ## into SONiC from other NOS.
@@ -204,6 +285,32 @@ elif [ "$IMAGE_TYPE" = "aboot" ]; then
         [ -f "$CA_CERT" ] && cp "$CA_CERT" "$TARGET_CA_CERT"
         ./scripts/sign_image.sh -i "$OUTPUT_ABOOT_IMAGE" -k "$SIGNING_KEY" -c "$SIGNING_CERT" -a "$TARGET_CA_CERT"
     fi
+
+elif [[ $IMAGE_TYPE = s2s && $CONFIGURED_PLATFORM == nvidia-bluefield ]]; then
+    # bluefield Doesn't have ONiE Support yet and this *.bin image generated can only be used for SONiC-SONiC installation
+    echo "Build SONiC to SONiC installer for Bluefield"
+    mkdir -p `dirname $OUTPUT_ONIE_IMAGE`
+    sudo rm -f $OUTPUT_ONIE_IMAGE
+    generate_s2s_installer_image_bluefield
+
+elif [[ $IMAGE_TYPE == bfb && $CONFIGURED_PLATFORM == nvidia-bluefield ]]; then
+    # if [[ $CROSS_BUILD_ENVIRON == y ]]; then
+    #     PYTHON_EX=$VENV_BIN/python3 /sonic/installer/bluefield/create_sonic_bfb --sonic_version $IMAGE_VERSION
+    # else
+    #     PYTHON_EX=/usr/bin/python3 /sonic/installer/bluefield/create_sonic_bfb --sonic_version $IMAGE_VERSION
+    # fi
+    # if [ $? -ne 0 ]; then
+    #     echo "create_sonic_bfb script failed with non-zero exit status"
+    #     exit 1
+    # fi
+    # Mote: Using this workaround, until the issue around bfb generation in the slave docker is resolved
+    echo "$ONIE_INSTALLER_PAYLOAD is available. To generate a bfb image, directly run the ./bfb-mk-demo.sh on the build server"
+    exit 0
+
+elif [[ $IMAGE_TYPE == pxe && $CONFIGURED_PLATFORM == nvidia-bluefield ]]; then
+    echo "Not Yet Implemented"
+    exit 1
+
 else
     echo "Error: Non supported image type $IMAGE_TYPE"
     exit 1

device/nvidia-bluefield/arm64-nvda_bf-mbf2h536c/Nvidia-MBF2H536C

@@ -0,0 +1 @@
+../arm64-nvda_bf-mbf2m516a/Nvidia-MBF2H516A

device/nvidia-bluefield/arm64-nvda_bf-mbf2h536c/default_sku

@@ -0,0 +1 @@
+Nvidia-MBF2H536C t1

device/nvidia-bluefield/arm64-nvda_bf-mbf2h536c/installer.conf

@@ -0,0 +1 @@
+../arm64-nvda_bf-mbf2m516a/installer.conf

device/nvidia-bluefield/arm64-nvda_bf-mbf2h536c/platform_asic

@@ -0,0 +1 @@
+../arm64-nvda_bf-mbf2m516a/platform_asic

device/nvidia-bluefield/arm64-nvda_bf-mbf2m516a/Nvidia-MBF2H516A/hwsku.json

@@ -0,0 +1,3 @@
+{
+    "interfaces": {}
+}

device/nvidia-bluefield/arm64-nvda_bf-mbf2m516a/Nvidia-MBF2H516A/port_config.ini

@@ -0,0 +1,16 @@
+##
+## Copyright (c) 2017-2021 NVIDIA CORPORATION & AFFILIATES.
+## Apache-2.0
+##
+## Licensed under the Apache License, Version 2.0 (the "License");
+## you may not use this file except in compliance with the License.
+## You may obtain a copy of the License at
+##
+## http://www.apache.org/licenses/LICENSE-2.0
+##
+## Unless required by applicable law or agreed to in writing, software
+## distributed under the License is distributed on an "AS IS" BASIS,
+## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+## See the License for the specific language governing permissions and
+## limitations under the License.
+##

device/nvidia-bluefield/arm64-nvda_bf-mbf2m516a/default_sku

@@ -0,0 +1 @@
+Nvidia-MBF2H516A t1

device/nvidia-bluefield/arm64-nvda_bf-mbf2m516a/platform_asic

@@ -0,0 +1 @@
+nvidia-bluefield

files/image_config/platform/rc.local

@@ -242,6 +242,16 @@ if [ -f $FIRST_BOOT_FILE ]; then
         platform=$aboot_platform
     elif [ -n "$onie_platform" ]; then
         platform=$onie_platform
+    elif [ -n "$bf_platform" ]; then
+        platform=$bf_platform
+    elif [ "$bf_machine" = "nvidia-bluefield" ]; then
+        platform=$(dmidecode -t 4 | grep "Part Number" | awk '{split($NF,a,"-"); print tolower(a[1])}')
+        if [ -z "$platform" ]; then
+            echo "Platform name for the nvidia-bluefield machine could not be inferred" 
+            firsttime_exit
+        fi
+        platform=arm64-nvda_bf-$platform
+        echo "bf_platform=${platform}" >> /host/machine.conf
     else
         echo "Unknown SONiC platform"
         firsttime_exit