From d2a57f6231ab11e76372c6aa2b219a89c84b6e71 Mon Sep 17 00:00:00 2001 From: Ryan Houdek Date: Thu, 9 Jun 2022 19:44:01 -0700 Subject: [PATCH 1/3] Scripts: Add CI rootfs fetch script This will allow our CI system to automatically pull their rootfs. --- Scripts/CI_FetchRootFS.py | 175 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 175 insertions(+) create mode 100755 Scripts/CI_FetchRootFS.py diff --git a/Scripts/CI_FetchRootFS.py b/Scripts/CI_FetchRootFS.py new file mode 100755 index 0000000000..9d1d9cbdc9 --- /dev/null +++ b/Scripts/CI_FetchRootFS.py @@ -0,0 +1,175 @@ +#!/usr/bin/python3 +import xxhash +import hashlib +import sys +import os +import shutil +import subprocess + +def GetDistroInfo(): + DistroName = "Unknown" + DistroVersion = "Unknown" + + with open("/etc/lsb-release", 'r') as f: + while True: + Line = f.readline() + if not Line: + break + Split = Line.split("=") + if Split[0] == "DISTRIB_ID": + DistroName = Split[1].lower().rstrip() + if Split[0] == "DISTRIB_RELEASE": + DistroVersion = Split[1].rstrip() + + return [DistroName, DistroVersion] + +def FindBestImageFit(Distro, links_file): + CurrentFitSize = 0 + BestFitDistro = None + BestFitDistroVersion = None + BestFitReadableName = None + BestFitImagePath = None + BestFitHash = None + + with open(links_file, 'r') as f: + while True: + # Order: + # Distro Name + # Distro Version + # User readable name + # File Path + # Hash + + DistroName = f.readline().strip() + if not DistroName: + break + + DistroVersion = f.readline().strip() + DistroReadableName = f.readline().strip() + DistroImagePath = f.readline().strip() + DistroHash = f.readline().strip() + + FitRate = 0 + if (DistroName == Distro[0] or + DistroName == None): + FitRate += 1 + + if (DistroVersion == Distro[1] or + DistroVersion == None): + FitRate += 1 + + if FitRate > CurrentFitSize: + CurrentFitSize = FitRate + BestFitDistro = DistroName + BestFitDistroVersion = DistroVersion + BestFitReadableName = DistroReadableName + BestFitImagePath = DistroImagePath + BestFitHash = DistroHash + + return [BestFitDistro, BestFitDistroVersion, BestFitReadableName, BestFitImagePath, int(BestFitHash, 16)] + + +def HashFile(file): + # 32MB buffer size + BUFFER_SIZE = 32 * 1024 * 1024 + + x = xxhash.xxh3_64(seed=0) + b = bytearray(BUFFER_SIZE) + mv = memoryview(b) + + with open(file, 'rb') as f: + while n := f.readinto(mv): + x.update(mv[:n]) + + return int.from_bytes(x.digest(), "big") + +def CheckFilesystemForFS(RootFSMountPath, RootFSPath, DistroFit): + # Check if rootfs mount path exists + if (not os.path.exists(RootFSMountPath) or + not os.path.isdir(RootFSMountPath)): + print("RootFS mount path is wrong") + return False + + # Check if rootfs path exists + if (not os.path.exists(RootFSPath) or + not os.path.isdir(RootFSPath)): + # Create this directory + os.makedirs(RootFSPath) + + # Check if rootfs path exists + if not os.path.isdir(RootFSPath): + print("RootFS path is not a directory") + return False + + # Check rootfs folder for image, copy and extract as necessary + MountRootFSImagePath = RootFSMountPath + DistroFit[3] + RootFSImagePath = RootFSPath + "/" + os.path.basename(DistroFit[3]) + NeedsExtraction = False + + if not os.path.exists(MountRootFSImagePath): + print("Image {} doesn't exist".format(MountRootFSImagePath)) + return False + + if not os.path.exists(RootFSImagePath): + # Copy over + print("RootFS image doesn't exist. Copying") + shutil.copyfile(MountRootFSImagePath, RootFSImagePath) + NeedsExtraction = True + + # Now hash the image + RootFSHash = HashFile(RootFSImagePath) + if RootFSHash != DistroFit[4]: + print("Hash {} did not match {}, copying new image".format(hex(RootFSHash), hex(DistroFit[4]))) + shutil.copyfile(MountRootFSImagePath, RootFSImagePath) + NeedsExtraction = True + + # Check if the image needs to be extracted + if not os.path.exists(RootFSPath + "/usr"): + NeedsExtraction = True + + if NeedsExtraction: + print("Extracting rootfs") + CmdResult = subprocess.call(["unsquashfs", "-f", "-d", RootFSPath, RootFSImagePath]) + if CmdResult != 0: + print("Couldn't extract squashfs") + return False + + if not os.path.exists(RootFSPath + "/usr"): + print("Couldn't extract squashfs") + return False + + print("RootFS successfully checked and extracted") + + return True + +def main(): + if sys.version_info[0] < 3: + logging.critical ("Python 3 or a more recent version is required.") + + FEX_ROOTFS_MOUNT = os.getenv("FEX_ROOTFS_MOUNT") + FEX_ROOTFS_PATH = os.getenv("FEX_ROOTFS_PATH") + + if FEX_ROOTFS_MOUNT == None: + print("Need FEX_ROOTFS_MOUNT set") + sys.exit(1) + + if FEX_ROOTFS_PATH == None: + print("Need FEX_ROOTFS_PATH set") + sys.exit(1) + + if shutil.which("unsquashfs") is None: + print("CI system didn't have unsquashfs installed") + sys.exit(1) + + Distro = GetDistroInfo() + DistroFit = FindBestImageFit(Distro, FEX_ROOTFS_MOUNT + "/RootFS_links.txt") + + if CheckFilesystemForFS(FEX_ROOTFS_MOUNT, FEX_ROOTFS_PATH, DistroFit) == False: + print("Couldn't load filesystem rootfs") + sys.exit(1) + + return 0 + +if __name__ == "__main__": +# execute only if run as a script + sys.exit(main()) From a3d8fe236205ef489292ca8ce0be6957afccd7c7 Mon Sep 17 00:00:00 2001 From: Ryan Houdek Date: Thu, 9 Jun 2022 19:44:35 -0700 Subject: [PATCH 2/3] github: Allow CI to fetch its own rootfs Just need to set some environment variables and execute our new script --- .github/workflows/ccpp.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/.github/workflows/ccpp.yml b/.github/workflows/ccpp.yml index f427e5ed3b..25f3dd0b6e 100644 --- a/.github/workflows/ccpp.yml +++ b/.github/workflows/ccpp.yml @@ -29,6 +29,20 @@ jobs: - name: Set runner label run: echo "runner_label=${{ matrix.arch[1] }}" >> $GITHUB_ENV + - name: Set rootfs paths + run: | + echo "FEX_ROOTFS_MOUNT=/mnt/AutoNFS/rootfs/" >> $GITHUB_ENV + echo "FEX_ROOTFS_PATH=$HOME/Rootfs/" >> $GITHUB_ENV + echo "FEX_ROOTFS=$HOME/Rootfs/" >> $GITHUB_ENV + echo "ROOTFS=$HOME/Rootfs/" >> $GITHUB_ENV + + - name: Update RootFS cache + # Use a bash shell so we can use the same syntax for environment variable + # access regardless of the host operating system + shell: bash + working-directory: ${{runner.workspace}}/build + run: $GITHUB_WORKSPACE/Scripts/CI_FetchRootFS.py + - name : submodule checkout # Need to update submodules run: | From 81dfc2170065cb68f4ecf5e8ff6b6d947a4690bb Mon Sep 17 00:00:00 2001 From: Ryan Houdek Date: Thu, 9 Jun 2022 20:03:08 -0700 Subject: [PATCH 3/3] unittests/gvisor-test: Disable semaphore test for now New rootfs images cause different data to be returned in permissions than this test was expecting. Disable this while we are upgrading CI. --- unittests/gvisor-tests/Disabled_Tests | 3 +++ unittests/gvisor-tests/Known_Failures | 3 +++ 2 files changed, 6 insertions(+) diff --git a/unittests/gvisor-tests/Disabled_Tests b/unittests/gvisor-tests/Disabled_Tests index adae043a0b..96a571862a 100644 --- a/unittests/gvisor-tests/Disabled_Tests +++ b/unittests/gvisor-tests/Disabled_Tests @@ -184,3 +184,6 @@ proc_net_udp_test # This test is broken since kernel commit: 36e2c7421f02 # Upstream gvisor has the offending tests removed inotify_test + +# Needs testing +semaphore_test diff --git a/unittests/gvisor-tests/Known_Failures b/unittests/gvisor-tests/Known_Failures index 6a622a0f30..f6927d72c3 100644 --- a/unittests/gvisor-tests/Known_Failures +++ b/unittests/gvisor-tests/Known_Failures @@ -210,3 +210,6 @@ proc_net_udp_test # This test is broken since kernel commit: 36e2c7421f02 # Upstream gvisor has the offending tests removed inotify_test + +# Needs testing +semaphore_test