fix(kernel_crawler): fixed bottlerocket crawler.

kernel_crawler/bottlerocket.py

@@ -50,7 +50,7 @@ def extract_flavor(self, flavorconfig_path):
         return re.match(r"^config-bottlerocket-(.*)", flavorconfig_file).group(1)
 
     def extract_kver(self, kverspec_file):
-        return re.match(r"^kernel-(.*).spec", kverspec_file).group(1)
+        return re.match(r"^kernel-(.*).spec$", kverspec_file).group(1)
 
     def set_kernel_config(self, baseconfig, key, value):
         for i, line in enumerate(baseconfig):
@@ -90,52 +90,74 @@ def get_package_tree(self, version=''):
             self.checkout_version(v)
 
             # Find supported kernels dynamically
-            supported_kernel_specs = self.match_file("kernel-.*.spec", False)
+            supported_kernel_specs = self.match_file("kernel-.*.spec", True)
             for kverspec_file in supported_kernel_specs:
-                kver = self.extract_kver(kverspec_file)
+                name = os.path.basename(kverspec_file)
+                wd = os.path.dirname(kverspec_file)
+                kver = self.extract_kver(name)
 
                 # same meaning as the output of "uname -r"
                 kernel_release = self.extract_value(kverspec_file, "Version", ":")
                 if kernel_release is None:
                     continue
 
                 # Load base config
-                baseconfig = self.fetch_base_config(kverspec_file)
-                if baseconfig is None:
+                vanillaconfig = self.fetch_base_config(kverspec_file)
+                if vanillaconfig is None:
                     continue
 
                 # Load common config
-                commonconfig_file = self.search_file("config-bottlerocket")
-                if commonconfig_file is None:
+                specific_config_file = self.search_file("config-bottlerocket", wd)
+                if specific_config_file is None:
                     continue
-                with open(commonconfig_file, 'r') as fd:
-                    commonconfig = fd.readlines()
+
+                with open(specific_config_file, 'r') as fd:
+                    specific_config = fd.readlines()
 
                 # Find supported flavors dynamically
-                supported_flavors = self.match_file("config-bottlerocket-.*")
-                for flavorconfig_file in supported_flavors:
-                    flavor = self.extract_flavor(flavorconfig_file)
-
-                    # Load flavor specific config
-                    with open(flavorconfig_file, 'r') as fd:
-                        flavorconfig = fd.readlines()
-
-                    # Merge flavor and common config
-                    flavorconfig += commonconfig
-
-                    # Finally, patch baseconfig with flavor config
-                    finalconfig = self.patch_config(baseconfig, flavorconfig)
-                    defconfig_base64 = base64.b64encode(b''.join(finalconfig)).decode()
-
-                    kernel_version = "1_" + v + "-" + flavor
-
-                    # Unique key
-                    kernel_configs[v + "_" + kver + "-" + flavor] = {
-                        self.KERNEL_VERSION: kernel_version,
-                        self.KERNEL_RELEASE: kernel_release,
-                        self.DISTRO_TARGET: "bottlerocket",
-                        self.BASE_64_CONFIG_DATA: defconfig_base64,
-                    }
+                supported_flavors = self.match_file("config-bottlerocket-.*", True, wd)
+                if supported_flavors:
+                    for flavorconfig_file in supported_flavors:
+                        flavor = self.extract_flavor(flavorconfig_file)
+
+                        # Load flavor specific config
+                        with open(flavorconfig_file, 'r') as fd:
+                            flavorconfig = fd.readlines()
+
+                        # Merge flavor and common config
+                        flavorconfig += specific_config
+
+                        # Finally, patch baseconfig with flavor config
+                        finalconfig = self.patch_config(vanillaconfig, flavorconfig)
+                        defconfig_base64 = base64.b64encode(b''.join(finalconfig)).decode()
+
+                        kernel_version = "1_" + v + "-" + flavor
+
+                        # Unique key
+                        kernel_configs[v + "_" + kver + "-" + flavor] = {
+                            self.KERNEL_VERSION: kernel_version,
+                            self.KERNEL_RELEASE: kernel_release,
+                            self.DISTRO_TARGET: "bottlerocket",
+                            self.BASE_64_CONFIG_DATA: defconfig_base64,
+                        }
+                else:
+                    # NOTE: to keep backward compatibility with existing drivers
+                    # and driver loader logic, push these kernels for each flavor
+                    # even if the config is the same among all of them.
+                    # We will build 3x the drivers but we will be backward compatible.
+                    for flavor in ['aws','metal','vmware']:
+                        finalconfig = self.patch_config(vanillaconfig, specific_config)
+                        defconfig_base64 = base64.b64encode(b''.join(finalconfig)).decode()
+
+                        kernel_version = "1_" + v + "-" + flavor
+
+                        # Unique key
+                        kernel_configs[v + "_" + kver + "-" + flavor] = {
+                            self.KERNEL_VERSION: kernel_version,
+                            self.KERNEL_RELEASE: kernel_release,
+                            self.DISTRO_TARGET: "bottlerocket",
+                            self.BASE_64_CONFIG_DATA: defconfig_base64,
+                        }
 
             bar.update(1)
             bar.render_finish()

kernel_crawler/git.py

@@ -67,7 +67,7 @@ def cleanup_repo(self):
         shutil.rmtree(self.repo.workdir, True)
 
     def getVersions(self, last_n=0):
-        re_tags = re.compile('^refs/tags/v(?P<major>0|[1-9]\d*)\.(?P<minor>0|[1-9]\d*)\.(?P<patch>0|[1-9]\d*)$')
+        re_tags = re.compile(r'^refs/tags/v(?P<major>0|[1-9]\d*)\.(?P<minor>0|[1-9]\d*)\.(?P<patch>0|[1-9]\d*)$')
 
         all_versions = [os.path.basename(v).strip('v') for v in self.repo.references if re_tags.match(v)]
         all_versions.sort(key=SemVersion)
@@ -97,16 +97,20 @@ def checkout_hash(self, commithash):
 
         return self.checkout_version(commithash)
 
-    def search_file(self, file_name):
-        for dirpath, dirnames, files in os.walk(self.repo.workdir):
+    def search_file(self, file_name, wd=''):
+        if wd == '':
+            wd = self.repo.workdir
+        for dirpath, dirnames, files in os.walk(wd):
             for name in files:
                 if name == file_name:
                     return os.path.join(dirpath, name)
         return None
 
-    def match_file(self, pattern, fullpath=True):
+    def match_file(self, pattern, fullpath=True, wd=''):
         matches = []
-        for dirpath, dirnames, files in os.walk(self.repo.workdir):
+        if wd == '':
+            wd = self.repo.workdir
+        for dirpath, dirnames, files in os.walk(wd):
             for name in files:
                 if re.search(r'^'+pattern, name):
                     if fullpath:
@@ -116,8 +120,10 @@ def match_file(self, pattern, fullpath=True):
         return matches
 
     def extract_value(self, file_name, key, sep):
-        # here kernel release is the same as the one given by "uname -r"
-        full_path = self.search_file(file_name)
+        if os.path.isabs(file_name):
+            full_path = file_name
+        else:
+            full_path = self.search_file(file_name)
         for line in open(full_path):
             stripped_line = line.lstrip()
             if re.search(r'^'+key + sep, stripped_line):