Add Kernel config diff script between different kernel versions (soni…

…c-net#375)

Signed-off-by: Vivek Reddy <vkarri@nvidia.com>
Co-authored-by: Saikrishna Arcot <sarcot@microsoft.com>

.artifactignore

@@ -1,2 +1,3 @@
 **/*
 !*.deb
+!kconfig-diff-*.rst

.azure-pipelines/build-template.yml

@@ -32,6 +32,9 @@ parameters:
 - name: artifact_name
   type: string
 
+- name: linux_deb_pattern
+  type: string
+
 jobs:
 - job:
   pool: ${{ parameters.pool }}
@@ -44,14 +47,37 @@ jobs:
   steps:
   - checkout: self
     clean: true
-    displayName: 'Checkout code'
+    displayName: 'Checkout code'  
+  - task: DownloadPipelineArtifact@2
+    inputs:
+      source: specific
+      project: build
+      pipelineId: 426041
+      artifact: ${{ parameters.artifact_name }}
+      runVersion: 'specific'
+      itemPattern: ${{ parameters.linux_deb_pattern }}
+      targetPath: $(Agent.TempDirectory)/
+    displayName: "Download the linux-image artifact from the last build with a different kernel version"
   - script: |
       git config --global user.email "lguohan@build.com"
       git config --global user.name "Guohan Lu"
       export kernel_procure_method=build
       cat /proc/cpuinfo
       CONFIGURED_ARCH=${{ parameters.arch }} CONFIGURED_PLATFORM=${{ parameters.platform }} make
     displayName: "Compile sonic kernel"
+  - script: |
+      dpkg-deb -x $(Agent.TempDirectory)/${{ parameters.linux_deb_pattern }} $(Agent.TempDirectory)/old
+      dpkg-deb -x $(System.DefaultWorkingDirectory)/${{ parameters.linux_deb_pattern }} $(Agent.TempDirectory)/new
+      pip3 install tabulate
+      python3 $(System.DefaultWorkingDirectory)/.azure-pipelines/kcfg-diff.py \
+              --buildid $(Build.BuildId) \
+              --ref_buildid 426041 \
+              --arch ${{ parameters.arch }} \
+              --old_kcfg $(Agent.TempDirectory)/old/boot/ \
+              --new_kcfg $(Agent.TempDirectory)/new/boot/ \
+              --output $(System.DefaultWorkingDirectory)/kconfig-diff-${{ parameters.platform }}-${{ parameters.arch }}.rst
+      cat $(System.DefaultWorkingDirectory)/kconfig-diff-${{ parameters.platform }}-${{ parameters.arch }}.rst
+    displayName: "Compute the kconfig diff"
   - publish: $(System.DefaultWorkingDirectory)/
     artifact: ${{ parameters.artifact_name }}
     displayName: "Archive sonic kernel debian packages"

.azure-pipelines/kcfg-diff.py

@@ -0,0 +1,160 @@
+import argparse
+import os
+import glob
+import sys
+import re
+from tabulate import tabulate
+
+COLUMN_WIDTH_MAX = 40
+MODIF_COLUMN_WIDTH_MAX = 80
+
+RST_TEMPLATE = '''\
+SONiC-linux-kernel KConfig Difference:
+==========
+
+Reference Kernel
+------------
+- Version: {}
+- BuildID: https://dev.azure.com/mssonic/build/_build/results?buildId={}&view=results
+
+Latest Kernel
+------------
+- Version: {}
+- BuildID: https://dev.azure.com/mssonic/build/_build/results?buildId={}&view=results
+
+
+Additions & Deletions
+------------
+{}
+
+Modifications
+------------
+{}
+'''
+
+def read_data(file1):
+    data = []
+    try:
+        with open(file1, 'r') as f1:
+            data = f1.readlines()
+    except Exception as e:
+        print("ABORT: Reading failed from {}, {}".format(file1, str(e)))
+        sys.exit(1)
+
+    data = filter(lambda line: not(line.startswith('#')), data)
+    ret = dict()
+    for line in data:
+        tokens = line.split('=')
+        if len(tokens) == 2:
+            key, val = tokens[0].strip(), tokens[-1].strip()
+            ret[key] = val
+    return ret
+
+def write_data(fname, data):
+    try:
+        with open(fname, 'w') as f:
+            f.write(data)
+    except Exception as e:
+        print("ABORT: Writing to the file {} failed {}".format(fname, str(e)))
+        sys.exit(1)
+
+def generate_diff(file1, file2):
+    data_f1 = read_data(file1)
+    data_f2 = read_data(file2)
+
+    additions = []
+    modifications = []
+    deletions = []
+
+    for key_old, val_old in data_f1.items():
+        val_new = data_f2.get(key_old, None)
+        if not val_new:
+            deletions.append("{}={}".format(key_old, val_old))
+        elif val_old != val_new:
+            modifications.append("{}={}->{}".format(key_old, val_old, val_new))
+        if val_new:
+            del data_f2[key_old]
+
+    for key, val in data_f2.items():
+        additions.append("{}={}".format(key, val))
+    return additions, modifications, deletions
+
+def restrict_column_width(lis, width):
+    for i in range(0, len(lis)):
+        curr_width = len(lis[i])
+        new_val = ''
+        num_newlines = int(curr_width/width)
+        for j in range(0, num_newlines+1):
+            if (j+1)*width < curr_width:
+                new_val += lis[i][j*width:(j+1)*width]
+                new_val += "\n"
+            else:
+                new_val += lis[i][j*width:]
+        lis[i] = new_val
+
+def format_diff_table(additions, modifications, deletions):
+    max_len = max(len(additions), len(deletions))
+    additions += [''] * (max_len - len(additions))
+    deletions += [''] * (max_len - len(deletions))
+
+    restrict_column_width(additions, COLUMN_WIDTH_MAX)
+    restrict_column_width(deletions, COLUMN_WIDTH_MAX)
+    restrict_column_width(modifications, MODIF_COLUMN_WIDTH_MAX)
+
+    table_data = list(zip(additions, deletions))
+    headers = ["ADDITIONS", "DELETIONS"]
+
+    add_del = tabulate(table_data, headers=headers, tablefmt="grid")
+    mod = tabulate(list(zip(modifications)), headers=["MODIFICATIONS"], tablefmt="grid")
+    return add_del, mod
+
+def parse_kver(loc):
+    files = glob.glob(os.path.join(loc, "config-*"))
+    if len(files) > 1:
+        print("WARNING: Multiple config- files present under {}, {}".format(loc, files))
+    result = re.search(r"config-(.*)", os.path.basename(files[-1]))
+    return result.group(1)
+
+def create_parser():
+    # Create argument parser
+    parser = argparse.ArgumentParser()
+
+    # Optional arguments
+    parser.add_argument("--arch", type=str, required=True)
+    parser.add_argument("--old_kcfg", type=str, required=True)
+    parser.add_argument("--new_kcfg", type=str, required=True)
+    parser.add_argument("--output", type=str, required=True)
+    parser.add_argument("--ref_buildid", type=str, required=True)
+    parser.add_argument("--buildid", type=str, required=True)
+    return parser
+
+def verify_args(args):
+    if not glob.glob(os.path.join(args.old_kcfg, "config-*")):
+        print("ABORT: config file missing under {}".format(args.old_kcfg))
+        return False
+
+    if not glob.glob(os.path.join(args.new_kcfg, "config-*")):
+        print("ABORT: config file missing under {}".format(args.new_kcfg))
+        return False
+
+    if not os.path.exists(os.path.dirname(args.output)):
+        print("ABORT: Output Folder {} doesn't exist".format(args.output))
+        return False
+    return True
+
+if __name__ == "__main__":
+    parser = create_parser()
+    args = parser.parse_args()
+    if not verify_args(args):
+        sys.exit(1)
+
+    ver_old = parse_kver(args.old_kcfg)
+    ver_new = parse_kver(args.new_kcfg)
+    f_old = os.path.join(args.old_kcfg, 'config-{}'.format(ver_old))
+    f_new = os.path.join(args.new_kcfg, 'config-{}'.format(ver_new))
+
+    additions, modifications, deletions = generate_diff(f_old, f_new)
+    add_del, mod = format_diff_table(additions, modifications, deletions)
+
+    diff = RST_TEMPLATE.format(ver_old, args.ref_buildid, ver_new, args.buildid, add_del, mod)
+    write_data(args.output, diff)

README.md

@@ -50,3 +50,9 @@ If the files patch/kconfig-exclusions and patch/kconfig-inclusions exist, they w
 Also, the final kernel configuration will be checked to verify that:
 - all options asked to be excluded are effectively not present in the kernel,
 - and all options asked to be included are effectively present in the kernel, using the exact type (module or built-in) or string or number.
+
+## Kernel Configuration Difference during Upgrades
+
+During Kernel Upgrades, the maintainer must update the configuration diff in the wiki of this repo. This acts as a guide for keeping track of configuration changes between Kernel upgrades. Applies for minor version upgrades as well
+
+The diff is saved as kconfig-diff-{platform}-{arch}.rst under the artifacts of Azure.sonic-linux-kernel job runs.

azure-pipelines.yml

@@ -17,20 +17,23 @@ stages:
       arch: amd64
       sonic_slave: sonic-slave-bookworm
       artifact_name: sonic-linux-kernel
+      linux_deb_pattern: linux-image-*-unsigned_*deb
 
   - template: .azure-pipelines/build-template.yml
     parameters:
       arch: arm64
       pool: sonicbld-arm64
       sonic_slave: sonic-slave-bookworm-arm64
       artifact_name: sonic-linux-kernel.arm64
+      linux_deb_pattern: linux-image-*-unsigned_*deb
 
   - template: .azure-pipelines/build-template.yml
     parameters:
       arch: armhf
       pool: sonicbld-armhf
       sonic_slave: sonic-slave-bookworm-armhf
       artifact_name: sonic-linux-kernel.armhf
+      linux_deb_pattern: linux-image-*-armmp_*deb
 
   - template: .azure-pipelines/build-template.yml
     parameters:
@@ -39,4 +42,5 @@ stages:
       pool: sonicbld-arm64
       sonic_slave: sonic-slave-bookworm-arm64
       artifact_name: sonic-linux-kernel.pensando.arm64
+      linux_deb_pattern: linux-image-*-unsigned_*deb