Skip to content

Commit

Permalink
[Mellanox] Enhance the processing of Kconfig in the hw-mgmt integrati…
Browse files Browse the repository at this point in the history 
…on (#16752) (#17009)
  • Loading branch information
@mssonicbld
mssonicbld authored Oct 25, 2023
1 parent 7982c3a commit da90d56
Show file tree
Hide file tree
Showing 20 changed files with 752 additions and 331 deletions.
62 changes: 54 additions & 8 deletions platform/mellanox/integration-scripts.mk
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,6 @@ TEMP_HW_MGMT_DIR = /tmp/hw_mgmt
PTCH_DIR = $(TEMP_HW_MGMT_DIR)/patch_dir/
NON_UP_PTCH_DIR = $(TEMP_HW_MGMT_DIR)/non_up_patch_dir/
PTCH_LIST = $(TEMP_HW_MGMT_DIR)/series
KCFG_LIST = $(TEMP_HW_MGMT_DIR)/kconfig
HWMGMT_NONUP_LIST = $(BUILD_WORKDIR)/$($(MLNX_HW_MANAGEMENT)_SRC_PATH)/hwmgmt_nonup_patches
HWMGMT_USER_OUTFILE = $(BUILD_WORKDIR)/integrate-mlnx-hw-mgmt_user.out
SDK_USER_OUTFILE = $(BUILD_WORKDIR)/integrate-mlnx-sdk_user.out
Expand All @@ -36,11 +35,33 @@ SLK_COM_MSG := $(shell mktemp -t slk_commit_msg_file_XXXXX.log)
SB_HEAD = $(shell git rev-parse --short HEAD)
SLK_HEAD = $(shell cd src/sonic-linux-kernel; git rev-parse --short HEAD)

# kconfig related variables
KCFG_BASE_TMPDIR = $(TEMP_HW_MGMT_DIR)/linux_kconfig/
KCFG_BASE = $(KCFG_BASE_TMPDIR)/amd64.config
KCFG_LIST = $(TEMP_HW_MGMT_DIR)/kconfig_amd64
KCFG_DOWN_LIST = $(TEMP_HW_MGMT_DIR)/kconfig_downstream_amd64
KCFG_BASE_ARM = $(KCFG_BASE_TMPDIR)/arm64.config
KCFG_LIST_ARM = $(TEMP_HW_MGMT_DIR)/kconfig_arm64
KCFG_DOWN_LIST_ARM = $(TEMP_HW_MGMT_DIR)/kconfig_downstream_arm64


integrate-mlnx-hw-mgmt:
$(FLUSH_LOG)
rm -rf $(TEMP_HW_MGMT_DIR) $(TMPFILE_OUT)
mkdir -p $(PTCH_DIR) $(NON_UP_PTCH_DIR)
touch $(PTCH_LIST) $(KCFG_LIST)
mkdir -p $(PTCH_DIR) $(NON_UP_PTCH_DIR) $(KCFG_BASE_TMPDIR)
touch $(PTCH_LIST) $(KCFG_LIST) $(KCFG_DOWN_LIST) $(KCFG_LIST_ARM) $(KCFG_DOWN_LIST_ARM)

# Fetch the vanilla .config files
pushd $(KCFG_BASE_TMPDIR) $(LOG_SIMPLE)
rm -rf linux/; mkdir linux
# Note: gregkh is the stable linux mirror
git clone --depth 1 --branch v$(KERNEL_VERSION) https://github.com/gregkh/linux.git linux $(LOG_SIMPLE)

pushd linux
rm -rf .config; make ARCH=x86_64 defconfig; cp -f .config $(KCFG_BASE) $(LOG_SIMPLE)
rm -rf .config; make ARCH=arm64 defconfig; cp -f .config $(KCFG_BASE_ARM) $(LOG_SIMPLE)
popd
popd $(LOG_SIMPLE)

# clean up existing untracked files
pushd $(BUILD_WORKDIR); git clean -f -- platform/mellanox/
Expand Down Expand Up @@ -69,17 +90,32 @@ endif

# Pre-processing before runing hw_mgmt script
integration-scripts/hwmgmt_kernel_patches.py pre \
--config_inclusion $(KCFG_LIST) \
--config_base_amd $(KCFG_BASE) \
--config_base_arm $(KCFG_BASE_ARM) \
--config_inc_amd $(KCFG_LIST) \
--config_inc_arm $(KCFG_LIST_ARM) \
--build_root $(BUILD_WORKDIR) \
--kernel_version $(KERNEL_VERSION) \
--hw_mgmt_ver ${MLNX_HW_MANAGEMENT_VERSION} $(LOG_SIMPLE)
--hw_mgmt_ver ${MLNX_HW_MANAGEMENT_VERSION} $(LOG_SIMPLE)

$(BUILD_WORKDIR)/$($(MLNX_HW_MANAGEMENT)_SRC_PATH)/hw-mgmt/recipes-kernel/linux/deploy_kernel_patches.py \
--dst_accepted_folder $(PTCH_DIR) \
--dst_candidate_folder $(NON_UP_PTCH_DIR) \
--series_file $(PTCH_LIST) \
--config_file $(KCFG_LIST_ARM) \
--config_file_downstream $(KCFG_DOWN_LIST_ARM) \
--kernel_version $(KERNEL_VERSION) \
--arch arm64 \
--os_type sonic $(LOG_SIMPLE)

$(BUILD_WORKDIR)/$($(MLNX_HW_MANAGEMENT)_SRC_PATH)/hw-mgmt/recipes-kernel/linux/deploy_kernel_patches.py \
--dst_accepted_folder $(PTCH_DIR) \
--dst_candidate_folder $(NON_UP_PTCH_DIR) \
--series_file $(PTCH_LIST) \
--config_file $(KCFG_LIST) \
--config_file_downstream $(KCFG_DOWN_LIST) \
--kernel_version $(KERNEL_VERSION) \
--arch amd64 \
--os_type sonic $(LOG_SIMPLE)

# Post-processing
Expand All @@ -88,7 +124,12 @@ endif
--non_up_patches $(NON_UP_PTCH_DIR) \
--kernel_version $(KERNEL_VERSION) \
--hw_mgmt_ver ${MLNX_HW_MANAGEMENT_VERSION} \
--config_inclusion $(KCFG_LIST) \
--config_base_amd $(KCFG_BASE) \
--config_base_arm $(KCFG_BASE_ARM) \
--config_inc_amd $(KCFG_LIST) \
--config_inc_arm $(KCFG_LIST_ARM) \
--config_inc_down_amd $(KCFG_DOWN_LIST) \
--config_inc_down_arm $(KCFG_DOWN_LIST_ARM) \
--series $(PTCH_LIST) \
--current_non_up_patches $(HWMGMT_NONUP_LIST) \
--build_root $(BUILD_WORKDIR) \
Expand Down Expand Up @@ -121,8 +162,11 @@ endif
git add -- $(PLATFORM_PATH)/non-upstream-patches/
git add -- $(PLATFORM_PATH)/hw-management.mk

echo -en '\n###-> Non Upstream series.patch changes <-###\n' >> ${HWMGMT_USER_OUTFILE}
git diff --no-color --staged -- $(PLATFORM_PATH)/non-upstream-patches/series.patch >> ${HWMGMT_USER_OUTFILE}
echo -en '\n###-> Non Upstream external-changes.patch changes <-###\n' >> ${HWMGMT_USER_OUTFILE}
git diff --no-color --staged -- $(PLATFORM_PATH)/non-upstream-patches/external-changes.patch >> ${HWMGMT_USER_OUTFILE}

echo -en '\n###-> Non Upstream kconfig-inclusions.patch changes <-###\n' >> ${HWMGMT_USER_OUTFILE}
git diff --no-color --staged -- $(PLATFORM_PATH)/non-upstream-patches/kconfig-inclusions.patch >> ${HWMGMT_USER_OUTFILE}

echo -en '\n###-> Non Upstream patch list file <-###\n' >> ${HWMGMT_USER_OUTFILE}
git diff --no-color --staged -- $($(MLNX_HW_MANAGEMENT)_SRC_PATH)/hwmgmt_nonup_patches >> ${HWMGMT_USER_OUTFILE}
Expand All @@ -142,6 +186,8 @@ endif

popd $(LOG_SIMPLE)

rm -rf $(TEMP_HW_MGMT_DIR)

integrate-mlnx-sdk:
$(FLUSH_LOG)
rm -rf $(MLNX_SDK_VERSION).zip sx_kernel-$(MLNX_SDK_VERSION)-$(MLNX_SDK_ISSU_VERSION).tar.gz
Expand Down
102 changes: 28 additions & 74 deletions platform/mellanox/integration-scripts/helper.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,12 @@

import os
import glob
import re
from collections import OrderedDict

MARK_ID = "###->"
MLNX_KFG_MARKER = "mellanox"
MLNX_KFG_MARKER = "mellanox_amd64"
MLNX_NOARCH_MARKER = "mellanox_common"
MLNX_ARM_KFG_SECTION = "mellanox-arm64"
SDK_MARKER = "mellanox_sdk"
HW_MGMT_MARKER = "mellanox_hw_mgmt"
SLK_PATCH_LOC = "src/sonic-linux-kernel/patch/"
Expand All @@ -29,7 +31,7 @@
SLK_SERIES = SLK_PATCH_LOC + "series"
NON_UP_PATCH_DIR = "platform/mellanox/non-upstream-patches/"
NON_UP_PATCH_LOC = NON_UP_PATCH_DIR + "patches"
NON_UP_PATCH_DIFF = NON_UP_PATCH_DIR + "series.patch"
NON_UP_DIFF = NON_UP_PATCH_DIR + "external-changes.patch"
KCFG_HDR_RE = "\[(.*)\]"
KERNEL_BACKPORTS = "kernel_backports"
# kconfig_inclusion headers to consider
Expand Down Expand Up @@ -100,83 +102,35 @@ def find_marker_indices(lines: list, marker=None) -> tuple:
return (i_start, i_end)

@staticmethod
def read_kconfig_inclusion(path, marker=MLNX_KFG_MARKER):
lines = FileHandler.read_strip(path)
if not marker:
return lines
i_start, i_end = FileHandler.find_marker_indices(lines, marker)

if i_start < 0 or i_end >= len(lines):
print("-> WARNING No Marker Found")
return []

return lines[i_start+1:i_end]

@staticmethod
def write_lines_marker(path, writable_opts: list, marker=None):
# if marker is none, just write the opts into the file,
# otherwise write the data only b/w the marker
curr_data = FileHandler.read_raw(path)
i_start, i_end = FileHandler.find_marker_indices(curr_data, marker)
newline_writ_opts = [opt + "\n" for opt in writable_opts]
if i_start < 0 or i_end >= len(curr_data):
print("-> WARNING No Marker Found, writing data at the end of file")
curr_data.extend(["\n"])
curr_data.extend(newline_writ_opts)
else:
curr_data = curr_data[0:i_start+1] + newline_writ_opts + curr_data[i_end:]

print("-> INFO Written the following opts: \n{}".format("".join(FileHandler.read_raw(path))))
FileHandler.write_lines(path, curr_data, True)

@staticmethod
def read_kconfig_parser(path) -> dict:
# kconfig_inclusion output formatted to {"no_parent", "common":[,], "amd64": [,], "arm64": [,]}
def read_kconfig(path) -> dict:
# read the .config file generated during kernel compilation
lines = FileHandler.read_strip_minimal(path)
ret = dict({"no_parent":[]})
curr_hdr = ""
config_data = OrderedDict()
for line in lines:
match = re.search(KCFG_HDR_RE, line)
if match:
curr_hdr = match.group(1)
ret[curr_hdr] = []
elif curr_hdr in ret:
ret[curr_hdr].append(line)
else:
ret["no_parent"].append(line)
return ret


class KCFG:
if line.strip().startswith("#"):
continue
tokens = line.strip().split('=')
if len(tokens) == 2:
key = tokens[0].strip()
value = tokens[1].strip()
config_data[key] = value
return config_data

@staticmethod
def parse_opt_str(opt: str) -> tuple:
if not opt.startswith("CONFIG"):
print("-> DEBUG: Malformed kconfig opt, {}".format(opt))
return ()

tmp = opt.split("=")
if len(tmp) != 2:
print("-> DEBUG: Malformed kconfig opt, {}".format(opt))
return ()

return (tmp[0], tmp[1])

def insert_lines(lines: list, start: int, end: int, new_data: list) -> list:
return lines[0:start+1] + new_data + lines[end:]

@staticmethod
def parse_opts_strs(kcfg_sec: list) -> list(tuple()):
opts = [] # list of tuples (CONFIG_*, "m|y|n")
for kcfg in kcfg_sec:
tmp = KCFG.parse_opt_str(kcfg)
if tmp:
opts.append(tmp)
return opts

def insert_kcfg_data(lines: list, start: int, end: int, new_data: OrderedDict) -> dict:
# inserts data into the lines, escape every lines
new_data_lines = ["{}={}\n".format(cfg, val) for (cfg, val) in new_data.items()]
return FileHandler.insert_lines(lines, start, end, new_data_lines)

@staticmethod
def get_writable_opts(opts):
lines = []
for opt in opts:
lines.append("{}={}".format(opt[0], opt[1]))
return lines
def insert_kcfg_excl_data(lines: list, start: int, end: int, new_data: OrderedDict) -> dict:
# inserts data into the lines, escape every lines
new_data_lines = ["{}\n".format(cfg) for (cfg, val) in new_data.items()]
return FileHandler.insert_lines(lines, start, end, new_data_lines)


class Action():
Expand Down
Loading

0 comments on commit da90d56

Please sign in to comment.