allow installation to external disks, configure multipathd and extra …

…kernel arguments

fix typo in grub2-editenv

fixing issues for multipath config and kernel arguments

enable multipath in initrd

tidy up and vendor go mod deps

add logic to filter interfaces in use for iscsi session

additional logic to dedup disks and only show one device for multipath'd disks

tweak disk filter logic and make multipathd conditional

tweak comments on disk deduplication logic

fix comment in harv-install describing usage of HARVESTER_ADDITIONAL_KERNEL_ARGUMENTS

rebase upstream changes

remove dracut config files needed to enable multipathd. This is now handled in harvester/os2 directly

(cherry picked from commit f0b21b9)

go.mod

@@ -3,6 +3,7 @@ module github.com/harvester/harvester-installer
 go 1.22.5
 
 require (
+	github.com/dell/goiscsi v1.9.0
 	github.com/harvester/go-common v0.0.0-20230718010724-11313421a8f5
 	github.com/imdario/mergo v0.3.16
 	github.com/insomniacslk/dhcp v0.0.0-20210827173440-b95caade3eac
@@ -71,4 +72,5 @@ replace (
 	k8s.io/api => k8s.io/api v0.24.10
 	k8s.io/apimachinery => k8s.io/apimachinery v0.24.10
 	k8s.io/client-go => k8s.io/client-go v0.24.10
+	k8s.io/kubelet => k8s.io/kubelet v0.24.10
 )

go.sum

@@ -121,6 +121,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/daviddengcn/go-colortext v0.0.0-20160507010035-511bcaf42ccd/go.mod h1:dv4zxwHi5C/8AeI+4gX4dCWOIvNi7I6JCSX0HvlKPgE=
+github.com/dell/goiscsi v1.9.0 h1:VvMHbAO4vk80oc/TAbQPYlxysscCqVBW78GyPoUxgik=
+github.com/dell/goiscsi v1.9.0/go.mod h1:NI/W/0O1UrMW2zVdMxy4z395Jn0r7utH6RQDFSZiFyQ=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
 github.com/docker/cli v20.10.20+incompatible h1:lWQbHSHUFs7KraSN2jOJK7zbMS2jNCHI4mt4xUFUVQ4=

package/harvester-os/files/etc/cos/bootargs.cfg

@@ -4,7 +4,7 @@ set crash_kernel_params="crashkernel=219M,high crashkernel=72M,low"
 if [ "${img}" == "/cOS/recovery.img" ]; then
     set kernelcmd="$console_params root=LABEL=$recovery_label cos-img/filename=$img rd.neednet=1 rd.cos.oemlabel=$oem_label rd.cos.mount=LABEL=$oem_label:/oem rd.cos.oemtimeout=120"
 else
-    set kernelcmd="$console_params root=LABEL=$state_label cos-img/filename=$img panic=0 net.ifnames=1 rd.cos.oemlabel=$oem_label rd.cos.mount=LABEL=$oem_label:/oem rd.cos.mount=LABEL=$persistent_label:/usr/local rd.cos.oemtimeout=120 audit=1 audit_backlog_limit=8192 intel_iommu=on amd_iommu=on iommu=pt"
+    set kernelcmd="$console_params root=LABEL=$state_label cos-img/filename=$img panic=0 net.ifnames=1 rd.cos.oemlabel=$oem_label rd.cos.mount=LABEL=$oem_label:/oem rd.cos.mount=LABEL=$persistent_label:/usr/local rd.cos.oemtimeout=120 audit=1 audit_backlog_limit=8192 intel_iommu=on amd_iommu=on iommu=pt $third_party_kernel_args"
 fi
 
 set initramfs=/boot/initrd

package/harvester-os/files/usr/sbin/harv-install

@@ -435,6 +435,14 @@ update_grub_settings()
     if ! [ -f ${TARGET_FILE} ]; then
         touch ${TARGET_FILE}
     fi
+    # /etc/cos/bootargs.cfg appends a new variable $third_party_kernel_args
+    # if harvester config has os.externalStorageConfig.additionalKernelArguments specified
+    # then these will be mapped to HARVESTER_ADDITIONAL_KERNEL_ARGUMENTS
+    # and will be added to /oem/grubenv file
+    TARGET_FILE="${oem_dir}/grubenv"
+    if [ -n "${HARVESTER_ADDITIONAL_KERNEL_ARGUMENTS}" ]; then
+        grub2-editenv ${TARGET_FILE} set third_party_kernel_args="${HARVESTER_ADDITIONAL_KERNEL_ARGUMENTS}"
+    fi
 
     add_debug_grub_entry
 }

pkg/config/config.go

@@ -204,7 +204,19 @@ type OS struct {
 	Labels         map[string]string `json:"labels,omitempty"`
 	SSHD           SSHDConfig        `json:"sshd,omitempty"`
 
-	PersistentStatePaths []string `json:"persistentStatePaths,omitempty"`
+	PersistentStatePaths      []string              `json:"persistentStatePaths,omitempty"`
+	ExternalStorage           ExternalStorageConfig `json:"externalStorageConfig,omitempty"`
+	AdditionalKernelArguments string                `json:"additionalKernelArguments,omitempty"`
+}
+
+type ExternalStorageConfig struct {
+	Enabled         bool         `json:"enabled,omitempty"`
+	MultiPathConfig []DiskConfig `json:"multiPathConfig,omitempty"`
+}
+
+type DiskConfig struct {
+	Vendor  string `json:"vendor"`
+	Product string `json:"product"`
 }
 
 // SSHDConfig is the SSHD configuration for the node

pkg/config/config_test.go

@@ -7,6 +7,7 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 	"gopkg.in/yaml.v3"
 )
 
@@ -539,3 +540,48 @@ func TestCalculateCPUReservedInMilliCPU(t *testing.T) {
 		assert.Equal(t, tc.reservedMilliCores, calculateCPUReservedInMilliCPU(tc.coreNum, tc.maxPods))
 	}
 }
+func Test_MultipathConfig(t *testing.T) {
+	assert := require.New(t)
+	config := NewHarvesterConfig()
+	config.OS.ExternalStorage = ExternalStorageConfig{
+		Enabled: true,
+		MultiPathConfig: []DiskConfig{
+			{
+				Vendor:  "DELL",
+				Product: "DISK1",
+			},
+			{
+				Vendor:  "HPE",
+				Product: "DISK2",
+			},
+		},
+	}
+
+	content, err := render("multipath.conf.tmpl", config)
+	assert.NoError(err, "expected no error while rending multipath config")
+	t.Log("rendered multipath config:")
+	t.Log(content)
+}
+
+func Test_ToCosInstallEnv(t *testing.T) {
+	hvConfig := NewHarvesterConfig()
+	hvConfig.OS.ExternalStorage = ExternalStorageConfig{
+		Enabled: true,
+		MultiPathConfig: []DiskConfig{
+			{
+				Vendor:  "DELL",
+				Product: "DISK1",
+			},
+			{
+				Vendor:  "HPE",
+				Product: "DISK2",
+			},
+		},
+	}
+	hvConfig.OS.AdditionalKernelArguments = "rd.iscsi.firmware rd.iscsi.ibft"
+	assert := require.New(t)
+	env, err := hvConfig.ToCosInstallEnv()
+	assert.NoError(err)
+	t.Log(env)
+
+}

pkg/config/cos.go

@@ -175,6 +175,11 @@ func ConvertToCOS(config *HarvesterConfig) (*yipSchema.YipConfig, error) {
 		})
 	}
 
+	// enable multipathd for external storage support
+	if err := setupExternalStorage(config, &initramfs); err != nil {
+		return nil, err
+	}
+
 	// TOP
 	if cfg.Mode != ModeInstall {
 		if err := initRancherdStage(config, &initramfs); err != nil {
@@ -840,3 +845,24 @@ func CreateRootPartitioningLayout(elementalConfig *ElementalConfig, hvstConfig *
 
 	return elementalConfig, nil
 }
+
+// setupExternalStorage is needed to support boot of external disks
+// this involves enable multipath service and configuring it to blacklist
+// all devices except the ones listed in the config.OS.ExternalStorage.MultiPathConfig
+
+func setupExternalStorage(config *HarvesterConfig, stage *yipSchema.Stage) error {
+	if !config.OS.ExternalStorage.Enabled {
+		return nil
+	}
+	stage.Systemctl.Enable = append(stage.Systemctl.Enable, "multipathd")
+	content, err := render("multipath.conf.tmpl", config)
+	if err != nil {
+		return fmt.Errorf("error rending multipath.conf template: %v", err)
+	}
+	stage.Files = append(stage.Files, yipSchema.File{
+		Path:        "/etc/multipath.conf",
+		Content:     content,
+		Permissions: 0755,
+	})
+	return nil
+}

pkg/config/templates/multipath.conf.tmpl

@@ -0,0 +1,8 @@
+blacklist {
+{{ range $val := .ExternalStorage.MultiPathConfig }}
+    device {
+        vendor "!{{ $val.Vendor }}"
+        product "!{{ $val.Product }}"
+    }
+{{ end }}
+}

pkg/console/install_panels.go

@@ -289,11 +289,13 @@ func getDataDiskOptions(hvstConfig *config.HarvesterConfig) ([]widgets.Option, e
 }
 
 func getDiskOptions() ([]widgets.Option, error) {
-	output, err := exec.Command("/bin/sh", "-c", `lsblk -r -o NAME,SIZE,TYPE | grep -w disk | cut -d ' ' -f 1,2`).CombinedOutput()
+	output, err := exec.Command("/bin/sh", "-c", `lsblk -J -o NAME,SIZE,TYPE,WWN,SERIAL`).CombinedOutput()
 	if err != nil {
 		return nil, err
 	}
-	lines := strings.Split(strings.TrimSuffix(string(output), "\n"), "\n")
+
+	lines, err := identifyUniqueDisks(output)
+
 	var options []widgets.Option
 	for _, line := range lines {
 		splits := strings.SplitN(line, " ", 2)

pkg/console/network.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
+	"net"
 	"os"
 	"os/exec"
 	"syscall"
@@ -14,6 +15,8 @@ import (
 	"golang.org/x/sys/unix"
 	"gopkg.in/yaml.v3"
 
+	"github.com/dell/goiscsi"
+
 	"github.com/harvester/harvester-installer/pkg/config"
 )
 
@@ -137,7 +140,7 @@ func getNICs() ([]netlink.Link, error) {
 		}
 	}
 
-	return nics, nil
+	return filterISCSIInterfaces(nics)
 }
 
 func getNICState(name string) int {
@@ -190,3 +193,47 @@ func getManagementInterfaceName(mgmtInterface config.Network) string {
 	}
 	return mgmtName
 }
+
+// filterISCSIInterfaces will query the host to identify iscsi sessions, and skip interfaces
+// used by the existing iscsi session.
+func filterISCSIInterfaces(links []netlink.Link) ([]netlink.Link, error) {
+	iscsi := goiscsi.NewLinuxISCSI(nil)
+	sessions, err := iscsi.GetSessions()
+	if err != nil {
+		return nil, fmt.Errorf("error querying iscsi sessions: %v", err)
+	}
+
+	var returnLinks []netlink.Link
+	for _, link := range links {
+		var inuse bool
+		if getNICState(link.Attrs().Name) == NICStateUP {
+			iface, err := net.InterfaceByName(link.Attrs().Name)
+			if err != nil {
+				return nil, fmt.Errorf("error fetching interface details: %v", err)
+			}
+
+			addresses, err := iface.Addrs()
+			if err != nil {
+				return nil, fmt.Errorf("error fetching addresses from interface: %v", err)
+			}
+
+			for _, address := range addresses {
+				// interface addresses are in cidr format, and need to be converted before comparison
+				// since iscsi session contains just the ip address
+				ipAddress, _, err := net.ParseCIDR(address.String())
+				if err != nil {
+					return nil, fmt.Errorf("error parsing ip address: %v", err)
+				}
+				for _, session := range sessions {
+					if session.IfaceIPaddress == ipAddress.String() {
+						inuse = true
+					}
+				}
+			}
+		}
+		if !inuse {
+			returnLinks = append(returnLinks, link)
+		}
+	}
+	return returnLinks, nil
+}

pkg/console/util.go

@@ -504,6 +504,10 @@ func doInstall(g *gocui.Gui, hvstConfig *config.HarvesterConfig, webhooks Render
 		env = append(env, fmt.Sprintf("HARVESTER_DATA_DISK=%s", hvstConfig.DataDisk))
 	}
 
+	if hvstConfig.OS.AdditionalKernelArguments != "" {
+		env = append(env, fmt.Sprintf("HARVESTER_ADDITIONAL_KERNEL_ARGUMENTS=%s", hvstConfig.OS.AdditionalKernelArguments))
+	}
+
 	elementalConfigDir, elementalConfigFile, err := saveElementalConfig(elementalConfig)
 	if err != nil {
 		return nil
@@ -982,3 +986,78 @@ func generateEnvAndConfig(g *gocui.Gui, hvstConfig *config.HarvesterConfig) ([]s
 	env = append(env, fmt.Sprintf("HARVESTER_STREAMDISK_CLOUDINIT_URL=%s", userDataURL))
 	return env, elementalConfig, nil
 }
+
+// internal objects to parse lsblk output
+type BlockDevices struct {
+	Disks []Device `json:"blockdevices"`
+}
+
+type Device struct {
+	Name     string   `json:"name"`
+	Size     string   `json:"size"`
+	DiskType string   `json:"type"`
+	WWN      string   `json:"wwn,omitempty"`
+	Serial   string   `json:"serial,omitempty"`
+	Children []Device `json:"children,omitempty"`
+}
+
+func generateDiskEntry(d Device) string {
+	return fmt.Sprintf("%s %s", d.Name, d.Size)
+}
+
+const (
+	diskType = "disk"
+)
+
+// identifyUniqueDisks parses the json output of lsblk and identifies
+// unique disks by comparing their serial number info and wwn details
+func identifyUniqueDisks(output []byte) ([]string, error) {
+	var returnDisks []string
+	disks := &BlockDevices{}
+	err := json.Unmarshal(output, disks)
+	if err != nil {
+		return nil, fmt.Errorf("error unmarshalling lsblk json output: %v", err)
+	}
+
+	// identify devices which may be unique
+	dedupMap := make(map[string]Device)
+	for _, disk := range disks.Disks {
+		if disk.DiskType == diskType {
+			// no serial or wwn info present
+			// add to list of disks
+			if disk.WWN == "" && disk.Serial == "" {
+				returnDisks = append(returnDisks, generateDiskEntry(disk))
+				continue
+			}
+
+			if disk.Serial != "" {
+				_, ok := dedupMap[disk.Serial]
+				if !ok {
+					dedupMap[disk.Serial] = disk
+				}
+				continue
+			}
+
+			if disk.WWN != "" {
+				_, ok := dedupMap[disk.WWN]
+				if !ok {
+					dedupMap[disk.WWN] = disk
+				}
+				continue
+			}
+		}
+	}
+	// devices may appear twice in the map when both serial number and wwn info is present
+	// we need to ensure only unique names are shown in the console
+	resultMap := make(map[string]Device)
+	for _, v := range dedupMap {
+		resultMap[v.Name] = v
+	}
+
+	// generate list of disks
+	for _, v := range resultMap {
+		returnDisks = append(returnDisks, generateDiskEntry(v))
+	}
+
+	return returnDisks, nil
+}