Skip to content

Commit

Permalink
chore: refactor vgmanager into different interfaces and introduce tests
Browse files Browse the repository at this point in the history
  • Loading branch information
jakobmoellerdev committed Sep 7, 2023
1 parent 8da92ee commit ef14733
Show file tree
Hide file tree
Showing 23 changed files with 1,279 additions and 834 deletions.
10 changes: 10 additions & 0 deletions cmd/vgmanager/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,11 @@ import (
"os"

lvmv1alpha1 "github.com/openshift/lvm-operator/api/v1alpha1"
"github.com/openshift/lvm-operator/pkg/exec"
"github.com/openshift/lvm-operator/pkg/filter"
"github.com/openshift/lvm-operator/pkg/lsblk"
"github.com/openshift/lvm-operator/pkg/lvm"
"github.com/openshift/lvm-operator/pkg/lvmd"
"github.com/openshift/lvm-operator/pkg/vgmanager"

"k8s.io/apimachinery/pkg/runtime"
Expand Down Expand Up @@ -76,12 +81,17 @@ func main() {
os.Exit(1)
}

executor := &exec.CommandExecutor{}
if err = (&vgmanager.VGReconciler{
Client: mgr.GetClient(),
EventRecorder: mgr.GetEventRecorderFor(vgmanager.ControllerName),
Scheme: mgr.GetScheme(),
NodeName: os.Getenv("NODE_NAME"),
Namespace: os.Getenv("POD_NAMESPACE"),
LVM: lvm.NewHostLVM(executor),
LSBLK: lsblk.NewHostLSBLK(executor, lsblk.DefaultLsblk, lsblk.DefaultMountinfo, lsblk.DefaultLosetup),
LVMDConfig: lvmd.NewDefaultLMVDFileConfig(),
Filters: filter.DefaultFilters,
}).SetupWithManager(mgr); err != nil {
setupLog.Error(err, "unable to create controller", "controller", "VGManager")
os.Exit(1)
Expand Down
1 change: 0 additions & 1 deletion controllers/constants.go
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,6 @@ const (
CSIKubeletRootDir = "/var/lib/kubelet/"
NodeContainerName = "topolvm-node"
TopolvmNodeContainerHealthzName = "healthz"
LvmdConfigFile = "/etc/topolvm/lvmd.yaml"

DefaultCSISocket = "/run/topolvm/csi-topolvm.sock"
DefaultLVMdSocket = "/run/lvmd/lvmd.sock"
Expand Down
62 changes: 27 additions & 35 deletions controllers/lvm_volumegroup.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,12 @@ import (
)

const (
lvmVGName = "lvmvg-manager"
lvmvgFinalizer = "lvm.openshift.io/lvmvolumegroup"
lvmVGName = "lvmvg-manager"

// LegacyVGFinalizer is an old finalizer that was maintained before the Node specific finalizers in vgmanager
// DEPRECATED
// Use no finalizer and remove with 4.16
LegacyVGFinalizer = "lvm.openshift.io/lvmvolumegroup"
)

type lvmVG struct{}
Expand All @@ -56,7 +60,18 @@ func (c lvmVG) ensureCreated(r *LVMClusterReconciler, ctx context.Context, lvmCl
}

result, err := cutil.CreateOrUpdate(ctx, r.Client, existingVolumeGroup, func() error {
existingVolumeGroup.Finalizers = volumeGroup.Finalizers
// removes the old finalizer that was maintained up until 4.15, now we have vgmanager owned finalizers
// per node starting with 4.15
// This code path makes sure to remove the old finalizer if it is encountered from a previous installation
finalizers := existingVolumeGroup.GetFinalizers()
if finalizers != nil {
for i, finalizer := range finalizers {
if finalizer == LegacyVGFinalizer {
existingVolumeGroup.SetFinalizers(append(finalizers[:i], finalizers[i+1:]...))
break
}
}
}
existingVolumeGroup.Spec = volumeGroup.Spec
return nil
})
Expand All @@ -74,7 +89,7 @@ func (c lvmVG) ensureDeleted(r *LVMClusterReconciler, ctx context.Context, lvmCl
logger := log.FromContext(ctx).WithValues("resourceManager", c.getName())
vgcrs := lvmVolumeGroups(r.Namespace, lvmCluster.Spec.Storage.DeviceClasses)

var volumeGroupsPendingInStatus []string
var volumeGroupsPendingDelete []string

for _, volumeGroup := range vgcrs {
vgName := client.ObjectKeyFromObject(volumeGroup)
Expand All @@ -91,28 +106,18 @@ func (c lvmVG) ensureDeleted(r *LVMClusterReconciler, ctx context.Context, lvmCl
if err := r.Client.Delete(ctx, volumeGroup); err != nil {
return fmt.Errorf("failed to delete LVMVolumeGroup %s: %w", volumeGroup.GetName(), err)
}
logger.Info("initiated LVMVolumeGroup deletion", "volumeGroup", client.ObjectKeyFromObject(volumeGroup))
} else {
logger.Info("waiting for LVMVolumeGroup to be deleted", "volumeGroup", client.ObjectKeyFromObject(volumeGroup),
"finalizers", volumeGroup.GetFinalizers())
}

// Has the VG been cleaned up on all hosts?
if doesVGExistInDeviceClassStatus(volumeGroup.Name, lvmCluster) {
volumeGroupsPendingInStatus = append(volumeGroupsPendingInStatus, vgName.String())
continue
}

// Remove finalizer
if update := cutil.RemoveFinalizer(volumeGroup, lvmvgFinalizer); update {
if err := r.Client.Update(ctx, volumeGroup); err != nil {
return fmt.Errorf("failed to remove finalizer from LVMVolumeGroup %s: %w", volumeGroup.GetName(), err)
}
}

logger.Info("initiated LVMVolumeGroup deletion")
volumeGroupsPendingInStatus = append(volumeGroupsPendingInStatus, vgName.String())
volumeGroupsPendingDelete = append(volumeGroupsPendingDelete, vgName.String())
}

if len(volumeGroupsPendingInStatus) > 0 {
return fmt.Errorf("waiting for LVMVolumeGroup's to be removed from nodestatus of %s: %v",
client.ObjectKeyFromObject(lvmCluster), volumeGroupsPendingInStatus)
if len(volumeGroupsPendingDelete) > 0 {
return fmt.Errorf("waiting for LVMVolumeGroup's to be removed of %s: %v",
client.ObjectKeyFromObject(lvmCluster), volumeGroupsPendingDelete)
}

return nil
Expand All @@ -127,9 +132,6 @@ func lvmVolumeGroups(namespace string, deviceClasses []lvmv1alpha1.DeviceClass)
ObjectMeta: metav1.ObjectMeta{
Name: deviceClass.Name,
Namespace: namespace,
Finalizers: []string{
lvmvgFinalizer,
},
},
Spec: lvmv1alpha1.LVMVolumeGroupSpec{
NodeSelector: deviceClass.NodeSelector,
Expand All @@ -142,13 +144,3 @@ func lvmVolumeGroups(namespace string, deviceClasses []lvmv1alpha1.DeviceClass)
}
return lvmVolumeGroups
}

func doesVGExistInDeviceClassStatus(volumeGroup string, instance *lvmv1alpha1.LVMCluster) bool {
dcStatuses := instance.Status.DeviceClassStatuses
for _, dc := range dcStatuses {
if dc.Name == volumeGroup {
return true
}
}
return false
}
11 changes: 6 additions & 5 deletions controllers/topolvm_node.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import (
"strings"

lvmv1alpha1 "github.com/openshift/lvm-operator/api/v1alpha1"
"github.com/openshift/lvm-operator/pkg/lvmd"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
Expand Down Expand Up @@ -137,7 +138,7 @@ func getNodeDaemonSet(lvmCluster *lvmv1alpha1.LVMCluster, namespace string, init
{Name: "lvmd-config-dir",
VolumeSource: corev1.VolumeSource{
HostPath: &corev1.HostPathVolumeSource{
Path: filepath.Dir(LvmdConfigFile),
Path: filepath.Dir(lvmd.DefaultLVMDConfigFilePath),
Type: &hostPathDirectory}}},
{Name: "lvmd-socket-dir",
VolumeSource: corev1.VolumeSource{
Expand Down Expand Up @@ -202,11 +203,11 @@ func getNodeInitContainer(initImage string) *corev1.Container {
command := []string{
"/usr/bin/bash",
"-c",
fmt.Sprintf("until [ -f %s ]; do echo waiting for lvmd config file; sleep 5; done", LvmdConfigFile),
fmt.Sprintf("until [ -f %s ]; do echo waiting for lvmd config file; sleep 5; done", lvmd.DefaultLVMDConfigFilePath),
}

volumeMounts := []corev1.VolumeMount{
{Name: "lvmd-config-dir", MountPath: filepath.Dir(LvmdConfigFile)},
{Name: "lvmd-config-dir", MountPath: filepath.Dir(lvmd.DefaultLVMDConfigFilePath)},
}

fileChecker := &corev1.Container{
Expand All @@ -228,7 +229,7 @@ func getNodeInitContainer(initImage string) *corev1.Container {
func getLvmdContainer() *corev1.Container {
command := []string{
"/lvmd",
fmt.Sprintf("--config=%s", LvmdConfigFile),
fmt.Sprintf("--config=%s", lvmd.DefaultLVMDConfigFilePath),
"--container=true",
}

Expand All @@ -241,7 +242,7 @@ func getLvmdContainer() *corev1.Container {

volumeMounts := []corev1.VolumeMount{
{Name: "lvmd-socket-dir", MountPath: filepath.Dir(DefaultLVMdSocket)},
{Name: "lvmd-config-dir", MountPath: filepath.Dir(LvmdConfigFile)},
{Name: "lvmd-config-dir", MountPath: filepath.Dir(lvmd.DefaultLVMDConfigFilePath)},
}

privilege := true
Expand Down
5 changes: 2 additions & 3 deletions pkg/internal/exec.go → pkg/exec/exec.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License.
*/

package internal
package exec

import (
"fmt"
Expand All @@ -24,7 +24,6 @@ import (

var (
nsenterPath = "/usr/bin/nsenter"
losetupPath = "/usr/sbin/losetup"
)

// Executor is the interface for running exec commands
Expand All @@ -42,7 +41,7 @@ func (*CommandExecutor) ExecuteCommandWithOutput(command string, arg ...string)
return runCommandWithOutput(cmd)
}

// ExecuteCommandWithOutput executes a command with output using nsenter
// ExecuteCommandWithOutputAsHost executes a command with output using nsenter
func (*CommandExecutor) ExecuteCommandWithOutputAsHost(command string, arg ...string) (string, error) {
args := append([]string{"-m", "-u", "-i", "-n", "-p", "-t", "1", command}, arg...)
cmd := exec.Command(nsenterPath, args...)
Expand Down
132 changes: 132 additions & 0 deletions pkg/filter/filter.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,132 @@
/*
Copyright © 2023 Red Hat, Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package filter

import (
"fmt"
"strings"

"github.com/openshift/lvm-operator/pkg/lsblk"
"github.com/openshift/lvm-operator/pkg/lvm"
)

const (
// StateSuspended is a possible value of BlockDevice.State
StateSuspended = "suspended"

// DeviceTypeLoop is the device type for loop devices in lsblk output
DeviceTypeLoop = "loop"

// DeviceTypeROM is the device type for ROM devices in lsblk output
DeviceTypeROM = "rom"

// DeviceTypeLVM is the device type for lvm devices in lsblk output
DeviceTypeLVM = "lvm"

// filter names:
notReadOnly = "notReadOnly"
notSuspended = "notSuspended"
noBiosBootInPartLabel = "noBiosBootInPartLabel"
noReservedInPartLabel = "noReservedInPartLabel"
noValidFilesystemSignature = "noValidFilesystemSignature"
noBindMounts = "noBindMounts"
noChildren = "noChildren"
usableDeviceType = "usableDeviceType"
)

type Filters map[string]func(lsblk.BlockDevice) (bool, error)

func DefaultFilters(lvm lvm.LVM, lsblkInstance lsblk.LSBLK) Filters {
return Filters{
notReadOnly: func(dev lsblk.BlockDevice) (bool, error) {
return !dev.ReadOnly, nil
},

notSuspended: func(dev lsblk.BlockDevice) (bool, error) {
matched := dev.State != StateSuspended
return matched, nil
},

noBiosBootInPartLabel: func(dev lsblk.BlockDevice) (bool, error) {
biosBootInPartLabel := strings.Contains(strings.ToLower(dev.PartLabel), strings.ToLower("bios")) ||
strings.Contains(strings.ToLower(dev.PartLabel), strings.ToLower("boot"))
return !biosBootInPartLabel, nil
},

noReservedInPartLabel: func(dev lsblk.BlockDevice) (bool, error) {
reservedInPartLabel := strings.Contains(strings.ToLower(dev.PartLabel), "reserved")
return !reservedInPartLabel, nil
},

noValidFilesystemSignature: func(dev lsblk.BlockDevice) (bool, error) {
// if no fs type is set, it's always okay
if dev.FSType == "" {
return true, nil
}

// if fstype is set to LVM2_member then it already was created as a PV
// this means that if the disk has no children, we can safely reuse it if it's a valid LVM PV.
if dev.FSType == "LVM2_member" && !dev.HasChildren() {
pvs, err := lvm.ListPVs("")
if err != nil {
return false, fmt.Errorf("could not determine if block device has valid filesystem signature, since it is flagged as LVM2_member but physical volumes could not be verified: %w", err)
}

for _, pv := range pvs {
// a volume is a valid PV if it has the same name as the block device and no associated volume group and has available disk space
// however if there is a PV that matches the Device and there is a VG associated with it or no available space, we cannot use it
if pv.PvName == dev.KName {
if pv.VgName == "" && pv.PvFree != "0G" {
return true, nil
} else {
return false, nil
}
}
}

// if there was no PV that matched it and it still is flagged as LVM2_member, it is formatted but not recognized by LVM
// configuration. We can assume that in this case, the Volume can be reused by simply recalling the vgcreate command on it
return true, nil
}
return false, nil
},

noChildren: func(dev lsblk.BlockDevice) (bool, error) {
hasChildren := dev.HasChildren()
return !hasChildren, nil
},

noBindMounts: func(dev lsblk.BlockDevice) (bool, error) {
hasBindMounts, _, err := lsblkInstance.HasBindMounts(dev)
return !hasBindMounts, err
},

usableDeviceType: func(dev lsblk.BlockDevice) (bool, error) {
switch dev.Type {
case DeviceTypeLoop:
// check loop device isn't being used by kubernetes
return lsblkInstance.IsUsableLoopDev(dev)
case DeviceTypeROM:
return false, nil
case DeviceTypeLVM:
return false, nil
default:
return true, nil
}
},
}
}
Loading

0 comments on commit ef14733

Please sign in to comment.