Skip to content

Commit

Permalink
Volume handling
Browse files Browse the repository at this point in the history
  • Loading branch information
@lukasfrank
lukasfrank committed Nov 2, 2023
1 parent c5fe8fc commit 39c48c3
Show file tree
Hide file tree
Showing 7 changed files with 114 additions and 59 deletions.
71 changes: 48 additions & 23 deletions driver/cmd/app/app.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@ import (
"context"
goflag "flag"
"fmt"
virtlethost "github.com/onmetal/libvirt-driver/pkg/virtlethost"
"net"
"os"
"path/filepath"
Expand All @@ -29,6 +28,7 @@ import (
"github.com/onmetal/libvirt-driver/pkg/controllers"
"github.com/onmetal/libvirt-driver/pkg/event"
"github.com/onmetal/libvirt-driver/pkg/host"
virtletimage "github.com/onmetal/libvirt-driver/pkg/image"
"github.com/onmetal/libvirt-driver/pkg/libvirt/guest"
libvirtutils "github.com/onmetal/libvirt-driver/pkg/libvirt/utils"
"github.com/onmetal/libvirt-driver/pkg/mcr"
Expand All @@ -38,9 +38,11 @@ import (
"github.com/onmetal/libvirt-driver/pkg/qcow2"
"github.com/onmetal/libvirt-driver/pkg/raw"
"github.com/onmetal/libvirt-driver/pkg/utils"
virtlethost "github.com/onmetal/libvirt-driver/pkg/virtlethost"
"github.com/onmetal/onmetal-api/broker/common"
commongrpc "github.com/onmetal/onmetal-api/broker/common/grpc"
ori "github.com/onmetal/onmetal-api/ori/apis/machine/v1alpha1"
"github.com/onmetal/onmetal-image/oci/remote"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
"golang.org/x/sync/errgroup"
Expand Down Expand Up @@ -139,6 +141,30 @@ func Run(ctx context.Context, opts Options) error {
os.Exit(1)
}

reg, err := remote.DockerRegistry(nil)
if err != nil {
setupLog.Error(err, "error creating registry")
os.Exit(1)
}

imgCache, err := virtletimage.NewLocalCache(log, reg, virtletHost.OCIStore())
if err != nil {
setupLog.Error(err, "error setting up image manager")
os.Exit(1)
}

qcow2Inst, err := qcow2.Instance(opts.Libvirt.Qcow2Type)
if err != nil {
setupLog.Error(err, "error creating qcow2 instance")
os.Exit(1)
}

rawInst, err := raw.Instance(raw.Default())
if err != nil {
setupLog.Error(err, "error creating raw instance")
os.Exit(1)
}

// Detect Guest Capabilities
caps, err := guest.DetectCapabilities(libvirt, guest.CapabilitiesOptions{
PreferredDomainTypes: opts.Libvirt.PreferredDomainTypes,
Expand All @@ -149,6 +175,14 @@ func Run(ctx context.Context, opts Options) error {
os.Exit(1)
}

volumePlugins := volumeplugin.NewPluginManager()
if err := volumePlugins.InitPlugins(virtletHost, []volumeplugin.Plugin{
ceph.NewPlugin(),
emptydisk.NewPlugin(qcow2Inst, rawInst),
}); err != nil {
return fmt.Errorf("failed to initialize machine class registry: %w", err)
}

setupLog.Info("Configuring machine store", "Directory", virtletHost.MachineStoreDir())
machineStore, err := host.NewStore(host.Options[*api.Machine]{
NewFunc: func() *api.Machine { return &api.Machine{} },
Expand All @@ -174,8 +208,11 @@ func Run(ctx context.Context, opts Options) error {
machineStore,
machineEvents,
controllers.MachineReconcilerOptions{
GuestCapabilities: caps,
Host: virtletHost,
GuestCapabilities: caps,
ImageCache: imgCache,
Raw: rawInst,
Host: virtletHost,
VolumePluginManager: volumePlugins,
},
)
if err != nil {
Expand All @@ -195,26 +232,6 @@ func Run(ctx context.Context, opts Options) error {
return fmt.Errorf("failed to initialize machine class registry: %w", err)
}

qcow2Inst, err := qcow2.Instance(opts.Libvirt.Qcow2Type)
if err != nil {
setupLog.Error(err, "error creating qcow2 instance")
os.Exit(1)
}

rawInst, err := raw.Instance(raw.Default())
if err != nil {
setupLog.Error(err, "error creating raw instance")
os.Exit(1)
}

volumePlugins := volumeplugin.NewPluginManager()
if err := volumePlugins.InitPlugins(virtletHost, []volumeplugin.Plugin{
ceph.NewPlugin(),
emptydisk.NewPlugin(qcow2Inst, rawInst),
}); err != nil {
return fmt.Errorf("failed to initialize machine class registry: %w", err)
}

srv, err := server.New(server.Options{
MachineStore: machineStore,
MachineClasses: machineClasses,
Expand All @@ -225,6 +242,14 @@ func Run(ctx context.Context, opts Options) error {
}

g, ctx := errgroup.WithContext(ctx)
g.Go(func() error {
setupLog.Info("Starting image cache")
if err := imgCache.Start(ctx); err != nil {
log.Error(err, "failed to start image cache")
return err
}
return nil
})

g.Go(func() error {
setupLog.Info("Starting machine reconciler")
Expand Down
54 changes: 32 additions & 22 deletions pkg/controllers/machine_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@ import (
"context"
"errors"
"fmt"
virtletvolume "github.com/onmetal/libvirt-driver/pkg/plugins/volume"
"os"
"slices"
"sync"
Expand All @@ -30,13 +29,14 @@ import (
virtletimage "github.com/onmetal/libvirt-driver/pkg/image"
"github.com/onmetal/libvirt-driver/pkg/libvirt/guest"
"github.com/onmetal/libvirt-driver/pkg/os/osutils"
virtletvolume "github.com/onmetal/libvirt-driver/pkg/plugins/volume"
"github.com/onmetal/libvirt-driver/pkg/raw"
"github.com/onmetal/libvirt-driver/pkg/store"
"github.com/onmetal/libvirt-driver/pkg/utils"
virtlethost "github.com/onmetal/libvirt-driver/pkg/virtlethost" // TODO: Change to a better naming for all imports, libvirthost?
"github.com/onmetal/virtlet/libvirt/libvirtutils"
"k8s.io/client-go/util/workqueue"
"k8s.io/utils/pointer"
"k8s.io/utils/ptr"
"libvirt.org/go/libvirtxml"
)

Expand All @@ -59,16 +59,15 @@ var (
//libvirt.DomainShutoff: api.MachineStateShutdown,
libvirt.DomainPmsuspended: api.MachineStatePending,
}

errNoNetworkInterfaceAlias = errors.New("no network interface alias")
)

type MachineReconcilerOptions struct {
GuestCapabilities guest.Capabilities
TCMallocLibPath string
ImageCache virtletimage.Cache
Raw raw.Raw
Host virtlethost.Host
GuestCapabilities guest.Capabilities
TCMallocLibPath string
ImageCache virtletimage.Cache
Raw raw.Raw
Host virtlethost.Host
VolumePluginManager *virtletvolume.PluginManager
}

func NewMachineReconciler(
Expand All @@ -91,16 +90,17 @@ func NewMachineReconciler(
}

return &MachineReconciler{
log: log,
queue: workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter()),
libvirt: libvirt,
machines: machines,
machineEvents: machineEvents,
guestCapabilities: opts.GuestCapabilities,
tcMallocLibPath: opts.TCMallocLibPath,
host: opts.Host,
imageCache: opts.ImageCache,
raw: opts.Raw,
log: log,
queue: workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter()),
libvirt: libvirt,
machines: machines,
machineEvents: machineEvents,
guestCapabilities: opts.GuestCapabilities,
tcMallocLibPath: opts.TCMallocLibPath,
host: opts.Host,
imageCache: opts.ImageCache,
raw: opts.Raw,
volumePluginManager: opts.VolumePluginManager,
}, nil
}

Expand Down Expand Up @@ -272,6 +272,14 @@ func (r *MachineReconciler) deleteMachine(ctx context.Context, log logr.Logger,
return nil
}

log.V(1).Info("Finalizer present, doing cleanup")

log.V(1).Info("Removing volumes")
if err := r.deleteVolumes(ctx, log, machine); err != nil {
return fmt.Errorf("error removing machine disks: %w", err)
}
log.V(1).Info("Successfully removed machine disks")

//do libvirt cleanup

machine.Finalizers = utils.DeleteSliceElement(machine.Finalizers, MachineFinalizer)
Expand Down Expand Up @@ -459,8 +467,8 @@ func (r *MachineReconciler) domainFor(
return nil, nil, nil, err
}

if machineImgRef := machine.Spec.Image; machineImgRef != nil && pointer.StringDeref(machineImgRef, "") != "" {
if err := r.setDomainImage(ctx, machine, domainDesc, pointer.StringDeref(machineImgRef, "")); err != nil {
if machineImgRef := machine.Spec.Image; machineImgRef != nil && ptr.Deref(machineImgRef, "") != "" {
if err := r.setDomainImage(ctx, machine, domainDesc, ptr.Deref(machineImgRef, "")); err != nil {
return nil, nil, nil, err
}
}
Expand Down Expand Up @@ -495,8 +503,10 @@ func (r *MachineReconciler) setDomainResources(ctx context.Context, log logr.Log
Value: uint(machine.Spec.MemoryBytes),
Unit: "Byte",
}

cpu := uint(machine.Spec.CpuMillis / 1000)
domain.VCPU = &libvirtxml.DomainVCPU{
Value: uint(machine.Spec.CpuMillis),
Value: cpu,
}
return nil
}
Expand Down
6 changes: 3 additions & 3 deletions pkg/controllers/machine_controller_volumes.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,6 @@ import (
"encoding/base64"
"errors"
"fmt"
"github.com/onmetal/libvirt-driver/pkg/api"
virtlethost "github.com/onmetal/libvirt-driver/pkg/virtlethost"
utilstrings "k8s.io/utils/strings"
"os"
"path/filepath"
"slices"
Expand All @@ -31,9 +28,12 @@ import (
"github.com/digitalocean/go-libvirt"
"github.com/go-logr/logr"
"github.com/google/uuid"
"github.com/onmetal/libvirt-driver/pkg/api"
virtletvolume "github.com/onmetal/libvirt-driver/pkg/plugins/volume"
virtlethost "github.com/onmetal/libvirt-driver/pkg/virtlethost"
"github.com/onmetal/virtlet/libvirt/libvirtutils"
"k8s.io/apimachinery/pkg/util/sets"
utilstrings "k8s.io/utils/strings"
"libvirt.org/go/libvirtxml"
)

Expand Down
Empty file removed pkg/controllers/volume_controller.go
View file
Empty file.
16 changes: 15 additions & 1 deletion pkg/plugins/volume/ceph/ceph.go
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,17 @@
// Copyright 2023 OnMetal authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package ceph

import (
Expand All @@ -21,7 +35,7 @@ const (
secretUserIDKey = "userID"
secretUserKeyKey = "userKey"

secretEncryptionKey = "encryptionKey"
//secretEncryptionKey = "encryptionKey"
)

type plugin struct {
Expand Down
18 changes: 16 additions & 2 deletions pkg/plugins/volume/emptydisk/emptydisk.go
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,17 @@
// Copyright 2023 OnMetal authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package emptydisk

import (
Expand All @@ -6,13 +20,13 @@ import (
"encoding/hex"
"errors"
"fmt"
"github.com/onmetal/libvirt-driver/pkg/qcow2"
"github.com/onmetal/libvirt-driver/pkg/raw"
"os"
"path/filepath"

"github.com/onmetal/libvirt-driver/pkg/api"
"github.com/onmetal/libvirt-driver/pkg/plugins/volume"
"github.com/onmetal/libvirt-driver/pkg/qcow2"
"github.com/onmetal/libvirt-driver/pkg/raw"
utilstrings "k8s.io/utils/strings"
)

Expand Down
8 changes: 0 additions & 8 deletions pkg/utils/strategy.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,11 +25,3 @@ type machineStrategy struct{}
func (machineStrategy) PrepareForCreate(obj *api.Machine) {
obj.Status = api.MachineStatus{State: api.MachineStatePending}
}

var VolumeStrategy = volumeStrategy{}

type volumeStrategy struct{}

func (volumeStrategy) PrepareForCreate(obj *api.Volume) {
obj.Status = api.VolumeStatus{State: api.VolumeStatePending}
}

0 comments on commit 39c48c3

Please sign in to comment.