Machine Reconciler

driver/api/v1alpha1/common_types.go

@@ -21,9 +21,8 @@ const (
 )
 
 const (
-	ManagerLabel    = "libvirt-driver.api.onmetal.de/manager"
-	ClassLabel      = "libvirt-driver.api.onmetal.de/class"
-	MachineRefLabel = "libvirt-driver.api.onmetal.de/machine"
+	ManagerLabel = "libvirt-driver.api.onmetal.de/manager"
+	ClassLabel   = "libvirt-driver.api.onmetal.de/class"
 )
 
 const (

driver/apiutils/apiutils.go

@@ -115,20 +115,11 @@ func SetClassLabel(o api.Object, class string) {
 	metautils.SetLabel(o, machinev1alpha1.ClassLabel, class)
 }
 
-func SetMachineRefLabel(o api.Object, machineID string) {
-	metautils.SetLabel(o, machinev1alpha1.MachineRefLabel, machineID)
-}
-
 func GetClassLabel(o api.Object) (string, bool) {
 	class, found := o.GetLabels()[machinev1alpha1.ClassLabel]
 	return class, found
 }
 
-func GetMachineRefLabel(o api.Object) (string, bool) {
-	machineRef, found := o.GetLabels()[machinev1alpha1.MachineRefLabel]
-	return machineRef, found
-}
-
 func IsManagedBy(o api.Object, manager string) bool {
 	actual, ok := o.GetLabels()[machinev1alpha1.ManagerLabel]
 	return ok && actual == manager

driver/cmd/app/app.go

@@ -23,43 +23,61 @@ import (
 	"path/filepath"
 
 	"github.com/go-logr/logr"
+	"github.com/onmetal/libvirt-driver/driver/networkinterfaceplugin"
 	"github.com/onmetal/libvirt-driver/driver/server"
 	"github.com/onmetal/libvirt-driver/pkg/api"
 	"github.com/onmetal/libvirt-driver/pkg/controllers"
 	"github.com/onmetal/libvirt-driver/pkg/event"
 	"github.com/onmetal/libvirt-driver/pkg/host"
+	virtletimage "github.com/onmetal/libvirt-driver/pkg/image"
 	"github.com/onmetal/libvirt-driver/pkg/libvirt/guest"
+	libvirtutils "github.com/onmetal/libvirt-driver/pkg/libvirt/utils"
 	"github.com/onmetal/libvirt-driver/pkg/mcr"
 	volumeplugin "github.com/onmetal/libvirt-driver/pkg/plugins/volume"
 	"github.com/onmetal/libvirt-driver/pkg/plugins/volume/ceph"
 	"github.com/onmetal/libvirt-driver/pkg/plugins/volume/emptydisk"
 	"github.com/onmetal/libvirt-driver/pkg/qcow2"
 	"github.com/onmetal/libvirt-driver/pkg/raw"
 	"github.com/onmetal/libvirt-driver/pkg/utils"
+	virtlethost "github.com/onmetal/libvirt-driver/pkg/virtlethost"
+	apinetv1alpha1 "github.com/onmetal/onmetal-api-net/api/core/v1alpha1"
 	"github.com/onmetal/onmetal-api/broker/common"
 	commongrpc "github.com/onmetal/onmetal-api/broker/common/grpc"
 	ori "github.com/onmetal/onmetal-api/ori/apis/machine/v1alpha1"
-	"github.com/onmetal/virtlet/libvirt/libvirtutils"
+	"github.com/onmetal/onmetal-image/oci/remote"
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
 	"golang.org/x/sync/errgroup"
 	"google.golang.org/grpc"
+	"k8s.io/apimachinery/pkg/runtime"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
+	"k8s.io/client-go/tools/clientcmd"
 	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 )
 
 var (
 	homeDir string
+	scheme  = runtime.NewScheme()
 )
 
 func init() {
 	homeDir, _ = os.UserHomeDir()
+
+	utilruntime.Must(clientgoscheme.AddToScheme(scheme))
+	utilruntime.Must(apinetv1alpha1.AddToScheme(scheme))
 }
 
 type Options struct {
 	Address string
 	RootDir string
-	Libvirt LibvirtOptions
+
+	ApinetKubeconfig string
+
+	Libvirt   LibvirtOptions
+	NicPlugin *networkinterfaceplugin.Options
 }
 
 type LibvirtOptions struct {
@@ -77,6 +95,8 @@ func (o *Options) AddFlags(fs *pflag.FlagSet) {
 	fs.StringVar(&o.Address, "address", "/var/run/ori-machinebroker.sock", "Address to listen on.")
 	fs.StringVar(&o.RootDir, "virtlet-dir", filepath.Join(homeDir, ".virtlet"), "Path to the directory virtlet manages its content at.")
 
+	fs.StringVar(&o.ApinetKubeconfig, "apinet-kubeconfig", "", "Path to the kubeconfig file for the apinet-cluster.")
+
 	// LibvirtOptions
 	fs.StringVar(&o.Libvirt.Socket, "libvirt-socket", o.Libvirt.Socket, "Path to the libvirt socket to use.")
 	fs.StringVar(&o.Libvirt.Address, "libvirt-address", o.Libvirt.Address, "Address of a RPC libvirt socket to connect to.")
@@ -87,6 +107,9 @@ func (o *Options) AddFlags(fs *pflag.FlagSet) {
 	fs.StringSliceVar(&o.Libvirt.PreferredMachineTypes, "preferred-machine-types", []string{"pc-q35"}, "Ordered list of preferred machine types to use.")
 
 	fs.StringVar(&o.Libvirt.Qcow2Type, "qcow2-type", qcow2.Default(), fmt.Sprintf("qcow2 implementation to use. Available: %v", qcow2.Available()))
+
+	o.NicPlugin = networkinterfaceplugin.NewDefaultOptions()
+	o.NicPlugin.AddFlags(fs)
 }
 
 func Command() *cobra.Command {
@@ -124,52 +147,99 @@ func Run(ctx context.Context, opts Options) error {
 	libvirt, err := libvirtutils.GetLibvirt(opts.Libvirt.Socket, opts.Libvirt.Address, opts.Libvirt.URI)
 	if err != nil {
 		setupLog.Error(err, "error getting libvirt")
-		os.Exit(1)
+		return err
 	}
 	defer func() {
 		if err := libvirt.ConnectClose(); err != nil {
 			setupLog.Error(err, "Error closing libvirt connection")
 		}
 	}()
 
+	// Check if apinetKubeconfig is provided
+	var apinetClient client.Client
+	if opts.ApinetKubeconfig != "" {
+		apinetCfg, err := clientcmd.BuildConfigFromFlags("", opts.ApinetKubeconfig)
+		if err != nil {
+			setupLog.Error(err, "Failed to build config from apinet-kubeconfig")
+			return err
+		}
+
+		apinetClient, err = client.New(apinetCfg, client.Options{Scheme: scheme})
+		if err != nil {
+			setupLog.Error(err, "Error creating api-net client:")
+			return err
+		}
+	}
+
+	virtletHost, err := virtlethost.NewLibvirtAt(apinetClient, opts.RootDir, libvirt)
+	if err != nil {
+		setupLog.Error(err, "error creating virtlet host")
+		return err
+	}
+
+	reg, err := remote.DockerRegistry(nil)
+	if err != nil {
+		setupLog.Error(err, "error creating registry")
+		return err
+	}
+
+	imgCache, err := virtletimage.NewLocalCache(log, reg, virtletHost.OCIStore())
+	if err != nil {
+		setupLog.Error(err, "error setting up image manager")
+		return err
+	}
+
+	qcow2Inst, err := qcow2.Instance(opts.Libvirt.Qcow2Type)
+	if err != nil {
+		setupLog.Error(err, "error creating qcow2 instance")
+		return err
+	}
+
+	rawInst, err := raw.Instance(raw.Default())
+	if err != nil {
+		setupLog.Error(err, "error creating raw instance")
+		return err
+	}
+
 	// Detect Guest Capabilities
 	caps, err := guest.DetectCapabilities(libvirt, guest.CapabilitiesOptions{
 		PreferredDomainTypes:  opts.Libvirt.PreferredDomainTypes,
 		PreferredMachineTypes: opts.Libvirt.PreferredMachineTypes,
 	})
 	if err != nil {
 		setupLog.Error(err, "error detecting guest capabilities")
-		os.Exit(1)
+		return err
 	}
 
-	volumeStoreDir := filepath.Join(opts.RootDir, "volumes")
-	setupLog.Info("Configuring volume store", "Directory", volumeStoreDir)
-	volumeStore, err := host.NewStore(host.Options[*api.Volume]{
-		NewFunc:        func() *api.Volume { return &api.Volume{} },
-		CreateStrategy: utils.VolumeStrategy,
-		Dir:            volumeStoreDir,
-	})
-	if err != nil {
-		return fmt.Errorf("failed to initialize volume store: %w", err)
+	volumePlugins := volumeplugin.NewPluginManager()
+	if err := volumePlugins.InitPlugins(virtletHost, []volumeplugin.Plugin{
+		ceph.NewPlugin(),
+		emptydisk.NewPlugin(qcow2Inst, rawInst),
+	}); err != nil {
+		return fmt.Errorf("failed to initialize machine class registry: %w", err)
 	}
 
-	volumeEvents, err := event.NewListWatchSource[*api.Volume](
-		volumeStore.List,
-		volumeStore.Watch,
-		event.ListWatchSourceOptions{},
-	)
+	nicPlugin, nicPluginCleanup, err := opts.NicPlugin.NetworkInterfacePlugin()
 	if err != nil {
-		return fmt.Errorf("failed to initialize volume events: %w", err)
+		setupLog.Error(err, "Error creating network plugin")
+		return err
+	}
+	if nicPluginCleanup != nil {
+		defer nicPluginCleanup()
 	}
 
-	_ = volumeEvents
+	setupLog.Info("Initializing network interface plugin")
 
-	machineStoreDir := filepath.Join(opts.RootDir, "machines")
-	setupLog.Info("Configuring machine store", "Directory", machineStoreDir)
+	if err := nicPlugin.Init(virtletHost); err != nil {
+		setupLog.Error(err, "Error initializing network plugin")
+		return err
+	}
+
+	setupLog.Info("Configuring machine store", "Directory", virtletHost.MachineStoreDir())
 	machineStore, err := host.NewStore(host.Options[*api.Machine]{
 		NewFunc:        func() *api.Machine { return &api.Machine{} },
 		CreateStrategy: utils.MachineStrategy,
-		Dir:            machineStoreDir,
+		Dir:            virtletHost.MachineStoreDir(),
 	})
 	if err != nil {
 		return fmt.Errorf("failed to initialize machine store: %w", err)
@@ -189,9 +259,13 @@ func Run(ctx context.Context, opts Options) error {
 		libvirt,
 		machineStore,
 		machineEvents,
-		volumeStore,
 		controllers.MachineReconcilerOptions{
-			GuestCapabilities: caps,
+			GuestCapabilities:      caps,
+			ImageCache:             imgCache,
+			Raw:                    rawInst,
+			Host:                   virtletHost,
+			VolumePluginManager:    volumePlugins,
+			NetworkInterfacePlugin: nicPlugin,
 		},
 	)
 	if err != nil {
@@ -211,29 +285,8 @@ func Run(ctx context.Context, opts Options) error {
 		return fmt.Errorf("failed to initialize machine class registry: %w", err)
 	}
 
-	qcow2Inst, err := qcow2.Instance(opts.Libvirt.Qcow2Type)
-	if err != nil {
-		setupLog.Error(err, "error creating qcow2 instance")
-		os.Exit(1)
-	}
-
-	rawInst, err := raw.Instance(raw.Default())
-	if err != nil {
-		setupLog.Error(err, "error creating raw instance")
-		os.Exit(1)
-	}
-
-	volumePlugins := volumeplugin.NewPluginManager()
-	if err := volumePlugins.InitPlugins([]volumeplugin.Plugin{
-		ceph.NewPlugin(),
-		emptydisk.NewPlugin(qcow2Inst, rawInst),
-	}); err != nil {
-		return fmt.Errorf("failed to initialize machine class registry: %w", err)
-	}
-
 	srv, err := server.New(server.Options{
 		MachineStore:   machineStore,
-		VolumeStore:    volumeStore,
 		MachineClasses: machineClasses,
 		VolumePlugins:  volumePlugins,
 	})
@@ -242,6 +295,14 @@ func Run(ctx context.Context, opts Options) error {
 	}
 
 	g, ctx := errgroup.WithContext(ctx)
+	g.Go(func() error {
+		setupLog.Info("Starting image cache")
+		if err := imgCache.Start(ctx); err != nil {
+			log.Error(err, "failed to start image cache")
+			return err
+		}
+		return nil
+	})
 
 	g.Go(func() error {
 		setupLog.Info("Starting machine reconciler")

driver/networkinterfaceplugin/apinet.go

@@ -0,0 +1,48 @@
+// Copyright 2023 OnMetal authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package networkinterfaceplugin
+
+import (
+	"fmt"
+
+	virtletnetworkinterface "github.com/onmetal/libvirt-driver/pkg/plugins/networkinterface"
+	"github.com/onmetal/libvirt-driver/pkg/plugins/networkinterface/apinet"
+	"github.com/spf13/pflag"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+)
+
+type apinetOptions struct {
+	APInetNodeName string
+}
+
+func (o *apinetOptions) PluginName() string {
+	return "apinet"
+}
+
+func (o *apinetOptions) AddFlags(fs *pflag.FlagSet) {
+	fs.StringVar(&o.APInetNodeName, "apinet-node-name", "", "APInet node name")
+}
+
+func (o *apinetOptions) NetworkInterfacePlugin() (virtletnetworkinterface.Plugin, func(), error) {
+	if o.APInetNodeName == "" {
+		return nil, nil, fmt.Errorf("must specify apinet-node-name")
+	}
+
+	return apinet.NewPlugin(o.APInetNodeName), nil, nil
+}
+
+func init() {
+	utilruntime.Must(DefaultPluginTypeRegistry.Register(&apinetOptions{}, 1))
+}

driver/networkinterfaceplugin/isolated.go

@@ -0,0 +1,38 @@
+// Copyright 2022 OnMetal authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package networkinterfaceplugin
+
+import (
+	virtletnetworkinterface "github.com/onmetal/libvirt-driver/pkg/plugins/networkinterface"
+	"github.com/onmetal/libvirt-driver/pkg/plugins/networkinterface/isolated"
+	"github.com/spf13/pflag"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+)
+
+type isolatedOptions struct{}
+
+func (o *isolatedOptions) AddFlags(fs *pflag.FlagSet) {}
+
+func (o *isolatedOptions) PluginName() string {
+	return "isolated"
+}
+
+func (o *isolatedOptions) NetworkInterfacePlugin() (virtletnetworkinterface.Plugin, func(), error) {
+	return isolated.NewPlugin(), nil, nil
+}
+
+func init() {
+	utilruntime.Must(DefaultPluginTypeRegistry.Register(&isolatedOptions{}, 5))
+}