Skip to content

Commit

Permalink
Merge pull request #3 from ykulazhenkov/direct-configmap
Browse files Browse the repository at this point in the history 
Read configuration for the app directly from the k8s API
  • Loading branch information
@adrianchiris
adrianchiris authored Nov 23, 2023
2 parents a838261 + ebe66b0 commit 16ed235
Show file tree
Hide file tree
Showing 6 changed files with 155 additions and 96 deletions.
56 changes: 44 additions & 12 deletions README.md
View file
Original file line number Diff line number Diff line change
@@ -1,20 +1,29 @@
# network-operator-init-container
Init container for NVIDIA Network Operator

The network-operator-init-container container has two required command line arguments:
## Configuration
The network-operator-init-container container has following required command line arguments:

- `--config` path to the configuration file
- `--configmap-name` name of the configmap with configuration for the app
- `--configmap-namespace` namespace of the configmap with configuration for the app
- `--node-name` name of the k8s node on which this app runs

The configuration file should be in JSON format:
The ConfigMap should include configuration in JSON format:

```
{
"safeDriverLoad": {
"enable": true,
"annotation": "some-annotation"
}
}
apiVersion: v1
kind: ConfigMap
metadata:
name: ofed-init-container-config
namespace: default
data:
config.json: |-
{
"safeDriverLoad": {
"enable": true,
"annotation": "some-annotation"
}
}
```

- `safeDriverLoad` - contains settings related to safeDriverLoad feature
Expand All @@ -28,6 +37,25 @@ The container exits with code 0 when the annotation is removed from the Node obj

If `safeDriverLoad` feature is disabled then the container will immediately exit with code 0.

### Required permissions

```
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: network-operator-init-container
rules:
- apiGroups: [""]
resources: ["nodes"]
verbs: ["get", "list", "patch", "watch", "update"]
- apiGroups: [""]
resources: ["configmaps"]
verbs: ["get", "list"]
```

## Command line arguments

```
NVIDIA Network Operator init container
Expand All @@ -36,8 +64,12 @@ Usage:
Config flags:
--config string
path to the configuration file
--configmap-key string
key inside the configmap with configuration for the app (default "config.json")
--configmap-name string
name of the configmap with configuration for the app
--configmap-namespace string
namespace of the configmap with configuration for the app
--node-name string
name of the k8s node on which this app runs
Expand Down Expand Up @@ -70,4 +102,4 @@ Kubernetes flags:
--kubeconfig string
Paths to a kubeconfig. Only required if out-of-cluster.
```
```
38 changes: 25 additions & 13 deletions cmd/network-operator-init-container/app/app.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,18 +97,6 @@ func RunNetworkOperatorInitContainer(ctx context.Context, config *rest.Config, o
"Options", opts, "Version", version.GetVersionString())
ctrl.SetLogger(logger)

initContCfg, err := configPgk.FromFile(opts.ConfigPath)
if err != nil {
logger.Error(err, "failed to read configuration")
return err
}
logger.Info("network-operator-init-container configuration", "config", initContCfg.String())

if !initContCfg.SafeDriverLoad.Enable {
logger.Info("safe driver loading is disabled, exit")
return nil
}

mgr, err := ctrl.NewManager(config, ctrl.Options{
Metrics: metricsserver.Options{BindAddress: "0"},
Cache: cache.Options{
Expand All @@ -117,7 +105,7 @@ func RunNetworkOperatorInitContainer(ctx context.Context, config *rest.Config, o
fmt.Sprintf("metadata.name=%s", opts.NodeName))}}},
})
if err != nil {
logger.Error(err, "unable to start manager")
logger.Error(err, "unable to create manager")
return err
}

Expand All @@ -128,6 +116,30 @@ func RunNetworkOperatorInitContainer(ctx context.Context, config *rest.Config, o
return err
}

confConfigMap := &corev1.ConfigMap{}

err = k8sClient.Get(ctx, client.ObjectKey{
Name: opts.ConfigMapName,
Namespace: opts.ConfigMapNamespace,
}, confConfigMap)

if err != nil {
logger.Error(err, "failed to read config map with configuration")
return err
}

initContCfg, err := configPgk.Load(confConfigMap.Data[opts.ConfigMapKey])
if err != nil {
logger.Error(err, "failed to read configuration")
return err
}
logger.Info("network-operator-init-container configuration", "config", initContCfg.String())

if !initContCfg.SafeDriverLoad.Enable {
logger.Info("safe driver loading is disabled, exit")
return nil
}

errCh := make(chan error, 1)

if err = (&NodeReconciler{
Expand Down
82 changes: 52 additions & 30 deletions cmd/network-operator-init-container/app/app_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,13 +14,13 @@
package app_test

import (
"context"
"fmt"
"os"
"path/filepath"
"time"

. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
apiErrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/json"
"sigs.k8s.io/controller-runtime/pkg/client"
Expand All @@ -34,8 +34,11 @@ import (
)

const (
testNodeName = "node1"
testAnnotation = "foo.bar/spam"
testConfigMapName = "test"
testConfigMapNamespace = "default"
testConfigMapKey = "conf"
testNodeName = "node1"
testAnnotation = "foo.bar/spam"
)

func createNode(name string) *corev1.Node {
Expand All @@ -44,44 +47,67 @@ func createNode(name string) *corev1.Node {
return node
}

func createConfig(path string, cfg configPgk.Config) {
func createConfig(cfg configPgk.Config) {
data, err := json.Marshal(cfg)
ExpectWithOffset(1, err).NotTo(HaveOccurred())
ExpectWithOffset(1, os.WriteFile(path, data, 0x744))
err = k8sClient.Create(ctx, &corev1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{Name: testConfigMapName, Namespace: testConfigMapNamespace},
Data: map[string]string{testConfigMapKey: string(data)},
})
ExpectWithOffset(1, err).NotTo(HaveOccurred())
}

func newOpts() *options.Options {
return &options.Options{
ConfigMapName: testConfigMapName,
ConfigMapNamespace: testConfigMapNamespace,
ConfigMapKey: testConfigMapKey,
}
}

var _ = Describe("Init container", func() {
var (
configPath string
testCtx context.Context
testCFunc context.CancelFunc
)

BeforeEach(func() {
configPath = filepath.Join(GinkgoT().TempDir(), "config")
testCtx, testCFunc = context.WithCancel(ctx)
})

AfterEach(func() {
err := k8sClient.Delete(ctx, &corev1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{Name: testConfigMapName, Namespace: testConfigMapNamespace},
})
if !apiErrors.IsNotFound(err) {
Expect(err).NotTo(HaveOccurred())
}
testCFunc()
})
It("Succeed", func() {
testDone := make(chan interface{})
go func() {
defer close(testDone)
defer GinkgoRecover()
opts := options.New()
opts := newOpts()
opts.NodeName = testNodeName
opts.ConfigPath = configPath
createConfig(configPath, configPgk.Config{SafeDriverLoad: configPgk.SafeDriverLoadConfig{
createConfig(configPgk.Config{SafeDriverLoad: configPgk.SafeDriverLoadConfig{
Enable: true,
Annotation: testAnnotation,
}})
var err error
appExit := make(chan interface{})
go func() {
err = app.RunNetworkOperatorInitContainer(ctx, cfg, opts)
err = app.RunNetworkOperatorInitContainer(testCtx, cfg, opts)
close(appExit)
}()
node := &corev1.Node{}
Eventually(func(g Gomega) {
g.Expect(k8sClient.Get(ctx, types.NamespacedName{Name: testNodeName}, node)).NotTo(HaveOccurred())
g.Expect(k8sClient.Get(testCtx, types.NamespacedName{Name: testNodeName}, node)).NotTo(HaveOccurred())
g.Expect(node.GetAnnotations()[testAnnotation]).NotTo(BeEmpty())
}, 30, 1).Should(Succeed())
// remove annotation
Expect(k8sClient.Patch(ctx, node, client.RawPatch(
Expect(k8sClient.Patch(testCtx, node, client.RawPatch(
types.MergePatchType, []byte(
fmt.Sprintf(`{"metadata":{"annotations":{%q: null}}}`,
testAnnotation))))).NotTo(HaveOccurred())
Expand All @@ -95,17 +121,16 @@ var _ = Describe("Init container", func() {
go func() {
defer close(testDone)
defer GinkgoRecover()
opts := options.New()
opts := newOpts()
opts.NodeName = "unknown-node"
opts.ConfigPath = configPath
createConfig(configPath, configPgk.Config{SafeDriverLoad: configPgk.SafeDriverLoadConfig{
createConfig(configPgk.Config{SafeDriverLoad: configPgk.SafeDriverLoadConfig{
Enable: true,
Annotation: testAnnotation,
}})
var err error
appExit := make(chan interface{})
go func() {
err = app.RunNetworkOperatorInitContainer(ctx, cfg, opts)
err = app.RunNetworkOperatorInitContainer(testCtx, cfg, opts)
close(appExit)
}()
Eventually(appExit, 30, 1).Should(BeClosed())
Expand All @@ -118,20 +143,19 @@ var _ = Describe("Init container", func() {
go func() {
defer close(testDone)
defer GinkgoRecover()
opts := options.New()
opts := newOpts()
opts.NodeName = testNodeName
opts.ConfigPath = configPath
createConfig(configPath, configPgk.Config{SafeDriverLoad: configPgk.SafeDriverLoadConfig{
createConfig(configPgk.Config{SafeDriverLoad: configPgk.SafeDriverLoadConfig{
Enable: true,
Annotation: testAnnotation,
}})
var err error
appExit := make(chan interface{})
go func() {
err = app.RunNetworkOperatorInitContainer(ctx, cfg, opts)
err = app.RunNetworkOperatorInitContainer(testCtx, cfg, opts)
close(appExit)
}()
cFunc()
testCFunc()
Eventually(appExit, 30, 1).Should(BeClosed())
Expect(err).To(HaveOccurred())
}()
Expand All @@ -142,13 +166,12 @@ var _ = Describe("Init container", func() {
go func() {
defer close(testDone)
defer GinkgoRecover()
opts := options.New()
opts := newOpts()
opts.NodeName = "unknown-node"
opts.ConfigPath = configPath
var err error
appExit := make(chan interface{})
go func() {
err = app.RunNetworkOperatorInitContainer(ctx, cfg, opts)
err = app.RunNetworkOperatorInitContainer(testCtx, cfg, opts)
close(appExit)
}()
Eventually(appExit, 30, 1).Should(BeClosed())
Expand All @@ -161,16 +184,15 @@ var _ = Describe("Init container", func() {
go func() {
defer close(testDone)
defer GinkgoRecover()
opts := options.New()
opts := newOpts()
opts.NodeName = testNodeName
opts.ConfigPath = configPath
createConfig(configPath, configPgk.Config{SafeDriverLoad: configPgk.SafeDriverLoadConfig{
createConfig(configPgk.Config{SafeDriverLoad: configPgk.SafeDriverLoadConfig{
Enable: false,
}})
var err error
appExit := make(chan interface{})
go func() {
err = app.RunNetworkOperatorInitContainer(ctx, cfg, opts)
err = app.RunNetworkOperatorInitContainer(testCtx, cfg, opts)
close(appExit)
}()
Eventually(appExit, 30, 1).Should(BeClosed())
Expand Down
28 changes: 21 additions & 7 deletions cmd/network-operator-init-container/app/options/options.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,18 +32,24 @@ func New() *Options {

// Options contains application options
type Options struct {
NodeName string
ConfigPath string
LogConfig *logsapi.LoggingConfiguration
NodeName string
ConfigMapName string
ConfigMapNamespace string
ConfigMapKey string
LogConfig *logsapi.LoggingConfiguration
}

// AddNamedFlagSets returns FlagSet for Options
func (o *Options) AddNamedFlagSets(sharedFS *cliflag.NamedFlagSets) {
configFS := sharedFS.FlagSet("Config")
configFS.StringVar(&o.NodeName, "node-name", "",
"name of the k8s node on which this app runs")
configFS.StringVar(&o.ConfigPath, "config", "",
"path to the configuration file")
configFS.StringVar(&o.ConfigMapName, "configmap-name", "",
"name of the configmap with configuration for the app")
configFS.StringVar(&o.ConfigMapNamespace, "configmap-namespace", "",
"namespace of the configmap with configuration for the app")
configFS.StringVar(&o.ConfigMapKey, "configmap-key", "config.json",
"key inside the configmap with configuration for the app")

logFS := sharedFS.FlagSet("Logging")
logsapi.AddFlags(o.LogConfig, logFS)
Expand All @@ -68,8 +74,16 @@ func (o *Options) Validate() error {
return fmt.Errorf("node-name is required parameter")
}

if o.ConfigPath == "" {
return fmt.Errorf("config is required parameter")
if o.ConfigMapName == "" {
return fmt.Errorf("configmap-name is required parameter")
}

if o.ConfigMapNamespace == "" {
return fmt.Errorf("configmap-namespace is required parameter")
}

if o.ConfigMapKey == "" {
return fmt.Errorf("configmap-key is required parameter")
}

if err = logsapi.ValidateAndApply(o.LogConfig, nil); err != nil {
Expand Down
Loading

0 comments on commit 16ed235

Please sign in to comment.