Skip to content

Commit

Permalink
feat: add scheduling logic to zarf injector (#1731)
Browse files Browse the repository at this point in the history
## Description

Checks to ensure that a node does not have a `NoSchedule` taint before
choosing that node/image for the Zarf injector pod

## Related Issue

Fixes #1730
Fixes #905

## Type of change

- [x] Bug fix (non-breaking change which fixes an issue)
- [ ] New feature (non-breaking change which adds functionality)
- [ ] Other (security config, docs update, etc)

## Checklist before merging

- [ ] Test, docs, adr added or updated as needed
- [ ] [Contributor Guide
Steps](https://github.com/defenseunicorns/zarf/blob/main/CONTRIBUTING.md#developer-workflow)
followed


Gave this a WAG. It seems to work locally, but am unsure if this is the
direction you guys want to go. Let me know!
  • Loading branch information
ragingpastry authored May 18, 2023
1 parent 78a3cb5 commit d7e0cfd
Show file tree
Hide file tree
Showing 3 changed files with 30 additions and 9 deletions.
7 changes: 5 additions & 2 deletions src/internal/cluster/injector.go
Original file line number Diff line number Diff line change
Expand Up @@ -42,10 +42,13 @@ func (c *Cluster) StartInjectionMadness(tempPath types.TempPaths, injectorSeedTa
var seedImages []transform.Image

// Get all the images from the cluster
spinner.Updatef("Getting the list of existing cluster images")
if images, err = c.Kube.GetAllImages(); err != nil {
timeout := 5 * time.Minute
spinner.Updatef("Getting the list of existing cluster images (%s timeout)", timeout.String())
if images, err = c.Kube.GetAllImages(timeout); err != nil {
spinner.Fatalf(err, "Unable to generate a list of candidate images to perform the registry injection")
}
message.Debugf("Found %d images in the cluster", len(images))
message.Debugf("Images: %#v", images)

spinner.Updatef("Creating the injector configmap")
if err = c.createInjectorConfigmap(tempPath); err != nil {
Expand Down
26 changes: 19 additions & 7 deletions src/pkg/k8s/images.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,8 @@ type ImageMap map[string]bool
type ImageNodeMap map[string][]string

// GetAllImages returns a list of images and their nodes found in pods in the cluster.
func (k *K8s) GetAllImages() (ImageNodeMap, error) {
timeout := time.After(5 * time.Minute)
func (k *K8s) GetAllImages(timeoutDuration time.Duration) (ImageNodeMap, error) {
timeout := time.After(timeoutDuration)

for {
// Delay check 2 seconds.
Expand All @@ -44,7 +44,8 @@ func (k *K8s) GetAllImages() (ImageNodeMap, error) {
}
}

// GetImagesWithNodes returns all images and their nodes in a given namespace.
// GetImagesWithNodes checks for images on schedulable nodes and returns
// a map of these images and their nodes in a given namespace.
func (k *K8s) GetImagesWithNodes(namespace string) (ImageNodeMap, error) {
result := make(ImageNodeMap)

Expand All @@ -53,16 +54,27 @@ func (k *K8s) GetImagesWithNodes(namespace string) (ImageNodeMap, error) {
return nil, fmt.Errorf("unable to get the list of pods in the cluster")
}

findImages:
for _, pod := range pods.Items {
node := pod.Spec.NodeName
nodeName := pod.Spec.NodeName
nodeDetails, err := k.GetNode(nodeName)
if err != nil {
return nil, fmt.Errorf("unable to get the node %s", pod.Spec.NodeName)
}

for _, taint := range nodeDetails.Spec.Taints {
if (taint.Effect == corev1.TaintEffectNoSchedule || taint.Effect == corev1.TaintEffectNoExecute) {
continue findImages
}
}
for _, container := range pod.Spec.InitContainers {
result[container.Image] = append(result[container.Image], node)
result[container.Image] = append(result[container.Image], nodeName)
}
for _, container := range pod.Spec.Containers {
result[container.Image] = append(result[container.Image], node)
result[container.Image] = append(result[container.Image], nodeName)
}
for _, container := range pod.Spec.EphemeralContainers {
result[container.Image] = append(result[container.Image], node)
result[container.Image] = append(result[container.Image], nodeName)
}
}

Expand Down
6 changes: 6 additions & 0 deletions src/pkg/k8s/nodes.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,3 +16,9 @@ func (k *K8s) GetNodes() (*corev1.NodeList, error) {
metaOptions := metav1.ListOptions{}
return k.Clientset.CoreV1().Nodes().List(context.TODO(), metaOptions)
}

// GetNode returns a node from the k8s cluster.
func (k *K8s) GetNode(nodeName string) (*corev1.Node, error) {
return k.Clientset.CoreV1().Nodes().Get(context.TODO(), nodeName, metav1.GetOptions{})
}

0 comments on commit d7e0cfd

Please sign in to comment.