Skip to content

Commit

Permalink
feat: Upgrade Karpenter APIs to v1 (#1406)
Browse files Browse the repository at this point in the history
  • Loading branch information
@rschalo
rschalo authored Jul 10, 2024
1 parent 15581ad commit c691dd4
Show file tree
Hide file tree
Showing 135 changed files with 6,396 additions and 6,426 deletions.
6 changes: 3 additions & 3 deletions kwok/apis/v1alpha1/labels.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ package v1alpha1

import (
"sigs.k8s.io/karpenter/kwok/apis"
"sigs.k8s.io/karpenter/pkg/apis/v1beta1"
v1 "sigs.k8s.io/karpenter/pkg/apis/v1"
)

const (
Expand All @@ -36,8 +36,8 @@ const (
)

func init() {
v1beta1.RestrictedLabelDomains = v1beta1.RestrictedLabelDomains.Insert(apis.Group)
v1beta1.WellKnownLabels = v1beta1.WellKnownLabels.Insert(
v1.RestrictedLabelDomains = v1.RestrictedLabelDomains.Insert(apis.Group)
v1.WellKnownLabels = v1.WellKnownLabels.Insert(
InstanceSizeLabelKey,
InstanceFamilyLabelKey,
InstanceCPULabelKey,
Expand Down
4 changes: 2 additions & 2 deletions kwok/charts/crds/karpenter.sh_nodeclaims.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -348,7 +348,7 @@ spec:
- spec
type: object
served: true
storage: false
storage: true
subresources:
status: {}
- additionalPrinterColumns:
Expand Down Expand Up @@ -792,6 +792,6 @@ spec:
- spec
type: object
served: true
storage: true
storage: false
subresources:
status: {}
4 changes: 2 additions & 2 deletions kwok/charts/crds/karpenter.sh_nodepools.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -486,7 +486,7 @@ spec:
- spec
type: object
served: true
storage: false
storage: true
subresources:
status: {}
- additionalPrinterColumns:
Expand Down Expand Up @@ -1068,6 +1068,6 @@ spec:
- spec
type: object
served: true
storage: true
storage: false
subresources:
status: {}
63 changes: 31 additions & 32 deletions kwok/cloudprovider/cloudprovider.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,14 +27,14 @@ import (

"github.com/docker/docker/pkg/namesgenerator"
"github.com/samber/lo"
v1 "k8s.io/api/core/v1"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"sigs.k8s.io/controller-runtime/pkg/client"

"sigs.k8s.io/karpenter/kwok/apis/v1alpha1"
"sigs.k8s.io/karpenter/pkg/apis/v1beta1"
v1 "sigs.k8s.io/karpenter/pkg/apis/v1"
"sigs.k8s.io/karpenter/pkg/cloudprovider"
"sigs.k8s.io/karpenter/pkg/scheduling"
)
Expand All @@ -51,7 +51,7 @@ type CloudProvider struct {
instanceTypes []*cloudprovider.InstanceType
}

func (c CloudProvider) Create(ctx context.Context, nodeClaim *v1beta1.NodeClaim) (*v1beta1.NodeClaim, error) {
func (c CloudProvider) Create(ctx context.Context, nodeClaim *v1.NodeClaim) (*v1.NodeClaim, error) {
// Create the Node because KwoK nodes don't have a kubelet, which is what Karpenter normally relies on to create the node.
node, err := c.toNode(nodeClaim)
if err != nil {
Expand All @@ -64,7 +64,7 @@ func (c CloudProvider) Create(ctx context.Context, nodeClaim *v1beta1.NodeClaim)
return c.toNodeClaim(node)
}

func (c CloudProvider) Delete(ctx context.Context, nodeClaim *v1beta1.NodeClaim) error {
func (c CloudProvider) Delete(ctx context.Context, nodeClaim *v1.NodeClaim) error {
if err := c.kubeClient.Delete(ctx, nodeClaim); err != nil {
if errors.IsNotFound(err) {
return fmt.Errorf("deleting node, %w", cloudprovider.NewNodeClaimNotFoundError(err))
Expand All @@ -74,9 +74,9 @@ func (c CloudProvider) Delete(ctx context.Context, nodeClaim *v1beta1.NodeClaim)
return nil
}

func (c CloudProvider) Get(ctx context.Context, providerID string) (*v1beta1.NodeClaim, error) {
func (c CloudProvider) Get(ctx context.Context, providerID string) (*v1.NodeClaim, error) {
nodeName := strings.Replace(providerID, kwokProviderPrefix, "", -1)
node := &v1.Node{}
node := &corev1.Node{}
if err := c.kubeClient.Get(ctx, types.NamespacedName{Name: nodeName}, node); err != nil {
if errors.IsNotFound(err) {
return nil, fmt.Errorf("finding node, %w", cloudprovider.NewNodeClaimNotFoundError(err))
Expand All @@ -89,12 +89,12 @@ func (c CloudProvider) Get(ctx context.Context, providerID string) (*v1beta1.Nod
return c.toNodeClaim(node)
}

func (c CloudProvider) List(ctx context.Context) ([]*v1beta1.NodeClaim, error) {
nodeList := &v1.NodeList{}
func (c CloudProvider) List(ctx context.Context) ([]*v1.NodeClaim, error) {
nodeList := &corev1.NodeList{}
if err := c.kubeClient.List(ctx, nodeList); err != nil {
return nil, fmt.Errorf("listing nodes, %w", err)
}
var nodeClaims []*v1beta1.NodeClaim
var nodeClaims []*v1.NodeClaim
for i, node := range nodeList.Items {
if !strings.HasPrefix(node.Spec.ProviderID, kwokProviderPrefix) {
continue
Expand All @@ -110,12 +110,12 @@ func (c CloudProvider) List(ctx context.Context) ([]*v1beta1.NodeClaim, error) {
}

// Return the hard-coded instance types.
func (c CloudProvider) GetInstanceTypes(ctx context.Context, nodePool *v1beta1.NodePool) ([]*cloudprovider.InstanceType, error) {
func (c CloudProvider) GetInstanceTypes(ctx context.Context, nodePool *v1.NodePool) ([]*cloudprovider.InstanceType, error) {
return c.instanceTypes, nil
}

// Return nothing since there's no cloud provider drift.
func (c CloudProvider) IsDrifted(ctx context.Context, nodeClaim *v1beta1.NodeClaim) (cloudprovider.DriftReason, error) {
func (c CloudProvider) IsDrifted(ctx context.Context, nodeClaim *v1.NodeClaim) (cloudprovider.DriftReason, error) {
return "", nil
}

Expand All @@ -137,14 +137,14 @@ func (c CloudProvider) getInstanceType(instanceTypeName string) (*cloudprovider.
return it, nil
}

func (c CloudProvider) toNode(nodeClaim *v1beta1.NodeClaim) (*v1.Node, error) {
func (c CloudProvider) toNode(nodeClaim *v1.NodeClaim) (*corev1.Node, error) {
newName := strings.Replace(namesgenerator.GetRandomName(0), "_", "-", -1)
//nolint
newName = fmt.Sprintf("%s-%d", newName, rand.Uint32())

requirements := scheduling.NewNodeSelectorRequirementsWithMinValues(nodeClaim.Spec.Requirements...)
req, found := lo.Find(nodeClaim.Spec.Requirements, func(req v1beta1.NodeSelectorRequirementWithMinValues) bool {
return req.Key == v1.LabelInstanceTypeStable
req, found := lo.Find(nodeClaim.Spec.Requirements, func(req v1.NodeSelectorRequirementWithMinValues) bool {
return req.Key == corev1.LabelInstanceTypeStable
})
if !found {
return nil, fmt.Errorf("instance type requirement not found")
Expand All @@ -169,25 +169,25 @@ func (c CloudProvider) toNode(nodeClaim *v1beta1.NodeClaim) (*v1.Node, error) {
}
}

return &v1.Node{
return &corev1.Node{
ObjectMeta: metav1.ObjectMeta{
Name: newName,
Labels: addInstanceLabels(nodeClaim.Labels, instanceType, nodeClaim, cheapestOffering),
Annotations: addKwokAnnotation(nodeClaim.Annotations),
},
Spec: v1.NodeSpec{
Spec: corev1.NodeSpec{
ProviderID: kwokProviderPrefix + newName,
Taints: []v1.Taint{v1beta1.UnregisteredNoExecuteTaint},
Taints: []corev1.Taint{v1.UnregisteredNoExecuteTaint},
},
Status: v1.NodeStatus{
Status: corev1.NodeStatus{
Capacity: instanceType.Capacity,
Allocatable: instanceType.Allocatable(),
Phase: v1.NodePending,
Phase: corev1.NodePending,
},
}, nil
}

func addInstanceLabels(labels map[string]string, instanceType *cloudprovider.InstanceType, nodeClaim *v1beta1.NodeClaim, offering *cloudprovider.Offering) map[string]string {
func addInstanceLabels(labels map[string]string, instanceType *cloudprovider.InstanceType, nodeClaim *v1.NodeClaim, offering *cloudprovider.Offering) map[string]string {
ret := make(map[string]string, len(labels))
// start with labels on the nodeclaim
for k, v := range labels {
Expand All @@ -196,15 +196,15 @@ func addInstanceLabels(labels map[string]string, instanceType *cloudprovider.Ins

// add the derived nodeclaim requirement labels
for _, r := range nodeClaim.Spec.Requirements {
if len(r.Values) == 1 && r.Operator == v1.NodeSelectorOpIn {
if len(r.Values) == 1 && r.Operator == corev1.NodeSelectorOpIn {
ret[r.Key] = r.Values[0]
}
}

// ensure we have an instance type and then any instance type requirements
ret[v1.LabelInstanceTypeStable] = instanceType.Name
ret[corev1.LabelInstanceTypeStable] = instanceType.Name
for _, r := range instanceType.Requirements {
if r.Len() == 1 && r.Operator() == v1.NodeSelectorOpIn {
if r.Len() == 1 && r.Operator() == corev1.NodeSelectorOpIn {
ret[r.Key] = r.Values()[0]
}
}
Expand All @@ -214,9 +214,9 @@ func addInstanceLabels(labels map[string]string, instanceType *cloudprovider.Ins
// Randomly add each new node to one of the pre-created kwokPartitions.

ret[v1alpha1.KwokPartitionLabelKey] = lo.Sample(kwokPartitions)
ret[v1beta1.CapacityTypeLabelKey] = offering.Requirements.Get(v1beta1.CapacityTypeLabelKey).Any()
ret[v1.LabelTopologyZone] = offering.Requirements.Get(v1.LabelTopologyZone).Any()
ret[v1.LabelHostname] = nodeClaim.Name
ret[v1.CapacityTypeLabelKey] = offering.Requirements.Get(v1.CapacityTypeLabelKey).Any()
ret[corev1.LabelTopologyZone] = offering.Requirements.Get(corev1.LabelTopologyZone).Any()
ret[corev1.LabelHostname] = nodeClaim.Name

ret[v1alpha1.KwokLabelKey] = v1alpha1.KwokLabelValue
return ret
Expand All @@ -231,22 +231,21 @@ func addKwokAnnotation(annotations map[string]string) map[string]string {
return ret
}

func (c CloudProvider) toNodeClaim(node *v1.Node) (*v1beta1.NodeClaim, error) {
return &v1beta1.NodeClaim{
func (c CloudProvider) toNodeClaim(node *corev1.Node) (*v1.NodeClaim, error) {
return &v1.NodeClaim{
ObjectMeta: metav1.ObjectMeta{
Name: node.Name,
Labels: node.Labels,
Annotations: addKwokAnnotation(node.Annotations),
},
Spec: v1beta1.NodeClaimSpec{
Spec: v1.NodeClaimSpec{
Taints: nil,
StartupTaints: nil,
Requirements: nil,
Resources: v1beta1.ResourceRequirements{},
Kubelet: nil,
Resources: v1.ResourceRequirements{},
NodeClassRef: nil,
},
Status: v1beta1.NodeClaimStatus{
Status: v1.NodeClaimStatus{
NodeName: node.Name,
ProviderID: node.Spec.ProviderID,
Capacity: node.Status.Capacity,
Expand Down
60 changes: 30 additions & 30 deletions kwok/cloudprovider/helpers.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,11 +23,11 @@ import (
"regexp"

"github.com/samber/lo"
v1 "k8s.io/api/core/v1"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"

"sigs.k8s.io/karpenter/kwok/apis/v1alpha1"
"sigs.k8s.io/karpenter/pkg/apis/v1beta1"
v1 "sigs.k8s.io/karpenter/pkg/apis/v1"
"sigs.k8s.io/karpenter/pkg/cloudprovider"
"sigs.k8s.io/karpenter/pkg/scheduling"
)
Expand All @@ -44,15 +44,15 @@ type KWOKOfferings []KWOKOffering

type KWOKOffering struct {
cloudprovider.Offering
Requirements []v1.NodeSelectorRequirement
Requirements []corev1.NodeSelectorRequirement
}

type InstanceTypeOptions struct {
Name string `json:"name"`
Offerings KWOKOfferings `json:"offerings"`
Architecture string `json:"architecture"`
OperatingSystems []v1.OSName `json:"operatingSystems"`
Resources v1.ResourceList `json:"resources"`
Name string `json:"name"`
Offerings KWOKOfferings `json:"offerings"`
Architecture string `json:"architecture"`
OperatingSystems []corev1.OSName `json:"operatingSystems"`
Resources corev1.ResourceList `json:"resources"`

// These are used for setting default requirements, they should not be used
// for setting arbitrary node labels. Set the labels on the created NodePool for
Expand Down Expand Up @@ -114,9 +114,9 @@ func setDefaultOptions(opts InstanceTypeOptions) InstanceTypeOptions {
var cpu, memory string
for res, q := range opts.Resources {
switch res {
case v1.ResourceCPU:
case corev1.ResourceCPU:
cpu = q.String()
case v1.ResourceMemory:
case corev1.ResourceMemory:
memory = q.String()
}
}
Expand All @@ -129,8 +129,8 @@ func setDefaultOptions(opts InstanceTypeOptions) InstanceTypeOptions {
}

// if the user specified a different pod limit, override the default
opts.Resources = lo.Assign(v1.ResourceList{
v1.ResourcePods: resource.MustParse("110"), // Default number of pods on a node in Kubernetes
opts.Resources = lo.Assign(corev1.ResourceList{
corev1.ResourcePods: resource.MustParse("110"), // Default number of pods on a node in Kubernetes
}, opts.Resources)

// make sure all the instance types are available
Expand All @@ -142,39 +142,39 @@ func setDefaultOptions(opts InstanceTypeOptions) InstanceTypeOptions {
}

func newInstanceType(options InstanceTypeOptions) *cloudprovider.InstanceType {
osNames := lo.Map(options.OperatingSystems, func(os v1.OSName, _ int) string { return string(os) })
osNames := lo.Map(options.OperatingSystems, func(os corev1.OSName, _ int) string { return string(os) })

zones := lo.Uniq(lo.Flatten(lo.Map(options.Offerings, func(o KWOKOffering, _ int) []string {
req, _ := lo.Find(o.Requirements, func(req v1.NodeSelectorRequirement) bool {
return req.Key == v1.LabelTopologyZone
req, _ := lo.Find(o.Requirements, func(req corev1.NodeSelectorRequirement) bool {
return req.Key == corev1.LabelTopologyZone
})
return req.Values
})))
capacityTypes := lo.Uniq(lo.Flatten(lo.Map(options.Offerings, func(o KWOKOffering, _ int) []string {
req, _ := lo.Find(o.Requirements, func(req v1.NodeSelectorRequirement) bool {
return req.Key == v1beta1.CapacityTypeLabelKey
req, _ := lo.Find(o.Requirements, func(req corev1.NodeSelectorRequirement) bool {
return req.Key == v1.CapacityTypeLabelKey
})
return req.Values
})))

requirements := scheduling.NewRequirements(
scheduling.NewRequirement(v1.LabelInstanceTypeStable, v1.NodeSelectorOpIn, options.Name),
scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, options.Architecture),
scheduling.NewRequirement(v1.LabelOSStable, v1.NodeSelectorOpIn, osNames...),
scheduling.NewRequirement(v1.LabelTopologyZone, v1.NodeSelectorOpIn, zones...),
scheduling.NewRequirement(v1beta1.CapacityTypeLabelKey, v1.NodeSelectorOpIn, capacityTypes...),
scheduling.NewRequirement(v1alpha1.InstanceSizeLabelKey, v1.NodeSelectorOpIn, options.instanceTypeLabels[v1alpha1.InstanceSizeLabelKey]),
scheduling.NewRequirement(v1alpha1.InstanceFamilyLabelKey, v1.NodeSelectorOpIn, options.instanceTypeLabels[v1alpha1.InstanceFamilyLabelKey]),
scheduling.NewRequirement(v1alpha1.InstanceCPULabelKey, v1.NodeSelectorOpIn, options.instanceTypeLabels[v1alpha1.InstanceCPULabelKey]),
scheduling.NewRequirement(v1alpha1.InstanceMemoryLabelKey, v1.NodeSelectorOpIn, options.instanceTypeLabels[v1alpha1.InstanceMemoryLabelKey]),
scheduling.NewRequirement(corev1.LabelInstanceTypeStable, corev1.NodeSelectorOpIn, options.Name),
scheduling.NewRequirement(corev1.LabelArchStable, corev1.NodeSelectorOpIn, options.Architecture),
scheduling.NewRequirement(corev1.LabelOSStable, corev1.NodeSelectorOpIn, osNames...),
scheduling.NewRequirement(corev1.LabelTopologyZone, corev1.NodeSelectorOpIn, zones...),
scheduling.NewRequirement(v1.CapacityTypeLabelKey, corev1.NodeSelectorOpIn, capacityTypes...),
scheduling.NewRequirement(v1alpha1.InstanceSizeLabelKey, corev1.NodeSelectorOpIn, options.instanceTypeLabels[v1alpha1.InstanceSizeLabelKey]),
scheduling.NewRequirement(v1alpha1.InstanceFamilyLabelKey, corev1.NodeSelectorOpIn, options.instanceTypeLabels[v1alpha1.InstanceFamilyLabelKey]),
scheduling.NewRequirement(v1alpha1.InstanceCPULabelKey, corev1.NodeSelectorOpIn, options.instanceTypeLabels[v1alpha1.InstanceCPULabelKey]),
scheduling.NewRequirement(v1alpha1.InstanceMemoryLabelKey, corev1.NodeSelectorOpIn, options.instanceTypeLabels[v1alpha1.InstanceMemoryLabelKey]),
)

return &cloudprovider.InstanceType{
Name: options.Name,
Requirements: requirements,
Offerings: lo.Map(options.Offerings, func(off KWOKOffering, _ int) cloudprovider.Offering {
return cloudprovider.Offering{
Requirements: scheduling.NewRequirements(lo.Map(off.Requirements, func(req v1.NodeSelectorRequirement, _ int) *scheduling.Requirement {
Requirements: scheduling.NewRequirements(lo.Map(off.Requirements, func(req corev1.NodeSelectorRequirement, _ int) *scheduling.Requirement {
return scheduling.NewRequirement(req.Key, req.Operator, req.Values...)
})...),
Price: off.Offering.Price,
Expand All @@ -183,9 +183,9 @@ func newInstanceType(options InstanceTypeOptions) *cloudprovider.InstanceType {
}),
Capacity: options.Resources,
Overhead: &cloudprovider.InstanceTypeOverhead{
KubeReserved: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("100m"),
v1.ResourceMemory: resource.MustParse("10Mi"),
KubeReserved: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("100m"),
corev1.ResourceMemory: resource.MustParse("10Mi"),
},
},
}
Expand Down
Loading

0 comments on commit c691dd4

Please sign in to comment.