Skip to content

Commit

Permalink
feat: allow fqdn to be used when registering k8s node
Browse files Browse the repository at this point in the history
This PR fixes a problem we had with AWS clusters. We now allow the
kubelet to register using the full fqdn instead of just hostname.

Signed-off-by: Spencer Smith <robertspencersmith@gmail.com>
  • Loading branch information
rsmitty committed Feb 1, 2021
1 parent 87ccf0e commit e4e6da3
Show file tree
Hide file tree
Showing 9 changed files with 116 additions and 11 deletions.
1 change: 1 addition & 0 deletions internal/app/machined/pkg/runtime/runtime.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,4 +16,5 @@ type Runtime interface {
State() State
Events() EventStream
Logging() LoggingManager
NodeName() (string, error)
}
45 changes: 45 additions & 0 deletions internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_runtime.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,9 @@ package v1alpha1

import (
"fmt"
"log"
"os"
"syscall"

"github.com/talos-systems/talos/internal/app/machined/pkg/runtime"
"github.com/talos-systems/talos/pkg/machinery/config"
Expand Down Expand Up @@ -75,3 +78,45 @@ func (r *Runtime) Events() runtime.EventStream {
func (r *Runtime) Logging() runtime.LoggingManager {
return r.l
}

// NodeName implements the Runtime interface.
func (r *Runtime) NodeName() (string, error) {
// attempt to fetch hostname and domain name via syscalls and concat them if necessary
if r.Config().Machine().Kubelet().RegisterWithFQDN() {
var utsName syscall.Utsname
if err := syscall.Uname(&utsName); err != nil {
return "", err
}

nodeName := utsNameVarToString(utsName.Nodename)
log.Printf("Nodename is %s", nodeName)

domainName := utsNameVarToString(utsName.Domainname)
log.Printf("Domain name is %s", domainName)

// As odd as it looks, the Uname method sets domainName to this "(none)" string if not set.
if domainName != "(none)" {
return fmt.Sprintf("%s.%s", nodeName, domainName), nil
}

return nodeName, nil
}

// default to os.Hostname if we don't need to worry about fqdn.
return os.Hostname()
}

// converts int8 array to a string
// borrowed from https://github.com/aisola/go-coreutils/blob/master/uname/uname.go#L98
func utsNameVarToString(unameArray [65]int8) string {
var byteString [65]byte

var indexLength int

for unameArray[indexLength] != 0 {
byteString[indexLength] = uint8(unameArray[indexLength])
indexLength++
}

return string(byteString[:indexLength])
}
Original file line number Diff line number Diff line change
Expand Up @@ -1224,9 +1224,9 @@ func UnmountSystemDiskBindMounts(seq runtime.Sequence, data interface{}) (runtim
//nolint: dupl
func CordonAndDrainNode(seq runtime.Sequence, data interface{}) (runtime.TaskExecutionFunc, string) {
return func(ctx context.Context, logger *log.Logger, r runtime.Runtime) (err error) {
var hostname string
var nodename string

if hostname, err = os.Hostname(); err != nil {
if nodename, err = r.NodeName(); err != nil {
return err
}

Expand All @@ -1236,7 +1236,7 @@ func CordonAndDrainNode(seq runtime.Sequence, data interface{}) (runtime.TaskExe
return err
}

if err = kubeHelper.CordonAndDrain(hostname); err != nil {
if err = kubeHelper.CordonAndDrain(nodename); err != nil {
return err
}

Expand All @@ -1251,9 +1251,9 @@ func CordonAndDrainNode(seq runtime.Sequence, data interface{}) (runtime.TaskExe
//nolint: dupl
func UncordonNode(seq runtime.Sequence, data interface{}) (runtime.TaskExecutionFunc, string) {
return func(ctx context.Context, logger *log.Logger, r runtime.Runtime) (err error) {
var hostname string
var nodename string

if hostname, err = os.Hostname(); err != nil {
if nodename, err = r.NodeName(); err != nil {
return err
}

Expand All @@ -1263,11 +1263,11 @@ func UncordonNode(seq runtime.Sequence, data interface{}) (runtime.TaskExecution
return err
}

if err = kubeHelper.WaitUntilReady(hostname); err != nil {
if err = kubeHelper.WaitUntilReady(nodename); err != nil {
return err
}

if err = kubeHelper.Uncordon(hostname, false); err != nil {
if err = kubeHelper.Uncordon(nodename, false); err != nil {
return err
}

Expand Down Expand Up @@ -1475,13 +1475,14 @@ func LabelNodeAsMaster(seq runtime.Sequence, data interface{}) (runtime.TaskExec
return err
}

hostname, err := os.Hostname()
if err != nil {
var nodename string

if nodename, err = r.NodeName(); err != nil {
return err
}

err = retry.Constant(constants.NodeReadyTimeout, retry.WithUnits(3*time.Second), retry.WithErrorLogging(true)).Retry(func() error {
if err = h.LabelNodeAsMaster(hostname, !r.Config().Cluster().ScheduleOnMasters()); err != nil {
if err = h.LabelNodeAsMaster(nodename, !r.Config().Cluster().ScheduleOnMasters()); err != nil {
return retry.ExpectedError(err)
}

Expand Down
7 changes: 7 additions & 0 deletions internal/app/machined/pkg/system/services/kubelet.go
Original file line number Diff line number Diff line change
Expand Up @@ -267,6 +267,11 @@ func newKubeletConfiguration(clusterDNS []string, dnsDomain string) *kubeletconf
}

func (k *Kubelet) args(r runtime.Runtime) ([]string, error) {
nodename, err := r.NodeName()
if err != nil {
return nil, err
}

denyListArgs := argsbuilder.Args{
"bootstrap-kubeconfig": constants.KubeletBootstrapKubeconfig,
"kubeconfig": constants.KubeletKubeconfig,
Expand All @@ -277,6 +282,8 @@ func (k *Kubelet) args(r runtime.Runtime) ([]string, error) {

"cert-dir": constants.KubeletPKIDir,
"cni-conf-dir": cni.DefaultNetDir,

"hostname-override": nodename,
}

extraArgs := argsbuilder.Args(r.Config().Machine().Kubelet().ExtraArgs())
Expand Down
1 change: 1 addition & 0 deletions pkg/machinery/config/provider.go
Original file line number Diff line number Diff line change
Expand Up @@ -200,6 +200,7 @@ type Kubelet interface {
Image() string
ExtraArgs() map[string]string
ExtraMounts() []specs.Mount
RegisterWithFQDN() bool
}

// Registries defines the configuration for image fetching.
Expand Down
5 changes: 5 additions & 0 deletions pkg/machinery/config/types/v1alpha1/v1alpha1_provider.go
Original file line number Diff line number Diff line change
Expand Up @@ -262,6 +262,11 @@ func (k *KubeletConfig) ExtraMounts() []specs.Mount {
return k.KubeletExtraMounts
}

// RegisterWithFQDN implements the config.Provider interface.
func (k *KubeletConfig) RegisterWithFQDN() bool {
return k.KubeletRegisterWithFQDN
}

// Name implements the config.Provider interface.
func (c *ClusterConfig) Name() string {
return c.ClusterName
Expand Down
9 changes: 9 additions & 0 deletions pkg/machinery/config/types/v1alpha1/v1alpha1_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -677,6 +677,15 @@ type KubeletConfig struct {
// examples:
// - value: kubeletExtraMountsExample
KubeletExtraMounts []specs.Mount `yaml:"extraMounts,omitempty"`
// description: |
// The `registerWithFQDN` field is used to force kubelet to use the node FQDN for registration.
// This is required in clouds like AWS.
// values:
// - true
// - yes
// - false
// - no
KubeletRegisterWithFQDN bool `yaml:"registerWithFQDN,omitempty"`
}

// NetworkConfig represents the machine's networking config values.
Expand Down
13 changes: 12 additions & 1 deletion pkg/machinery/config/types/v1alpha1/v1alpha1_types_doc.go
Original file line number Diff line number Diff line change
Expand Up @@ -379,7 +379,7 @@ func init() {
FieldName: "kubelet",
},
}
KubeletConfigDoc.Fields = make([]encoder.Doc, 3)
KubeletConfigDoc.Fields = make([]encoder.Doc, 4)
KubeletConfigDoc.Fields[0].Name = "image"
KubeletConfigDoc.Fields[0].Type = "string"
KubeletConfigDoc.Fields[0].Note = ""
Expand All @@ -403,6 +403,17 @@ func init() {
KubeletConfigDoc.Fields[2].Comments[encoder.LineComment] = "The `extraMounts` field is used to add additional mounts to the kubelet container."

KubeletConfigDoc.Fields[2].AddExample("", kubeletExtraMountsExample)
KubeletConfigDoc.Fields[3].Name = "registerWithFQDN"
KubeletConfigDoc.Fields[3].Type = "bool"
KubeletConfigDoc.Fields[3].Note = ""
KubeletConfigDoc.Fields[3].Description = "The `registerWithFQDN` field is used to force kubelet to use the node FQDN for registration.\nThis is required in clouds like AWS."
KubeletConfigDoc.Fields[3].Comments[encoder.LineComment] = "The `registerWithFQDN` field is used to force kubelet to use the node FQDN for registration."
KubeletConfigDoc.Fields[3].Values = []string{
"true",
"yes",
"false",
"no",
}

NetworkConfigDoc.Type = "NetworkConfig"
NetworkConfigDoc.Comments[encoder.LineComment] = "NetworkConfig represents the machine's networking config values."
Expand Down
25 changes: 25 additions & 0 deletions website/content/docs/v0.9/Reference/configuration.md
Original file line number Diff line number Diff line change
Expand Up @@ -1286,6 +1286,31 @@ extraMounts:

<hr />

<div class="dd">

<code>registerWithFQDN</code> <i>bool</i>

</div>
<div class="dt">

The `registerWithFQDN` field is used to force kubelet to use the node FQDN for registration.
This is required in clouds like AWS.


Valid values:


- <code>true</code>

- <code>yes</code>

- <code>false</code>

- <code>no</code>
</div>

<hr />




Expand Down

0 comments on commit e4e6da3

Please sign in to comment.