feat: provide Talos machine config with field placeholders and docs

For config patching, it's beneficial to have some fields to be present
in the config, even with the default (empty) value. At the same time not
all fields should be present in all versions of the config, e.g. some
config value don't apply to worker node configuration.

Empty value and nil value are treated equal by `yaml` library, but Talos
encoder can be made more smart to still output empty (non-nil) value to
the config, while skipping completely nil fields.

This PR implements that via new `talos:"omitonlyifnil"` tag and plus
moves docs for such fields into comments under the value.

GC'ed pod checkpointer config, so it doesn't get generated even as
comments (it was empty by default even for 0.8, so this just removes
comments about it).

Signed-off-by: Andrey Smirnov <smirnov.andrey@gmail.com>

cmd/talosctl/cmd/talos/images.go

@@ -30,7 +30,6 @@ var imagesCmd = &cobra.Command{
 				SchedulerConfig:         &v1alpha1.SchedulerConfig{},
 				CoreDNSConfig:           &v1alpha1.CoreDNS{},
 				ProxyConfig:             &v1alpha1.ProxyConfig{},
-				PodCheckpointerConfig:   &v1alpha1.PodCheckpointer{},
 			},
 		})
 

pkg/machinery/config/encoder/documentation.go

@@ -206,7 +206,7 @@ func renderExample(key string, doc *Doc, flags CommentsFlags) string {
 	for i, e := range doc.Examples {
 		v := reflect.ValueOf(e.GetValue())
 
-		if !isSet(v) {
+		if isEmpty(v) {
 			continue
 		}
 
@@ -223,11 +223,13 @@ func renderExample(key string, doc *Doc, flags CommentsFlags) string {
 			continue
 		}
 
-		node, err = toYamlNode(map[string]*yaml.Node{
-			key: node,
-		}, flags)
-		if err != nil {
-			continue
+		if key != "" {
+			node, err = toYamlNode(map[string]*yaml.Node{
+				key: node,
+			}, flags)
+			if err != nil {
+				continue
+			}
 		}
 
 		if i == 0 && flags.enabled(CommentsDocs) {
@@ -253,14 +255,15 @@ func renderExample(key string, doc *Doc, flags CommentsFlags) string {
 			continue
 		}
 
-		var example string
-
-		// don't collapse comment
-		re := regexp.MustCompile(`(?m)^#`)
-		data = re.ReplaceAll(data, []byte("# #"))
+		if key == "" {
+			// re-indent
+			data = regexp.MustCompile(`(?m)^(.)`).ReplaceAll(data, []byte("  $1"))
+		} else {
+			// don't collapse comment
+			data = regexp.MustCompile(`(?m)^#`).ReplaceAll(data, []byte("# #"))
+		}
 
-		example += string(data)
-		examples = append(examples, example)
+		examples = append(examples, string(data))
 	}
 
 	return strings.Join(examples, "")
@@ -277,7 +280,7 @@ func getExample(v reflect.Value, doc *Doc, index int) *reflect.Value {
 	}
 
 	defaultValue := reflect.ValueOf(doc.Examples[index].GetValue())
-	if isSet(defaultValue) {
+	if !isEmpty(defaultValue) {
 		if v.Kind() != reflect.Ptr && defaultValue.Kind() == reflect.Ptr {
 			defaultValue = defaultValue.Elem()
 		}

pkg/machinery/config/encoder/encoder.go

@@ -75,21 +75,35 @@ func (e *Encoder) Encode() ([]byte, error) {
 	return yaml.Marshal(node)
 }
 
-func isSet(value reflect.Value) bool {
+func isEmpty(value reflect.Value) bool {
 	if !value.IsValid() {
-		return false
+		return true
 	}
 
 	//nolint:exhaustive
 	switch value.Kind() {
 	case reflect.Ptr:
-		return !value.IsNil()
+		return value.IsNil()
 	case reflect.Map:
-		return len(value.MapKeys()) != 0
+		return len(value.MapKeys()) == 0
 	case reflect.Slice:
-		return value.Len() > 0
+		return value.Len() == 0
 	default:
-		return !value.IsZero()
+		return value.IsZero()
+	}
+}
+
+func isNil(value reflect.Value) bool {
+	if !value.IsValid() {
+		return true
+	}
+
+	//nolint:exhaustive
+	switch value.Kind() {
+	case reflect.Ptr, reflect.Map, reflect.Slice:
+		return value.IsNil()
+	default:
+		return false
 	}
 }
 
@@ -142,6 +156,12 @@ func toYamlNode(in interface{}, flags CommentsFlags) (*yaml.Node, error) {
 			tag := t.Field(i).Tag.Get("yaml")
 			parts := strings.Split(tag, ",")
 			fieldName := parts[0]
+			parts = parts[1:]
+
+			tag = t.Field(i).Tag.Get("talos")
+			if tag != "" {
+				parts = append(parts, strings.Split(tag, ",")...)
+			}
 
 			if fieldName == "" {
 				fieldName = strings.ToLower(t.Field(i).Name)
@@ -152,21 +172,21 @@ func toYamlNode(in interface{}, flags CommentsFlags) (*yaml.Node, error) {
 			}
 
 			var (
-				defined = isSet(v.Field(i))
+				empty = isEmpty(v.Field(i))
+				null  = isNil(v.Field(i))
 
 				skip   bool
 				inline bool
 				flow   bool
 			)
 
-			for i, part := range parts {
-				// always skip the first argument
-				if i == 0 {
-					continue
+			for _, part := range parts {
+				if part == "omitempty" && empty {
+					skip = true
 				}
 
-				if part == "omitempty" && !defined {
-					skip = true
+				if part == "omitonlyifnil" && !null {
+					skip = false
 				}
 
 				if part == "inline" {
@@ -192,33 +212,56 @@ func toYamlNode(in interface{}, flags CommentsFlags) (*yaml.Node, error) {
 				fieldDoc = getDoc(value)
 			}
 
-			if !defined && flags.enabled(CommentsExamples) {
-				example := renderExample(fieldName, fieldDoc, flags)
+			// inlineExample is rendered after the value
+			var inlineExample string
 
-				if example != "" {
-					examples = append(examples, example)
-					skip = true
+			if empty && flags.enabled(CommentsExamples) && fieldDoc != nil {
+				if skip {
+					// render example to be appended to the end of the rendered struct
+					example := renderExample(fieldName, fieldDoc, flags)
+
+					if example != "" {
+						examples = append(examples, example)
+					}
+				} else {
+					// render example to be appended to the empty field
+					fieldDocCopy := *fieldDoc
+					fieldDocCopy.Comments = [3]string{}
+
+					inlineExample = renderExample("", &fieldDocCopy, flags)
 				}
 			}
 
+			if skip {
+				continue
+			}
+
 			var style yaml.Style
 			if flow {
 				style |= yaml.FlowStyle
 			}
 
-			if !skip {
-				if inline {
-					child, err := toYamlNode(value, flags)
-					if err != nil {
-						return nil, err
-					}
-
-					if child.Kind == yaml.MappingNode || child.Kind == yaml.SequenceNode {
-						appendNodes(node, child.Content...)
-					}
-				} else if err := addToMap(node, fieldDoc, fieldName, value, style, flags); err != nil {
+			if inline {
+				child, err := toYamlNode(value, flags)
+				if err != nil {
 					return nil, err
 				}
+
+				if child.Kind == yaml.MappingNode || child.Kind == yaml.SequenceNode {
+					appendNodes(node, child.Content...)
+				}
+			} else if err := addToMap(node, fieldDoc, fieldName, value, style, flags); err != nil {
+				return nil, err
+			}
+
+			if inlineExample != "" {
+				nodeToAttach := node.Content[len(node.Content)-1]
+
+				if nodeToAttach.FootComment != "" {
+					nodeToAttach.FootComment += "\n"
+				}
+
+				nodeToAttach.FootComment += inlineExample
 			}
 		}
 

pkg/machinery/config/encoder/encoder_test.go

@@ -27,6 +27,8 @@ type Config struct {
 	CustomMarshaller *WithCustomMarshaller `yaml:",omitempty"`
 	Bytes            []byte                `yaml:"bytes,flow,omitempty"`
 
+	NilSlice Manifests `yaml:"nilslice,omitempty" talos:"omitonlyifnil"`
+
 	unexported int
 }
 
@@ -45,14 +47,20 @@ type Endpoint struct {
 
 type Machine struct {
 	State  int
-	Config *MachineConfig
+	Config *MachineConfig `yaml:",omitempty"`
 }
 
 type MachineConfig struct {
 	Version      string
 	Capabilities []string
 }
 
+type Manifests []Manifest
+
+type Manifest struct {
+	Name string `yaml:"name"`
+}
+
 type WithCustomMarshaller struct {
 	value string
 }
@@ -82,7 +90,7 @@ var (
 
 func init() {
 	configDoc.Comments[encoder.LineComment] = "test configuration"
-	configDoc.Fields = make([]encoder.Doc, 8)
+	configDoc.Fields = make([]encoder.Doc, 11)
 	configDoc.Fields[1].Comments[encoder.LineComment] = "<<<"
 	configDoc.Fields[2].Comments[encoder.HeadComment] = "complex slice"
 	configDoc.Fields[3].Comments[encoder.FootComment] = "some text example for map"
@@ -92,6 +100,11 @@ func init() {
 		Port: 5554,
 	}})
 
+	configDoc.Fields[9].Comments[encoder.LineComment] = "A nilslice field is really cool."
+	configDoc.Fields[9].AddExample("nilslice example", Manifests{{
+		Name: "foo",
+	}})
+
 	endpointDoc.Comments[encoder.LineComment] = "endpoint settings"
 	endpointDoc.Fields = make([]encoder.Doc, 2)
 	endpointDoc.Fields[0].Comments[encoder.LineComment] = "endpoint host"
@@ -164,14 +177,19 @@ func (suite *EncoderSuite) TestRun() {
 			expectedYAML: `integer: 0
 # <<<
 slice: []
+# complex slice
+complex_slice: []
+#   # slice example
+#   - host: 127.0.0.1 # endpoint host
+#     port: 5554 # custom port
+
 map: {}
 # some text example for map
-# # complex slice
+# # A nilslice field is really cool.
 
-# # slice example
-# complex_slice:
-#     - host: 127.0.0.1 # endpoint host
-#       port: 5554 # custom port
+# # nilslice example
+# nilslice:
+#     - name: foo
 `,
 			options: []encoder.Option{
 				encoder.WithComments(encoder.CommentsAll),
@@ -183,12 +201,16 @@ map: {}
 			value: &Config{},
 			expectedYAML: `integer: 0
 slice: []
+complex_slice: []
+#   # slice example
+#   - host: 127.0.0.1
+#     port: 5554
+
 map: {}
 
-# # slice example
-# complex_slice:
-#     - host: 127.0.0.1
-#       port: 5554
+# # nilslice example
+# nilslice:
+#     - name: foo
 `,
 			options: []encoder.Option{
 				encoder.WithComments(encoder.CommentsExamples),
@@ -392,10 +414,9 @@ mixed_in: a # was inlined
 			expectedYAML: `first:
     - # this is some version
       version: ""
-      
-      # capabilities:
-      #     - reboot
-      #     - upgrade
+      capabilities: []
+      #   - reboot
+      #   - upgrade
 `,
 			incompatible: true,
 		},
@@ -408,7 +429,6 @@ mixed_in: a # was inlined
 			},
 			expectedYAML: `machine:
     state: 1000
-    config: null
 `,
 			incompatible: true,
 			options: []encoder.Option{
@@ -450,6 +470,33 @@ mixed_in: a # was inlined
 				encoder.WithComments(encoder.CommentsExamples),
 			},
 		},
+		{
+			name: "with onlyifnotnil tag",
+			value: &Config{
+				NilSlice: Manifests{},
+			},
+			expectedYAML: `integer: 0
+# <<<
+slice: []
+# complex slice
+complex_slice: []
+#   # slice example
+#   - host: 127.0.0.1 # endpoint host
+#     port: 5554 # custom port
+
+map: {}
+# some text example for map
+
+# A nilslice field is really cool.
+nilslice: []
+#   # nilslice example
+#   - name: foo
+`,
+			incompatible: true,
+			options: []encoder.Option{
+				encoder.WithComments(encoder.CommentsAll),
+			},
+		},
 	}
 
 	for _, test := range tests {

pkg/machinery/config/types/v1alpha1/generate/init.go

@@ -90,6 +90,7 @@ func initUd(in *Input) (*v1alpha1.Config, error) {
 		ClusterServiceAccount:         in.Certs.K8sServiceAccount,
 		BootstrapToken:                in.Secrets.BootstrapToken,
 		ClusterAESCBCEncryptionSecret: in.Secrets.AESCBCEncryptionSecret,
+		ExtraManifests:                []string{},
 		ClusterInlineManifests:        v1alpha1.ClusterInlineManifests{},
 	}