feat: implement virtual IP (shared IP) network operator

Code mostly copy-pasted from the networkd implementation, but adapted to
the resource model:

* watch for etcd state to start/stop activities
* watch for kube-api-server static pod state to give up VIP when
api-server goes down

Gratuitous ARP code re-implemented to drop dependency on kube-vip (once
the old networkd code is removed).

Signed-off-by: Andrey Smirnov <smirnov.andrey@gmail.com>

go.mod

@@ -51,6 +51,7 @@ require (
 	github.com/insomniacslk/dhcp v0.0.0-20210528123148-fb4eaaa00ad2
 	github.com/jsimonetti/rtnetlink v0.0.0-20210531051304-b34cb89a106b
 	github.com/mattn/go-isatty v0.0.13
+	github.com/mdlayher/arp v0.0.0-20191213142603-f72070a231fc
 	github.com/mdlayher/ethtool v0.0.0-20210210192532-2b88debcdd43
 	github.com/mdlayher/genetlink v1.0.0
 	github.com/mdlayher/netlink v1.4.1

go.sum

@@ -876,6 +876,9 @@ github.com/mattn/go-shellwords v1.0.3/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vq
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI=
 github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
+github.com/mdlayher/arp v0.0.0-20191213142603-f72070a231fc h1:m7rJJJeXrYCFpsxXYapkDW53wJCDmf9bsIXUg0HoeQY=
+github.com/mdlayher/arp v0.0.0-20191213142603-f72070a231fc/go.mod h1:eOj1DDj3NAZ6yv+WafaKzY37MFZ58TdfIhQ+8nQbiis=
+github.com/mdlayher/ethernet v0.0.0-20190313224307-5b5fc417d966/go.mod h1:5s5p/sMJ6sNsFl6uCh85lkFGV8kLuIYJCRJLavVJwvg=
 github.com/mdlayher/ethernet v0.0.0-20190606142754-0394541c37b7 h1:lez6TS6aAau+8wXUP3G9I3TGlmPFEq2CTxBaRqY6AGE=
 github.com/mdlayher/ethernet v0.0.0-20190606142754-0394541c37b7/go.mod h1:U6ZQobyTjI/tJyq2HG+i/dfSoFUt8/aZCM+GKtmFk/Y=
 github.com/mdlayher/ethtool v0.0.0-20210210192532-2b88debcdd43 h1:WgyLFv10Ov49JAQI/ZLUkCZ7VJS3r74hwFIGXJsgZlY=
@@ -895,6 +898,7 @@ github.com/mdlayher/netlink v1.3.0/go.mod h1:xK/BssKuwcRXHrtN04UBkwQ6dY9VviGGuri
 github.com/mdlayher/netlink v1.4.0/go.mod h1:dRJi5IABcZpBD2A3D0Mv/AiX8I9uDEu5oGkAVrekmf8=
 github.com/mdlayher/netlink v1.4.1 h1:I154BCU+mKlIf7BgcAJB2r7QjveNPty6uNY1g9ChVfI=
 github.com/mdlayher/netlink v1.4.1/go.mod h1:e4/KuJ+s8UhfUpO9z00/fDZZmhSrs+oxyqAS9cNgn6Q=
+github.com/mdlayher/raw v0.0.0-20190313224157-43dbcdd7739d/go.mod h1:r1fbeITl2xL/zLbVnNHFyOzQJTgr/3fpf1lJX/cjzR8=
 github.com/mdlayher/raw v0.0.0-20190606142536-fef19f00fc18/go.mod h1:7EpbotpCmVZcu+KCX4g9WaRNuu11uyhiW7+Le1dKawg=
 github.com/mdlayher/raw v0.0.0-20191009151244-50f2db8cc065/go.mod h1:7EpbotpCmVZcu+KCX4g9WaRNuu11uyhiW7+Le1dKawg=
 github.com/mdlayher/raw v0.0.0-20210412142147-51b895745faf h1:InctQoB89TIkmgIFQeIL4KXNvWc1iebQXdZggqPSwL8=
@@ -1468,6 +1472,7 @@ golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73r
 golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190313220215-9f648a60d977/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190320064053-1272bf9dcd53/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190419010253-1f3472d942ba/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=

internal/app/machined/pkg/controllers/network/address_spec.go

@@ -12,11 +12,13 @@ import (
 	"github.com/cosi-project/runtime/pkg/controller"
 	"github.com/cosi-project/runtime/pkg/resource"
 	"github.com/jsimonetti/rtnetlink"
+	"github.com/mdlayher/arp"
 	"go.uber.org/zap"
 	"golang.org/x/sys/unix"
 	"inet.af/netaddr"
 
 	"github.com/talos-systems/talos/internal/app/machined/pkg/controllers/network/watch"
+	"github.com/talos-systems/talos/pkg/machinery/nethelpers"
 	"github.com/talos-systems/talos/pkg/resources/network"
 )
 
@@ -176,18 +178,31 @@ func (ctrl *AddressSpecController) syncAddress(ctx context.Context, r controller
 		}
 
 		if existing := findAddress(addrs, linkIndex, address.TypedSpec().Address); existing != nil {
+			// clear out tentative flag, it is set by the kernel, we shouldn't try to enforce it
+			existing.Flags &= ^uint8(nethelpers.AddressTentative)
+			existing.Attributes.Flags &= ^uint32(nethelpers.AddressTentative)
+
 			// check if existing matches the spec: if it does, skip update
 			if existing.Scope == uint8(address.TypedSpec().Scope) && existing.Flags == uint8(address.TypedSpec().Flags) &&
 				existing.Attributes.Flags == uint32(address.TypedSpec().Flags) {
 				return nil
 			}
 
+			logger.Debug("replacing address",
+				zap.Stringer("address", address.TypedSpec().Address),
+				zap.String("link", address.TypedSpec().LinkName),
+				zap.Stringer("old_scope", nethelpers.Scope(existing.Scope)),
+				zap.Stringer("new_scope", address.TypedSpec().Scope),
+				zap.Stringer("old_flags", nethelpers.AddressFlags(existing.Attributes.Flags)),
+				zap.Stringer("new_flags", address.TypedSpec().Flags),
+			)
+
 			// delete address to get new one assigned below
 			if err := conn.Address.Delete(existing); err != nil {
 				return fmt.Errorf("error removing address: %w", err)
 			}
 
-			logger.Sugar().Infof("removed address %s from %q", address.TypedSpec().Address, address.TypedSpec().LinkName)
+			logger.Info("removed address", zap.Stringer("address", address.TypedSpec().Address), zap.String("link", address.TypedSpec().LinkName))
 		}
 
 		// add address
@@ -204,12 +219,56 @@ func (ctrl *AddressSpecController) syncAddress(ctx context.Context, r controller
 				Flags:     uint32(address.TypedSpec().Flags),
 			},
 		}); err != nil {
-			return fmt.Errorf("error adding address: %w", err)
+			return fmt.Errorf("error adding address %s to %q: %w", address.TypedSpec().Address, address.TypedSpec().LinkName, err)
+		}
+
+		logger.Info("assigned address", zap.Stringer("address", address.TypedSpec().Address), zap.String("link", address.TypedSpec().LinkName))
+
+		if address.TypedSpec().AnnounceWithARP {
+			if err := ctrl.gratuitousARP(logger, linkIndex, address.TypedSpec().Address.IP); err != nil {
+				logger.Warn("failure sending gratuitous ARP", zap.Stringer("address", address.TypedSpec().Address), zap.String("link", address.TypedSpec().LinkName), zap.Error(err))
+			}
 		}
+	}
+
+	return nil
+}
+
+func (ctrl *AddressSpecController) gratuitousARP(logger *zap.Logger, linkIndex uint32, ip netaddr.IP) error {
+	etherBrodcast := net.HardwareAddr{0xff, 0xff, 0xff, 0xff, 0xff, 0xff}
+
+	if !ip.Is4() {
+		return nil
+	}
 
-		logger.Sugar().Infof("assigned address %s to %q", address.TypedSpec().Address, address.TypedSpec().LinkName)
+	iface, err := net.InterfaceByIndex(int(linkIndex))
+	if err != nil {
+		return err
+	}
+
+	if len(iface.HardwareAddr) != 6 {
+		// not ethernet
+		return nil
+	}
+
+	cli, err := arp.Dial(iface)
+	if err != nil {
+		return fmt.Errorf("error creating arp client: %w", err)
 	}
 
+	defer cli.Close() //nolint:errcheck
+
+	packet, err := arp.NewPacket(arp.OperationRequest, cli.HardwareAddr(), ip.IPAddr().IP, cli.HardwareAddr(), ip.IPAddr().IP)
+	if err != nil {
+		return fmt.Errorf("error building packet: %w", err)
+	}
+
+	if err = cli.WriteTo(packet, etherBrodcast); err != nil {
+		return fmt.Errorf("error sending gratuitous ARP: %w", err)
+	}
+
+	logger.Info("sent gratuitous ARP", zap.Stringer("address", ip), zap.String("link", iface.Name))
+
 	return nil
 }