fix: do not change lrp even if restart kube ovn controller

pkg/controller/external-gw.go

@@ -45,8 +45,9 @@ func (c *Controller) resyncExternalGateway() {
 		if exGwEnabled == "true" && lastExGwCM != nil && reflect.DeepEqual(cm.Data, lastExGwCM) {
 			return
 		}
+		klog.Info("last external gw configmap: %v", lastExGwCM)
 		if (lastExGwCM["type"] == "distributed" && cm.Data["type"] == "centralized") ||
-			!reflect.DeepEqual(lastExGwCM["external-gw-nodes"], cm.Data["external-gw-nodes"]) {
+			lastExGwCM != nil && !reflect.DeepEqual(lastExGwCM["external-gw-nodes"], cm.Data["external-gw-nodes"]) {
 			klog.Info("external gw nodes list changed, start to remove ovn external gw")
 			if err := c.removeExternalGateway(); err != nil {
 				klog.Errorf("failed to remove old ovn external gw, %v", err)
@@ -94,6 +95,7 @@ func (c *Controller) removeExternalGateway() error {
 		}
 	}
 
+	klog.Infof("delete external gateway switch %s", c.config.ExternalGatewaySwitch)
 	if err := c.ovnLegacyClient.DeleteGatewaySwitch(c.config.ExternalGatewaySwitch); err != nil {
 		klog.Errorf("failed to delete external gateway switch, %v", err)
 		return err
@@ -117,6 +119,15 @@ func (c *Controller) establishExternalGateway(config map[string]string) error {
 		lrpIp = config["nic-ip"]
 		lrpMac = config["nic-mac"]
 	}
+	lrpName := fmt.Sprintf("%s-%s", c.config.ClusterRouter, c.config.ExternalGatewaySwitch)
+	exist, err := c.ovnClient.LogicalRouterPortExists(lrpName)
+	if err != nil {
+		return err
+	}
+	if exist {
+		klog.Infof("lrp %s exist", lrpName)
+		return nil
+	}
 	if err := c.ovnLegacyClient.CreateGatewaySwitch(c.config.ExternalGatewaySwitch, c.config.ExternalGatewayNet, c.config.ExternalGatewayVlanID, lrpIp, lrpMac, chassises); err != nil {
 		klog.Errorf("failed to create external gateway switch, %v", err)
 		return err

pkg/controller/gc.go

@@ -65,6 +65,7 @@ func (c *Controller) gcLogicalRouterPort() error {
 	}
 	for _, lrp := range lrps {
 		if !util.ContainsString(exceptPeerPorts, lrp) {
+			klog.Infof("gc logical router port %s", lrp)
 			if err = c.ovnLegacyClient.DeleteLogicalRouterPort(lrp); err != nil {
 				klog.Errorf("failed to delete logical router port %s, %v", lrp, err)
 				return err

pkg/controller/node.go

@@ -447,6 +447,7 @@ func (c *Controller) handleNodeAnnotationsForProviderNetworks(node *v1.Node) err
 
 func (c *Controller) handleDeleteNode(key string) error {
 	portName := fmt.Sprintf("node-%s", key)
+	klog.Infof("delete logical switch port %s", portName)
 	if err := c.ovnLegacyClient.DeleteLogicalSwitchPort(portName); err != nil {
 		klog.Errorf("failed to delete node switch port node-%s: %v", key, err)
 		return err

pkg/controller/pod.go

@@ -674,6 +674,7 @@ func (c *Controller) handleDeletePod(pod *v1.Pod) error {
 			klog.Warningf("failed to get port '%s' sg, %v", port.Name, err)
 		}
 		// when lsp is deleted, the port of pod is deleted from any port-group automatically.
+		klog.Infof("gc logical switch port %s", port.Name)
 		if err := c.ovnLegacyClient.DeleteLogicalSwitchPort(port.Name); err != nil {
 			klog.Errorf("failed to delete lsp %s, %v", port.Name, err)
 			return err

pkg/controller/subnet.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"net"
 	"reflect"
+	"sort"
 	"strings"
 	"time"
 
@@ -398,6 +399,7 @@ func checkAndUpdateExcludeIps(subnet *kubeovnv1.Subnet) bool {
 	changed := false
 	var excludeIps []string
 	excludeIps = append(excludeIps, strings.Split(subnet.Spec.Gateway, ",")...)
+	sort.Strings(excludeIps)
 	if len(subnet.Spec.ExcludeIps) == 0 {
 		subnet.Spec.ExcludeIps = excludeIps
 		changed = true
@@ -413,6 +415,7 @@ func checkAndUpdateExcludeIps(subnet *kubeovnv1.Subnet) bool {
 			}
 			if !gwExists {
 				subnet.Spec.ExcludeIps = append(subnet.Spec.ExcludeIps, gw)
+				sort.Strings(subnet.Spec.ExcludeIps)
 				changed = true
 			}
 		}
@@ -616,29 +619,42 @@ func (c *Controller) handleAddOrUpdateSubnet(key string) error {
 		c.patchSubnetStatus(subnet, "ListLogicalSwitchFailed", err.Error())
 		return err
 	}
-
-	needRouter := subnet.Spec.Vlan == "" || subnet.Spec.LogicalGateway
+	var isUnderlayGateway, needRouter bool
+	if subnet.Spec.Vlan == "" {
+		// subnet is overlay, should add lrp between vpc and subnet, lrp ip is subnet gw
+		needRouter = true
+	} else {
+		// subnet is underlay, lrp is controlled by vpc spec enableExternal
+		needRouter = subnet.Spec.LogicalGateway
+		isUnderlayGateway = !subnet.Spec.LogicalGateway
+	}
 	if !exist {
 		subnet.Status.EnsureStandardConditions()
 		// If multiple namespace use same ls name, only first one will success
 		if err := c.ovnLegacyClient.CreateLogicalSwitch(subnet.Name, vpc.Status.Router, subnet.Spec.CIDRBlock, subnet.Spec.Gateway, needRouter); err != nil {
 			c.patchSubnetStatus(subnet, "CreateLogicalSwitchFailed", err.Error())
 			return err
 		}
-
 		if needRouter {
 			if err := c.reconcileRouterPortBySubnet(vpc, subnet); err != nil {
 				klog.Errorf("failed to connect switch %s to router %s, %v", subnet.Name, vpc.Name, err)
 				return err
 			}
 		}
 	} else {
+		if !needRouter && isUnderlayGateway {
+			// do nothing if subnet is underlay vlan and use underlay gw
+			// TODO:// support update if spec changed
+			klog.Infof("skip reset external connection from vpc %s to switch %s", vpc.Status.Router, subnet.Name)
+			return nil
+		}
 		// logical switch exists, only update other_config
 		if err := c.ovnLegacyClient.SetLogicalSwitchConfig(subnet.Name, vpc.Status.Router, subnet.Spec.Protocol, subnet.Spec.CIDRBlock, subnet.Spec.Gateway, subnet.Spec.ExcludeIps, needRouter); err != nil {
 			c.patchSubnetStatus(subnet, "SetLogicalSwitchConfigFailed", err.Error())
 			return err
 		}
 		if !needRouter {
+			klog.Infof("remove connection from router %s to switch %s", vpc.Status.Router, subnet.Name)
 			if err := c.ovnLegacyClient.RemoveRouterPort(subnet.Name, vpc.Status.Router); err != nil {
 				klog.Errorf("failed to remove router port from %s, %v", subnet.Name, err)
 				return err
@@ -803,12 +819,14 @@ func (c *Controller) handleDeleteSubnet(subnet *kubeovnv1.Subnet) error {
 	}
 	vpc, err := c.vpcsLister.Get(subnet.Spec.Vpc)
 	if err == nil && vpc.Status.Router != "" {
+		klog.Infof("remove connection from router %s to switch %s", vpc.Status.Router, subnet.Name)
 		if err = c.ovnLegacyClient.RemoveRouterPort(subnet.Name, vpc.Status.Router); err != nil {
 			klog.Errorf("failed to delete router port %s %v", subnet.Name, err)
 			return err
 		}
 	} else {
 		if k8serrors.IsNotFound(err) {
+			klog.Infof("remove connection from router %s to switch %s", vpc.Status.Router, subnet.Name)
 			if err = c.ovnLegacyClient.RemoveRouterPort(subnet.Name, util.DefaultVpc); err != nil {
 				klog.Errorf("failed to delete router port %s %v", subnet.Name, err)
 				return err
@@ -894,6 +912,7 @@ func (c *Controller) reconcileVips(subnet *kubeovnv1.Subnet) error {
 				continue
 			}
 			if !util.ContainsString(subnet.Spec.Vips, vip) {
+				klog.Infof("delete logical switch port %s", ret["name"][0])
 				if err = c.ovnLegacyClient.DeleteLogicalSwitchPort(ret["name"][0]); err != nil {
 					klog.Errorf("failed to delete virtual port, %v", err)
 					return err
@@ -1025,12 +1044,16 @@ func (c *Controller) reconcileOvnRoute(subnet *kubeovnv1.Subnet) error {
 			return err
 		}
 
-		if !subnet.Spec.LogicalGateway {
-			if err := c.ovnLegacyClient.DeleteLogicalSwitchPort(fmt.Sprintf("%s-%s", subnet.Name, c.config.ClusterRouter)); err != nil {
+		if subnet.Spec.LogicalGateway {
+			lspName := fmt.Sprintf("%s-%s", subnet.Name, c.config.ClusterRouter)
+			klog.Infof("delete logical switch port %s", lspName)
+			if err := c.ovnLegacyClient.DeleteLogicalSwitchPort(lspName); err != nil {
 				klog.Errorf("failed to delete lsp %s-%s, %v", subnet.Name, c.config.ClusterRouter, err)
 				return err
 			}
-			if err := c.ovnLegacyClient.DeleteLogicalRouterPort(fmt.Sprintf("%s-%s", c.config.ClusterRouter, subnet.Name)); err != nil {
+			lrpName := fmt.Sprintf("%s-%s", c.config.ClusterRouter, subnet.Name)
+			klog.Infof("delete logical router port %s", lrpName)
+			if err := c.ovnLegacyClient.DeleteLogicalRouterPort(lrpName); err != nil {
 				klog.Errorf("failed to delete lrp %s-%s, %v", c.config.ClusterRouter, subnet.Name, err)
 				return err
 			}
@@ -1514,6 +1537,7 @@ func checkAndFormatsExcludeIps(subnet *kubeovnv1.Subnet) bool {
 			excludeIps = append(excludeIps, string(v.Start)+".."+string(v.End))
 		}
 	}
+	sort.Strings(excludeIps)
 	klog.V(3).Infof("excludeips before format is %v, after format is %v", subnet.Spec.ExcludeIps, excludeIps)
 	if !reflect.DeepEqual(subnet.Spec.ExcludeIps, excludeIps) {
 		subnet.Spec.ExcludeIps = excludeIps

pkg/controller/vip.go

@@ -300,7 +300,7 @@ func (c *Controller) acquireStaticIpAddress(subnetName, name, nicName, ip string
 func (c *Controller) acquireIpAddress(subnetName, name, nicName string) (string, string, string, error) {
 	var skippedAddrs []string
 	var v4ip, v6ip, mac string
-	checkConflict := false
+	checkConflict := true
 	var err error
 	for {
 		v4ip, v6ip, mac, err = c.ipam.GetRandomAddress(name, nicName, mac, subnetName, skippedAddrs, checkConflict)

pkg/controller/vlan.go

@@ -326,6 +326,7 @@ func (c *Controller) setLocalnetTag(subnet string, vlanID int) error {
 
 func (c *Controller) delLocalnet(subnet string) error {
 	localnetPort := ovs.GetLocalnetName(subnet)
+	klog.Infof("delete logical switch port %s", localnetPort)
 	if err := c.ovnLegacyClient.DeleteLogicalSwitchPort(localnetPort); err != nil {
 		klog.Errorf("failed to delete localnet port %s: %v", localnetPort, err)
 		return err

pkg/controller/vpc.go

@@ -192,9 +192,15 @@ func (c *Controller) reconcileRouterPorts(vpc *kubeovnv1.Vpc) error {
 				return err
 			}
 
-			klog.V(1).InfoS("router port not exists, trying to create", "vpc", vpc.Name, "subnet", subnetName)
+			if subnet.Spec.Vlan != "" && !subnet.Spec.LogicalGateway {
+				// skip vlan subnet which use underlay gw
+				// vpc connect to external vlan subnet is controlled by vpc spec enableExternal
+				klog.Infof("no need to connect vpc '%s' to vlan subnet %s", router, subnet.Name)
+				return nil
+			}
 
 			networks := util.GetIpAddrWithMask(subnet.Spec.Gateway, subnet.Spec.CIDRBlock)
+			klog.Infof("add vpc lrp %s, networks %s", routerPortName, networks)
 			if err := c.ovnClient.AddLogicalRouterPort(router, routerPortName, "", networks); err != nil {
 				klog.ErrorS(err, "unable to create router port", "vpc", vpc.Name, "subnet", subnetName)
 				return err
@@ -362,7 +368,9 @@ func (c *Controller) handleAddOrUpdateVpc(key string) error {
 	}
 	for _, oldPeer := range vpc.Status.VpcPeerings {
 		if !util.ContainsString(newPeers, oldPeer) {
-			if err = c.ovnLegacyClient.DeleteLogicalRouterPort(fmt.Sprintf("%s-%s", vpc.Name, oldPeer)); err != nil {
+			lrpName := fmt.Sprintf("%s-%s", vpc.Name, oldPeer)
+			klog.Infof("delete logical router port %s", lrpName)
+			if err = c.ovnLegacyClient.DeleteLogicalRouterPort(lrpName); err != nil {
 				klog.Errorf("failed to delete peer router port for vpc %s, %v", vpc.Name, err)
 				return err
 			}
@@ -822,16 +830,36 @@ func (c *Controller) handleAddVpcExternal(key string) error {
 }
 
 func (c *Controller) handleDelVpcExternal(key string) error {
-	lrpEipName := fmt.Sprintf("%s-%s", key, c.config.ExternalGatewaySwitch)
-	cachedEip, err := c.ovnEipsLister.Get(lrpEipName)
+	cachedVpc, err := c.vpcsLister.Get(key)
 	if err != nil {
+		if k8serrors.IsNotFound(err) {
+			return nil
+		}
 		return err
 	}
+	lrpEipName := fmt.Sprintf("%s-%s", key, c.config.ExternalGatewaySwitch)
 	klog.V(3).Infof("delete vpc lrp %s", lrpEipName)
 	if err := c.ovnLegacyClient.DisconnectRouterToExternal(c.config.ExternalGatewaySwitch, key); err != nil {
 		klog.Errorf("failed to disconnect router '%s' to external, %v", key, err)
 		return err
 	}
+	vpc := cachedVpc.DeepCopy()
+	vpc.Status.EnableExternal = cachedVpc.Spec.EnableExternal
+	bytes, err := vpc.Status.Bytes()
+	if err != nil {
+		return err
+	}
+	if _, err = c.config.KubeOvnClient.KubeovnV1().Vpcs().Patch(context.Background(),
+		vpc.Name, types.MergePatchType, bytes, metav1.PatchOptions{}, "status"); err != nil {
+		return err
+	}
+	cachedEip, err := c.ovnEipsLister.Get(lrpEipName)
+	if err != nil {
+		if k8serrors.IsNotFound(err) {
+			return nil
+		}
+		return err
+	}
 	if err = c.handleDelOvnEipFinalizer(cachedEip); err != nil {
 		klog.Errorf("failed to del finalizer for ovn eip, %v", err)
 		return err

pkg/ovs/ovn-nb-logical_router_port.go

@@ -12,6 +12,7 @@ import (
 	ovsclient "github.com/kubeovn/kube-ovn/pkg/ovsdb/client"
 	"github.com/kubeovn/kube-ovn/pkg/ovsdb/ovnnb"
 	"github.com/kubeovn/kube-ovn/pkg/util"
+	"k8s.io/klog/v2"
 )
 
 func (c OvnClient) GetLogicalRouterPort(name string, ignoreNotFound bool) (*ovnnb.LogicalRouterPort, error) {
@@ -67,7 +68,7 @@ func (c OvnClient) AddLogicalRouterPort(lr, name, mac, networks string) error {
 		return err
 	}
 	ops = append(ops, mutationOps...)
-
+	klog.Infof("add vpc lrp %s, networks %s", name, networks)
 	if err := Transact(c.ovnNbClient, "lrp-add", ops, c.ovnNbClient.Timeout); err != nil {
 		return fmt.Errorf("failed to create logical router port %s: %v", name, err)
 	}

pkg/ovs/ovn-nbctl-legacy.go

@@ -108,6 +108,7 @@ func (c LegacyClient) SetICAutoRoute(enable bool, blackList []string) error {
 
 // DeleteLogicalSwitchPort delete logical switch port in ovn
 func (c LegacyClient) DeleteLogicalSwitchPort(port string) error {
+	klog.Infof("delete lsp %s", port)
 	if _, err := c.ovnNbCommand(IfExists, "lsp-del", port); err != nil {
 		return fmt.Errorf("failed to delete logical switch port %s, %v", port, err)
 	}
@@ -116,14 +117,17 @@ func (c LegacyClient) DeleteLogicalSwitchPort(port string) error {
 
 // DeleteLogicalRouterPort delete logical switch port in ovn
 func (c LegacyClient) DeleteLogicalRouterPort(port string) error {
+	klog.Infof("delete lrp %s", port)
 	if _, err := c.ovnNbCommand(IfExists, "lrp-del", port); err != nil {
 		return fmt.Errorf("failed to delete logical router port %s, %v", port, err)
 	}
 	return nil
 }
 
 func (c LegacyClient) CreateICLogicalRouterPort(az, mac, subnet string, chassises []string) error {
-	if _, err := c.ovnNbCommand(MayExist, "lrp-add", c.ClusterRouter, fmt.Sprintf("%s-ts", az), mac, subnet); err != nil {
+	lrpName := fmt.Sprintf("%s-ts", az)
+	klog.Infof("add vpc lrp %s", lrpName)
+	if _, err := c.ovnNbCommand(MayExist, "lrp-add", c.ClusterRouter, lrpName, mac, subnet); err != nil {
 		return fmt.Errorf("failed to create ovn-ic lrp, %v", err)
 	}
 	if _, err := c.ovnNbCommand(MayExist, "lsp-add", util.InterconnectionSwitch, fmt.Sprintf("ts-%s", az), "--",
@@ -405,6 +409,7 @@ func (c LegacyClient) ListPodLogicalSwitchPorts(pod, namespace string) ([]string
 }
 
 func (c LegacyClient) SetLogicalSwitchConfig(ls, lr, protocol, subnet, gateway string, excludeIps []string, needRouter bool) error {
+	klog.Infof("set logical switch: ls %s, lr %s, protocol %s, subnet %s, gw %s", ls, lr, protocol, subnet, gateway)
 	var err error
 	cidrBlocks := strings.Split(subnet, ",")
 	temp := strings.Split(cidrBlocks[0], "/")
@@ -551,6 +556,8 @@ func (c LegacyClient) ConnectRouterToExternal(externalNet, vpcRouter, lrpIpCidr,
 	// add lrp and lsp between vpc router and external network
 	lsTolr := fmt.Sprintf("%s-%s", externalNet, vpcRouter)
 	lrTols := fmt.Sprintf("%s-%s", vpcRouter, externalNet)
+	klog.Infof("add vpc lrp %s, cidr %s", lrTols, lrpIpCidr)
+	klog.Infof("add lsp %s", lsTolr)
 	_, err := c.ovnNbCommand(
 		MayExist, "lrp-add", vpcRouter, lrTols, lrpMac, lrpIpCidr, "--",
 		MayExist, "lsp-add", externalNet, lsTolr, "--",
@@ -571,10 +578,12 @@ func (c LegacyClient) ConnectRouterToExternal(externalNet, vpcRouter, lrpIpCidr,
 
 func (c LegacyClient) DisconnectRouterToExternal(externalNet, vpcRouter string) error {
 	lrTols := fmt.Sprintf("%s-%s", vpcRouter, externalNet)
+	klog.Infof("delete lrp %s", lrTols)
 	if _, err := c.ovnNbCommand(IfExists, "lrp-del", lrTols); err != nil {
 		return err
 	}
 	lsTolr := fmt.Sprintf("%s-%s", externalNet, vpcRouter)
+	klog.Infof("delete lsp %s", lsTolr)
 	if _, err := c.ovnNbCommand(IfExists, "lsp-del", lsTolr); err != nil {
 		return err
 	}
@@ -584,6 +593,8 @@ func (c LegacyClient) DisconnectRouterToExternal(externalNet, vpcRouter string)
 func (c LegacyClient) CreateGatewaySwitch(name, network string, vlan int, ip, mac string, chassises []string) error {
 	lsTolr := fmt.Sprintf("%s-%s", name, c.ClusterRouter)
 	lrTols := fmt.Sprintf("%s-%s", c.ClusterRouter, name)
+	klog.Infof("add vpc lrp %s, ip %s", lrTols, ip)
+	klog.Infof("add lsp %s", lsTolr)
 	localnetPort := fmt.Sprintf("ln-%s", name)
 	if name != util.ExternalGatewaySwitch {
 		// compatiable with provider network and vlan and subnet
@@ -625,6 +636,8 @@ func (c LegacyClient) CreateGatewaySwitch(name, network string, vlan int, ip, ma
 
 func (c LegacyClient) DeleteGatewaySwitch(name string) error {
 	lrTols := fmt.Sprintf("%s-%s", c.ClusterRouter, name)
+	klog.Infof("delete gw switch %s", name)
+	klog.Infof("delete gw lrp %s", lrTols)
 	_, err := c.ovnNbCommand(
 		IfExists, "ls-del", name, "--",
 		IfExists, "lrp-del", lrTols,
@@ -819,6 +832,7 @@ func (c LegacyClient) DeleteLogicalRouter(lr string) error {
 func (c LegacyClient) RemoveRouterPort(ls, lr string) error {
 	lsTolr := fmt.Sprintf("%s-%s", ls, lr)
 	lrTols := fmt.Sprintf("%s-%s", lr, ls)
+	klog.Infof("remove router port %s, switch port %s", lrTols, lsTolr)
 	_, err := c.ovnNbCommand(IfExists, "lsp-del", lsTolr, "--",
 		IfExists, "lrp-del", lrTols)
 	if err != nil {
@@ -857,9 +871,11 @@ func (c LegacyClient) CreatePeerRouterPort(localRouter, remoteRouter, localRoute
 	if len(results) == 0 {
 		ipStr := strings.Split(localRouterPortIP, ",")
 		if len(ipStr) == 2 {
+			klog.Infof("add vpc lrp %s", localRouterPort)
 			_, err = c.ovnNbCommand(MayExist, "lrp-add", localRouter, localRouterPort, util.GenerateMac(), ipStr[0], ipStr[1], "--",
 				"set", "logical_router_port", localRouterPort, fmt.Sprintf("peer=%s", remoteRouterPort))
 		} else {
+			klog.Infof("add vpc lrp %s", localRouterPort)
 			_, err = c.ovnNbCommand(MayExist, "lrp-add", localRouter, localRouterPort, util.GenerateMac(), ipStr[0], "--",
 				"set", "logical_router_port", localRouterPort, fmt.Sprintf("peer=%s", remoteRouterPort))
 		}
@@ -869,6 +885,7 @@ func (c LegacyClient) CreatePeerRouterPort(localRouter, remoteRouter, localRoute
 		}
 	}
 
+	klog.Infof("set lrp %s, networks %s", localRouterPort, localRouterPortIP)
 	_, err = c.ovnNbCommand("set", "logical_router_port", localRouterPort,
 		fmt.Sprintf("networks=\"%s\"", strings.ReplaceAll(localRouterPortIP, ",", " ")))
 
@@ -2765,13 +2782,15 @@ func (c *LegacyClient) UpdateRouterPortIPv6RA(ls, lr, cidrBlock, gateway, ipv6RA
 		}
 
 		ipv6RAConfigsStr = strings.ReplaceAll(ipv6RAConfigsStr, " ", "")
+		klog.Infof("set lrp %s, ipv6_prefix %s", lrTols, ipv6Prefix)
 		_, err = c.ovnNbCommand("--",
 			"set", "logical_router_port", lrTols, fmt.Sprintf("ipv6_prefix=%s", ipv6Prefix), fmt.Sprintf("ipv6_ra_configs=%s", ipv6RAConfigsStr))
 		if err != nil {
 			klog.Errorf("failed to set ipv6_prefix: %s ans ipv6_ra_configs: %s for router port: %s, err: %s", ipv6Prefix, ipv6RAConfigsStr, lrTols, err)
 			return err
 		}
 	} else {
+		klog.Infof("set lrp %s", lrTols)
 		_, err = c.ovnNbCommand("--",
 			"set", "logical_router_port", lrTols, "ipv6_prefix=[]", "ipv6_ra_configs={}")
 		if err != nil {