Skip to content

Commit

Permalink
Support EgressIP assigning and failover in antrea-agent (#2186)
Browse files Browse the repository at this point in the history
1.Add memberlist cluster in antrea-agent

A cluster will be created in the background when the Egress feature is
turned on. And the local Node will join all the other K8s Nodes in a
memberlist cluster.

Each Node in the cluster holds the same consistent hash ring for each
ExternalIPPool, in order to distribute egress IPs equally among the
selected Nodes (which are part of the memberlist cluster). When a Node
leaves the cluster, its IPs are redistributed. When a Node joins the
cluster, it's added to the hash ring and a small fraction of IPs are
re-assigned to that Node.

2.Assign EgressIP to owner Node and update Egress status

Assign a owner node for Egress which with a valid externalIPPool. When
Egress has been assigned a owner node and egressIP has assigned, the
Egress status will updated, Egress status is the name of the Node that
holds the Egress IP.

Signed-off-by: wenqiq <wenqiq@vmware.com>
  • Loading branch information
wenqiq authored Jul 8, 2021
1 parent 05eee25 commit 5313dd7
Show file tree
Hide file tree
Showing 31 changed files with 1,932 additions and 37 deletions.
26 changes: 26 additions & 0 deletions build/yamls/antrea-aks.yml
Original file line number Diff line number Diff line change
Expand Up @@ -1345,6 +1345,10 @@ spec:
- jsonPath: .metadata.creationTimestamp
name: Age
type: date
- description: The Owner Node of egress IP
jsonPath: .status.egressNode
name: Node
type: string
name: v1alpha2
schema:
openAPIV3Schema:
Expand Down Expand Up @@ -1415,11 +1419,18 @@ spec:
required:
- appliedTo
type: object
status:
properties:
egressNode:
type: string
type: object
required:
- spec
type: object
served: true
storage: true
subresources:
status: {}
---
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
Expand Down Expand Up @@ -3282,6 +3293,20 @@ rules:
- get
- watch
- list
- apiGroups:
- crd.antrea.io
resources:
- egresses/status
verbs:
- update
- apiGroups:
- crd.antrea.io
resources:
- externalippools
verbs:
- get
- watch
- list
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
Expand Down Expand Up @@ -3497,6 +3522,7 @@ rules:
- watch
- list
- update
- patch
- apiGroups:
- crd.antrea.io
resources:
Expand Down
26 changes: 26 additions & 0 deletions build/yamls/antrea-eks.yml
Original file line number Diff line number Diff line change
Expand Up @@ -1345,6 +1345,10 @@ spec:
- jsonPath: .metadata.creationTimestamp
name: Age
type: date
- description: The Owner Node of egress IP
jsonPath: .status.egressNode
name: Node
type: string
name: v1alpha2
schema:
openAPIV3Schema:
Expand Down Expand Up @@ -1415,11 +1419,18 @@ spec:
required:
- appliedTo
type: object
status:
properties:
egressNode:
type: string
type: object
required:
- spec
type: object
served: true
storage: true
subresources:
status: {}
---
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
Expand Down Expand Up @@ -3282,6 +3293,20 @@ rules:
- get
- watch
- list
- apiGroups:
- crd.antrea.io
resources:
- egresses/status
verbs:
- update
- apiGroups:
- crd.antrea.io
resources:
- externalippools
verbs:
- get
- watch
- list
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
Expand Down Expand Up @@ -3497,6 +3522,7 @@ rules:
- watch
- list
- update
- patch
- apiGroups:
- crd.antrea.io
resources:
Expand Down
26 changes: 26 additions & 0 deletions build/yamls/antrea-gke.yml
Original file line number Diff line number Diff line change
Expand Up @@ -1345,6 +1345,10 @@ spec:
- jsonPath: .metadata.creationTimestamp
name: Age
type: date
- description: The Owner Node of egress IP
jsonPath: .status.egressNode
name: Node
type: string
name: v1alpha2
schema:
openAPIV3Schema:
Expand Down Expand Up @@ -1415,11 +1419,18 @@ spec:
required:
- appliedTo
type: object
status:
properties:
egressNode:
type: string
type: object
required:
- spec
type: object
served: true
storage: true
subresources:
status: {}
---
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
Expand Down Expand Up @@ -3282,6 +3293,20 @@ rules:
- get
- watch
- list
- apiGroups:
- crd.antrea.io
resources:
- egresses/status
verbs:
- update
- apiGroups:
- crd.antrea.io
resources:
- externalippools
verbs:
- get
- watch
- list
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
Expand Down Expand Up @@ -3497,6 +3522,7 @@ rules:
- watch
- list
- update
- patch
- apiGroups:
- crd.antrea.io
resources:
Expand Down
26 changes: 26 additions & 0 deletions build/yamls/antrea-ipsec.yml
Original file line number Diff line number Diff line change
Expand Up @@ -1345,6 +1345,10 @@ spec:
- jsonPath: .metadata.creationTimestamp
name: Age
type: date
- description: The Owner Node of egress IP
jsonPath: .status.egressNode
name: Node
type: string
name: v1alpha2
schema:
openAPIV3Schema:
Expand Down Expand Up @@ -1415,11 +1419,18 @@ spec:
required:
- appliedTo
type: object
status:
properties:
egressNode:
type: string
type: object
required:
- spec
type: object
served: true
storage: true
subresources:
status: {}
---
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
Expand Down Expand Up @@ -3282,6 +3293,20 @@ rules:
- get
- watch
- list
- apiGroups:
- crd.antrea.io
resources:
- egresses/status
verbs:
- update
- apiGroups:
- crd.antrea.io
resources:
- externalippools
verbs:
- get
- watch
- list
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
Expand Down Expand Up @@ -3497,6 +3522,7 @@ rules:
- watch
- list
- update
- patch
- apiGroups:
- crd.antrea.io
resources:
Expand Down
26 changes: 26 additions & 0 deletions build/yamls/antrea.yml
Original file line number Diff line number Diff line change
Expand Up @@ -1345,6 +1345,10 @@ spec:
- jsonPath: .metadata.creationTimestamp
name: Age
type: date
- description: The Owner Node of egress IP
jsonPath: .status.egressNode
name: Node
type: string
name: v1alpha2
schema:
openAPIV3Schema:
Expand Down Expand Up @@ -1415,11 +1419,18 @@ spec:
required:
- appliedTo
type: object
status:
properties:
egressNode:
type: string
type: object
required:
- spec
type: object
served: true
storage: true
subresources:
status: {}
---
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
Expand Down Expand Up @@ -3282,6 +3293,20 @@ rules:
- get
- watch
- list
- apiGroups:
- crd.antrea.io
resources:
- egresses/status
verbs:
- update
- apiGroups:
- crd.antrea.io
resources:
- externalippools
verbs:
- get
- watch
- list
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
Expand Down Expand Up @@ -3497,6 +3522,7 @@ rules:
- watch
- list
- update
- patch
- apiGroups:
- crd.antrea.io
resources:
Expand Down
14 changes: 14 additions & 0 deletions build/yamls/base/agent-rbac.yml
Original file line number Diff line number Diff line change
Expand Up @@ -149,6 +149,20 @@ rules:
- get
- watch
- list
- apiGroups:
- crd.antrea.io
resources:
- egresses/status
verbs:
- update
- apiGroups:
- crd.antrea.io
resources:
- externalippools
verbs:
- get
- watch
- list
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
Expand Down
1 change: 1 addition & 0 deletions build/yamls/base/controller-rbac.yml
Original file line number Diff line number Diff line change
Expand Up @@ -207,6 +207,7 @@ rules:
- watch
- list
- update
- patch
- apiGroups:
- crd.antrea.io
resources:
Expand Down
11 changes: 11 additions & 0 deletions build/yamls/base/crds.yml
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,11 @@ spec:
- format: ipv6
externalIPPool:
type: string
status:
type: object
properties:
egressNode:
type: string
additionalPrinterColumns:
- description: Specifies the SNAT IP address for the selected workloads.
jsonPath: .spec.egressIP
Expand All @@ -89,6 +94,12 @@ spec:
- jsonPath: .metadata.creationTimestamp
name: Age
type: date
- description: The Owner Node of egress IP
jsonPath: .status.egressNode
name: Node
type: string
subresources:
status: {}
scope: Cluster
names:
plural: egresses
Expand Down
10 changes: 9 additions & 1 deletion cmd/antrea-agent/agent.go
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,8 @@ func run(o *Options) error {
crdInformerFactory := crdinformers.NewSharedInformerFactory(crdClient, informerDefaultResync)
traceflowInformer := crdInformerFactory.Crd().V1alpha1().Traceflows()
egressInformer := crdInformerFactory.Crd().V1alpha2().Egresses()
nodeInformer := informerFactory.Core().V1().Nodes()
externalIPPoolInformer := crdInformerFactory.Crd().V1alpha2().ExternalIPPools()

// Create Antrea Clientset for the given config.
antreaClientProvider := agent.NewAntreaClientProvider(o.config.AntreaClientConnection, k8sClient)
Expand Down Expand Up @@ -223,7 +225,13 @@ func run(o *Options) error {

var egressController *egress.EgressController
if features.DefaultFeatureGate.Enabled(features.Egress) {
egressController = egress.NewEgressController(ofClient, egressInformer, antreaClientProvider, ifaceStore, routeClient, nodeConfig.Name)
egressController, err = egress.NewEgressController(
ofClient, antreaClientProvider, crdClient, ifaceStore, routeClient, nodeConfig.Name, nodeConfig.NodeIPAddr.IP,
o.config.ClusterMembershipPort, egressInformer, nodeInformer, externalIPPoolInformer,
)
if err != nil {
return fmt.Errorf("error creating new Egress controller: %v", err)
}
}

isChaining := false
Expand Down
5 changes: 5 additions & 0 deletions cmd/antrea-agent/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,11 @@ type AgentConfig struct {
// APIPort is the port for the antrea-agent APIServer to serve on.
// Defaults to 10350.
APIPort int `yaml:"apiPort,omitempty"`

// ClusterMembershipPort is the server port used by the antrea-agent to run a gossip-based cluster membership protocol. Currently it's used only when the Egress feature is enabled.
// Defaults to 10351.
ClusterMembershipPort int `yaml:"clusterPort,omitempty"`

// Enable metrics exposure via Prometheus. Initializes Prometheus metrics listener
// Defaults to true.
EnablePrometheusMetrics bool `yaml:"enablePrometheusMetrics,omitempty"`
Expand Down
4 changes: 4 additions & 0 deletions cmd/antrea-agent/options.go
Original file line number Diff line number Diff line change
Expand Up @@ -193,6 +193,10 @@ func (o *Options) setDefaults() {
o.config.APIPort = apis.AntreaAgentAPIPort
}

if o.config.ClusterMembershipPort == 0 {
o.config.ClusterMembershipPort = apis.AntreaAgentClusterMembershipPort
}

if features.DefaultFeatureGate.Enabled(features.FlowExporter) {
if o.config.FlowCollectorAddr == "" {
o.config.FlowCollectorAddr = defaultFlowCollectorAddress
Expand Down
Loading

0 comments on commit 5313dd7

Please sign in to comment.