Skip to content

Commit

Permalink
feat: support dynamically changing qos for EIP (#2671)
Browse files Browse the repository at this point in the history 
* feat: support dynamically changing qos for EIP

* style: improving readability

* fix: add qos yaml to chart directory

* fix: add log
  • Loading branch information
@shane965
shane965 authored Apr 25, 2023
1 parent d865b48 commit 15780bf
Show file tree
Hide file tree
Showing 8 changed files with 515 additions and 125 deletions.
77 changes: 59 additions & 18 deletions charts/templates/kube-ovn-crd.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1983,11 +1983,11 @@ spec:
subresources:
status: {}
additionalPrinterColumns:
- jsonPath: .spec.bandwidthLimitRule.ingressMax
name: IngressMax
- jsonPath: .spec.shared
name: Shared
type: string
- jsonPath: .spec.bandwidthLimitRule.egressMax
name: EgressMax
- jsonPath: .spec.bindingType
name: BindingType
type: string
schema:
openAPIV3Schema:
Expand All @@ -1996,13 +1996,31 @@ spec:
status:
type: object
properties:
bandwidthLimitRule:
type: object
properties:
ingressMax:
type: string
egressMax:
type: string
shared:
type: boolean
bindingType:
type: string
bandwidthLimitRules:
type: array
items:
type: object
properties:
name:
type: string
interface:
type: string
rateMax:
type: string
burstMax:
type: string
priority:
type: integer
direction:
type: string
matchType:
type: string
matchValue:
type: string
conditions:
type: array
items:
Expand All @@ -2023,10 +2041,33 @@ spec:
spec:
type: object
properties:
bandwidthLimitRule:
type: object
properties:
ingressMax:
type: string
egressMax:
type: string
shared:
type: boolean
bindingType:
type: string
bandwidthLimitRules:
type: array
items:
type: object
properties:
name:
type: string
interface:
type: string
rateMax:
type: string
burstMax:
type: string
priority:
type: integer
direction:
type: string
matchType:
type: string
matchValue:
type: string
required:
- name
x-kubernetes-list-map-keys:
- name
x-kubernetes-list-type: map
77 changes: 59 additions & 18 deletions dist/images/install.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2203,11 +2203,11 @@ spec:
subresources:
status: {}
additionalPrinterColumns:
- jsonPath: .spec.bandwidthLimitRule.ingressMax
name: IngressMax
- jsonPath: .spec.shared
name: Shared
type: string
- jsonPath: .spec.bandwidthLimitRule.egressMax
name: EgressMax
- jsonPath: .spec.bindingType
name: BindingType
type: string
schema:
openAPIV3Schema:
Expand All @@ -2216,13 +2216,31 @@ spec:
status:
type: object
properties:
bandwidthLimitRule:
type: object
properties:
ingressMax:
type: string
egressMax:
type: string
shared:
type: boolean
bindingType:
type: string
bandwidthLimitRules:
type: array
items:
type: object
properties:
name:
type: string
interface:
type: string
rateMax:
type: string
burstMax:
type: string
priority:
type: integer
direction:
type: string
matchType:
type: string
matchValue:
type: string
conditions:
type: array
items:
Expand All @@ -2243,13 +2261,36 @@ spec:
spec:
type: object
properties:
bandwidthLimitRule:
type: object
properties:
ingressMax:
type: string
egressMax:
type: string
shared:
type: boolean
bindingType:
type: string
bandwidthLimitRules:
type: array
items:
type: object
properties:
name:
type: string
interface:
type: string
rateMax:
type: string
burstMax:
type: string
priority:
type: integer
direction:
type: string
matchType:
type: string
matchValue:
type: string
required:
- name
x-kubernetes-list-map-keys:
- name
x-kubernetes-list-type: map
EOF

if $DPDK; then
Expand Down
54 changes: 35 additions & 19 deletions dist/images/vpcnatgateway/nat-gateway.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -248,63 +248,76 @@ function delete_tc_filter() {
v4ip=$2
direction=$3

# tc -p -s -d filter show dev net1 parent $qdisc_id prio 1
# output like this:
# filter protocol ip u32 chain 0
# filter protocol ip u32 chain 0 fh 800: ht divisor 1
# filter protocol ip u32 chain 0 fh 800::800 order 2048 key ht 800 bkt 0 *flowid :1 not_in_hw
# match IP dst x.x.x.1/32
# police 0x1 rate 1Mbit burst 1Mb mtu 2Kb action drop overhead 0b linklayer ethernet
# ref 1 bind 1 installed 392 sec used 392 sec firstused 18818153 sec
# Sent 0 bytes 0 pkts (dropped 0, overlimits 0)
# tc -p -s -d filter show dev net1 parent $qdisc_id
# filter protocol ip pref 10 u32 chain 0
# filter protocol ip pref 10 u32 chain 0 fh 800: ht divisor 1
# filter protocol ip pref 10 u32 chain 0 fh 800::800 order 2048 key ht 800 bkt 0 *flowid :1 not_in_hw
# match IP dst 172.18.11.2/32
# police 0x1 rate 10Mbit burst 10Mb mtu 2Kb action drop overhead 0b linklayer ethernet
# ref 1 bind 1 installed 47118 sec used 47118 sec firstused 18113444 sec

# Sent 0 bytes 0 pkts (dropped 0, overlimits 0)

# get the corresponding filterID by the EIP, and use the filterID to delete the corresponding filtering rule.
ipList=$(tc -p -s -d filter show dev net1 parent $qdisc_id prio 1 | grep "match IP $direction" | awk '{print $4}')
ipList=$(tc -p -s -d filter show dev net1 parent $qdisc_id | grep "match IP " | awk '{print $4}')
i=0
for line in $ipList; do
i=$((i+1))
if echo "$line" | grep $v4ip; then
filterID=$(tc -p -s -d filter show dev net1 parent $qdisc_id prio 1 | grep "filter protocol ip u32 \(fh\|chain [0-9]\+ fh\) \(\w\+::\w\+\) *" | awk '{print $8}' | sed -n $i"p")
exec_cmd "tc filter del dev net1 parent $qdisc_id protocol ip prio 1 handle $filterID u32"
result=$(tc -p -s -d filter show dev net1 parent $qdisc_id | grep "filter protocol ip pref [0-9]\+ u32 \(fh\|chain [0-9]\+ fh\) \(\w\+::\w\+\) *" | awk '{print $5,$10}' | sed -n $i"p")
arr=($result)
pref=${arr[0]}
filterID=${arr[1]}
exec_cmd "tc filter del dev net1 parent $qdisc_id protocol ip prio $pref handle $filterID u32"
break
fi
done
}

function eip_ingress_qos_add() {
# ingress:
# external --> net1 --> qos -->
# dst ip is iptables eip on net1
for rule in $@
do
arr=(${rule//,/ })
v4ip=(${arr[0]//\// })
rate=${arr[1]}
priority=${arr[1]}
rate=${arr[2]}
burst=${arr[3]}
direction="dst"
tc qdisc add dev net1 ingress 2>/dev/nul || true
# get qdisc id
qdisc_id=$(tc qdisc show dev net1 ingress | awk '{print $3}')
# del old filter
tc -p -s -d filter show dev net1 parent $qdisc_id prio 1 | grep -w $v4ip
tc -p -s -d filter show dev net1 parent $qdisc_id | grep -w $v4ip
if [ "$?" -eq 0 ];then
delete_tc_filter $qdisc_id $v4ip $direction
fi
exec_cmd "tc filter add dev net1 parent $qdisc_id protocol ip prio 1 u32 match ip $direction $v4ip police rate "$rate"Mbit burst "$rate"Mb drop flowid :1"
exec_cmd "tc filter add dev net1 parent $qdisc_id protocol ip prio $priority u32 match ip $direction $v4ip police rate "$rate"Mbit burst "$burst"Mb drop flowid :1"
done
}

function eip_egress_qos_add() {
# egress:
# net1 --> qos --> external
# src ip is iptables eip on net1
for rule in $@
do
arr=(${rule//,/ })
v4ip=(${arr[0]//\// })
rate=${arr[1]}
priority=${arr[1]}
rate=${arr[2]}
burst=${arr[3]}
qdisc_id="1:0"
direction="src"
tc qdisc add dev net1 root handle $qdisc_id htb 2>/dev/nul || true
# del old filter
tc -p -s -d filter show dev net1 parent $qdisc_id prio 1 | grep -w $v4ip
tc -p -s -d filter show dev net1 parent $qdisc_id | grep -w $v4ip
if [ "$?" -eq 0 ];then
delete_tc_filter $qdisc_id $v4ip $direction
fi
exec_cmd "tc filter add dev net1 parent $qdisc_id protocol ip prio 1 u32 match ip $direction $v4ip police rate "$rate"Mbit burst "$rate"Mb drop flowid :1"
exec_cmd "tc filter add dev net1 parent $qdisc_id protocol ip prio $priority u32 match ip $direction $v4ip police rate "$rate"Mbit burst "$burst"Mb drop flowid :1"
done
}

Expand All @@ -315,7 +328,10 @@ function eip_ingress_qos_del() {
v4ip=(${arr[0]//\// })
direction="dst"
qdisc_id=$(tc qdisc show dev net1 ingress | awk '{print $3}')
delete_tc_filter $qdisc_id $v4ip $direction
# if qdisc_id is empty, this means ingress qdisc is not added, so we don't need to delete filter.
if [ -n "$qdisc_id" ]; then
delete_tc_filter $qdisc_id $v4ip $direction
fi
done
}

Expand Down
51 changes: 45 additions & 6 deletions pkg/apis/kubeovn/v1/types.go
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
package v1

import (
"fmt"

corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

Expand Down Expand Up @@ -39,6 +41,25 @@ var (
PolicyDrop = SgPolicy(ovnnb.ACLActionDrop)
)

type QoSPolicyBindingType string

const (
QoSBindingTypeEIP QoSPolicyBindingType = "EIP"
)

type QoSPolicyRuleDirection string

const (
DirectionIngress QoSPolicyRuleDirection = "ingress"
DirectionEgress QoSPolicyRuleDirection = "egress"
)

type QoSPolicyRuleMatchType string

const (
MatchTypeIP QoSPolicyRuleMatchType = "ip"
)

// Constants for condition
const (
// Ready => controller considers this resource Ready
Expand Down Expand Up @@ -1266,7 +1287,9 @@ type QoSPolicy struct {
Status QoSPolicyStatus `json:"status,omitempty"`
}
type QoSPolicySpec struct {
BandwidthLimitRule QoSPolicyBandwidthLimitRule `json:"bandwidthLimitRule,omitempty"`
BandwidthLimitRules QoSPolicyBandwidthLimitRules `json:"bandwidthLimitRules"`
Shared bool `json:"shared"`
BindingType QoSPolicyBindingType `json:"bindingType"`
}

// Condition describes the state of an object at a certain point.
Expand All @@ -1292,14 +1315,30 @@ type QoSPolicyCondition struct {

// BandwidthLimitRule describes the rule of an bandwidth limit.
type QoSPolicyBandwidthLimitRule struct {
IngressMax string `json:"ingressMax"`
EgressMax string `json:"egressMax"`
Name string `json:"name"`
Interface string `json:"interface,omitempty"`
RateMax string `json:"rateMax,omitempty"`
BurstMax string `json:"burstMax,omitempty"`
Priority int `json:"priority,omitempty"`
Direction QoSPolicyRuleDirection `json:"direction,omitempty"`
MatchType QoSPolicyRuleMatchType `json:"matchType,omitempty"`
MatchValue string `json:"matchValue,omitempty"`
}

type QoSPolicyBandwidthLimitRules []*QoSPolicyBandwidthLimitRule

func (s QoSPolicyBandwidthLimitRules) Strings() string {
var resultNames []string
for _, rule := range s {
resultNames = append(resultNames, rule.Name)
}
return fmt.Sprintf("%s", resultNames)
}

type QoSPolicyStatus struct {
// +optional
// +patchStrategy=merge
BandwidthLimitRule QoSPolicyBandwidthLimitRule `json:"bandwidthLimitRule" patchStrategy:"merge"`
BandwidthLimitRules QoSPolicyBandwidthLimitRules `json:"bandwidthLimitRules" patchStrategy:"merge"`
Shared bool `json:"shared" patchStrategy:"merge"`
BindingType QoSPolicyBindingType `json:"bindingType"`

// Conditions represents the latest state of the object
// +optional
Expand Down
Loading

0 comments on commit 15780bf

Please sign in to comment.