diff --git a/go.mod b/go.mod index 32b42728..1d0211fc 100644 --- a/go.mod +++ b/go.mod @@ -31,6 +31,7 @@ require ( go.fd.io/govpp v0.10.0-alpha.0.20240110141843-761adec77524 golang.org/x/text v0.14.0 google.golang.org/grpc v1.59.0 + gopkg.in/yaml.v3 v3.0.1 ) require ( @@ -94,6 +95,5 @@ require ( gopkg.in/fsnotify.v1 v1.4.7 // indirect gopkg.in/square/go-jose.v2 v2.5.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/kubelet v0.28.3 // indirect ) diff --git a/internal/config/config.go b/internal/config/config.go index 214cb37f..e1a8f73f 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -1,6 +1,8 @@ -// Copyright (c) 2020-2023 Cisco and/or its affiliates. +// Copyright (c) 2020-2024 Cisco and/or its affiliates. // -// Copyright (c) 2021-2023 Doc.ai and/or its affiliates. +// Copyright (c) 2021-2024 Doc.ai and/or its affiliates. +// +// Copyright (c) 2024 Nordix and/or its affiliates. // // SPDX-License-Identifier: Apache-2.0 // @@ -46,10 +48,11 @@ type Config struct { OpenTelemetryEndpoint string `default:"otel-collector.observability.svc.cluster.local:4317" desc:"OpenTelemetry Collector Endpoint"` MetricsExportInterval time.Duration `default:"10s" desc:"interval between mertics exports" split_words:"true"` - TunnelIP net.IP `desc:"IP to use for tunnels" split_words:"true"` - VxlanPort uint16 `default:"0" desc:"VXLAN port to use" split_words:"true"` - VppAPISocket string `default:"/var/run/vpp/external/vpp-api.sock" desc:"filename of socket to connect to existing VPP instance. If empty a VPP instance is run in forwarder" split_words:"true"` - VppInit vppinit.Func `default:"AF_PACKET" desc:"type of VPP initialization. Must be AF_XDP, AF_PACKET or NONE" split_words:"true"` + TunnelIP net.IP `desc:"IP to use for tunnels" split_words:"true"` + VxlanPort uint16 `default:"0" desc:"VXLAN port to use" split_words:"true"` + VppAPISocket string `default:"/var/run/vpp/external/vpp-api.sock" desc:"filename of socket to connect to existing VPP instance. If empty a VPP instance is run in forwarder" split_words:"true"` + VppInit vppinit.Func `default:"AF_PACKET" desc:"type of VPP initialization. Must be AF_XDP, AF_PACKET or NONE" split_words:"true"` + VppInitParams string `desc:"Configuration file path containing VPP API parameters for initialization" split_words:"true"` ResourcePollTimeout time.Duration `default:"30s" desc:"device plugin polling timeout" split_words:"true"` DevicePluginPath string `default:"/var/lib/kubelet/device-plugins/" desc:"path to the device plugin directory" split_words:"true"` diff --git a/internal/imports/imports_linux.go b/internal/imports/imports_linux.go index 0c2f63a5..6c94703b 100644 --- a/internal/imports/imports_linux.go +++ b/internal/imports/imports_linux.go @@ -114,6 +114,7 @@ import ( _ "google.golang.org/grpc" _ "google.golang.org/grpc/credentials" _ "google.golang.org/grpc/health/grpc_health_v1" + _ "gopkg.in/yaml.v3" _ "net" _ "net/url" _ "os" diff --git a/internal/tests/apiparams_test.go b/internal/tests/apiparams_test.go new file mode 100644 index 00000000..e48cd6b2 --- /dev/null +++ b/internal/tests/apiparams_test.go @@ -0,0 +1,127 @@ +// Copyright (c) 2024 Nordix Foundation. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at: +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tests + +import ( + "context" + "os" + "path" + "testing" + + "github.com/stretchr/testify/require" + yaml "gopkg.in/yaml.v3" + + "github.com/networkservicemesh/govpp/binapi/af_packet" + + "github.com/networkservicemesh/cmd-forwarder-vpp/internal/vppinit" +) + +const ( + configFilePath = "/var/lib/networkservicemesh/vppapi-hostint-args.yaml" +) + +type SomeValuesType struct { + AfPacket *SomeAfPacketParams `yaml:"AF_PACKET"` + AfXdp *SomeAfXDPParams `yaml:"AF_XDP"` +} + +type SomeAfPacketParams struct { + RxFrameSize uint32 `yaml:"rxFrameSize"` + RxFramesPerBlock uint32 `yaml:"rxFramesPerBlock"` +} + +type SomeAfXDPParams struct { + RxqSize uint16 `yaml:"rxqSize"` +} + +var someValues = &SomeValuesType{ + AfPacket: &SomeAfPacketParams{ + RxFrameSize: 20480, + RxFramesPerBlock: 2048, + }, + AfXdp: &SomeAfXDPParams{ + RxqSize: 16384, + }, +} + +func (c *SomeValuesType) DumpToFile(filename string) error { + contents, err := yaml.Marshal(c) + if err != nil { + return err + } + if err := os.MkdirAll(path.Dir(filename), 0o700); err != nil { + return err + } + return os.WriteFile(filename, contents, 0o600) +} + +func TestDefaults(t *testing.T) { + packetValues := vppinit.GetAfPacketValues(context.Background()) + require.Equal(t, &vppinit.AfPacketParams{ + Mode: af_packet.AF_PACKET_API_MODE_ETHERNET, + Flags: af_packet.AF_PACKET_API_FLAG_VERSION_2, + RxFrameSize: 10240, + TxFrameSize: 10240, + RxFramesPerBlock: 1024, + TxFramesPerBlock: 1024, + NumRxQueues: 1, + NumTxQueues: 0, + }, + packetValues) + + xdpValues := vppinit.GetAfXdpValues(context.Background()) + require.Equal(t, &vppinit.AfXDPParams{ + Mode: 0, + RxqSize: 8192, + TxqSize: 8192, + Flags: 0, + }, + xdpValues) +} + +func TestSomeValuesSet(t *testing.T) { + _ = os.Setenv("NSM_VPP_INIT_PARAMS", configFilePath) + err := someValues.DumpToFile(configFilePath) + require.NoError(t, err) + defer func() { + if errRem := os.RemoveAll(configFilePath); errRem != nil { + t.Fatalf("no file generated or the generated file cannot removed") + } + }() + + packetValues := vppinit.GetAfPacketValues(context.Background()) + require.Equal(t, &vppinit.AfPacketParams{ + Mode: af_packet.AF_PACKET_API_MODE_ETHERNET, + Flags: af_packet.AF_PACKET_API_FLAG_VERSION_2, + RxFrameSize: 20480, + TxFrameSize: 10240, + RxFramesPerBlock: 2048, + TxFramesPerBlock: 1024, + NumRxQueues: 1, + NumTxQueues: 0, + }, + packetValues) + + xdpValues := vppinit.GetAfXdpValues(context.Background()) + require.Equal(t, &vppinit.AfXDPParams{ + Mode: 0, + RxqSize: 16384, + TxqSize: 8192, + Flags: 0, + }, + xdpValues) +} diff --git a/internal/vppinit/apiparams.go b/internal/vppinit/apiparams.go new file mode 100644 index 00000000..0535a737 --- /dev/null +++ b/internal/vppinit/apiparams.go @@ -0,0 +1,140 @@ +// Copyright (c) 2024 Nordix Foundation. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at: +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package vppinit + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/pkg/errors" + yaml "gopkg.in/yaml.v3" + + "github.com/networkservicemesh/govpp/binapi/af_packet" + "github.com/networkservicemesh/govpp/binapi/af_xdp" + "github.com/networkservicemesh/sdk/pkg/tools/log" +) + +// Parameters contains parameters for various AF types +type Parameters struct { + AfPacket *AfPacketParams `yaml:"AF_PACKET"` + AfXdp *AfXDPParams `yaml:"AF_XDP"` +} + +func (c *Parameters) String() string { + sb := &strings.Builder{} + _, _ = sb.WriteString("&{") + _, _ = sb.WriteString("AF_PACKET:{") + var strs []string + strs = append(strs, fmt.Sprintf("%+v", c.AfPacket)) + _, _ = sb.WriteString(strings.Join(strs, " ")) + _, _ = sb.WriteString("},") + + _, _ = sb.WriteString("AF_XDP:{") + var xdpStrs []string + xdpStrs = append(xdpStrs, fmt.Sprintf("%+v", c.AfXdp)) + _, _ = sb.WriteString(strings.Join(xdpStrs, " ")) + _, _ = sb.WriteString("},") + _, _ = sb.WriteString("}") + return sb.String() +} + +// AfPacketParams contains configuration parameters for AF_PACKET interface +type AfPacketParams struct { + Mode af_packet.AfPacketMode `yaml:"mode"` + RxFrameSize uint32 `yaml:"rxFrameSize"` + TxFrameSize uint32 `yaml:"txFrameSize"` + RxFramesPerBlock uint32 `yaml:"rxFramesPerBlock"` + TxFramesPerBlock uint32 `yaml:"txFramesPerBlock"` + NumRxQueues uint16 `yaml:"numRxQueues"` + NumTxQueues uint16 `yaml:"numTxQueues"` + Flags af_packet.AfPacketFlags `yaml:"flags"` +} + +// AfXDPParams contains configuration parameters for AF_XDP interface +type AfXDPParams struct { + Mode af_xdp.AfXdpMode `yaml:"mode"` + RxqSize uint16 `yaml:"rxqSize"` + TxqSize uint16 `yaml:"txqSize"` + Flags af_xdp.AfXdpFlag `yaml:"flags"` +} + +func getDefaults() *Parameters { + return &Parameters{ + AfPacket: &AfPacketParams{ + Mode: af_packet.AF_PACKET_API_MODE_ETHERNET, + RxFrameSize: 10240, + TxFrameSize: 10240, + RxFramesPerBlock: 1024, + TxFramesPerBlock: 1024, + NumRxQueues: 1, + NumTxQueues: 0, + Flags: af_packet.AF_PACKET_API_FLAG_VERSION_2, + }, + AfXdp: &AfXDPParams{ + Mode: af_xdp.AF_XDP_API_MODE_AUTO, + RxqSize: 8192, + TxqSize: 8192, + Flags: 0, + }, + } +} + +// GetAfPacketValues get parameter values for af-packet interface creation +func GetAfPacketValues(ctx context.Context) *AfPacketParams { + return getConfig(ctx).AfPacket +} + +// GetAfXdpValues get parameter values for af-xdp interface creation +func GetAfXdpValues(ctx context.Context) *AfXDPParams { + return getConfig(ctx).AfXdp +} + +func getConfig(ctx context.Context) *Parameters { + cfg := getDefaults() + confFilename := os.Getenv("NSM_VPP_INIT_PARAMS") + logger := log.FromContext(ctx).WithField("ReadConfig", confFilename) + if confFilename == "" { + logger.Infof("Using default VPP init parameters %+v", cfg) + return cfg + } + if _, err := os.Stat(confFilename); os.IsNotExist(err) { + logger.Infof("Configuration file: %q not found, using default VPP init parameters (%+v)", confFilename, cfg) + return cfg + } + err := readConfig(confFilename, cfg) + if err != nil { + defaultCfg := getDefaults() + logger.Warnf("Failed to read VPP init parameters %+v Using: %+v", err, defaultCfg) + return defaultCfg + } + logger.Infof("Unmarshalled VPP init parameters: %s", cfg) + return cfg +} + +func readConfig(configFile string, cfg *Parameters) error { + bytes, err := os.ReadFile(filepath.Clean(configFile)) + if err != nil { + return errors.Wrapf(err, "error reading file: %v", configFile) + } + if err = yaml.Unmarshal(bytes, cfg); err != nil { + return errors.Wrapf(err, "error unmarshalling yaml: %s", bytes) + } + return nil +} diff --git a/internal/vppinit/vppinit.go b/internal/vppinit/vppinit.go index 6307f7ad..74863008 100644 --- a/internal/vppinit/vppinit.go +++ b/internal/vppinit/vppinit.go @@ -1,5 +1,7 @@ // Copyright (c) 2020-2023 Cisco and/or its affiliates. // +// Copyright (c) 2024 Nordix Foundation. +// // SPDX-License-Identifier: Apache-2.0 // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -41,7 +43,6 @@ import ( "github.com/networkservicemesh/govpp/binapi/ip" "github.com/networkservicemesh/govpp/binapi/ip6_nd" "github.com/networkservicemesh/govpp/binapi/ip_neighbor" - "github.com/networkservicemesh/sdk-vpp/pkg/tools/types" "github.com/networkservicemesh/sdk/pkg/tools/log" ) @@ -285,13 +286,18 @@ func LinkToSocket(ctx context.Context, vppConn api.Connection, tunnelIP net.IP, } func createAfPacket(ctx context.Context, vppConn api.Connection, link netlink.Link) (interface_types.InterfaceIndex, error) { - afPacketCreate := &af_packet.AfPacketCreateV3{ - Mode: af_packet.AF_PACKET_API_MODE_ETHERNET, - HwAddr: types.ToVppMacAddress(&link.Attrs().HardwareAddr), - HostIfName: link.Attrs().Name, - RxFrameSize: 10240, - TxFrameSize: 10240, - Flags: af_packet.AF_PACKET_API_FLAG_VERSION_2, + c := GetAfPacketValues(ctx) + var afPacketCreate *af_packet.AfPacketCreateV3 = &af_packet.AfPacketCreateV3{ + Mode: c.Mode, + HwAddr: types.ToVppMacAddress(&link.Attrs().HardwareAddr), + HostIfName: link.Attrs().Name, + RxFrameSize: c.RxFrameSize, + TxFrameSize: c.TxFrameSize, + RxFramesPerBlock: c.RxFramesPerBlock, + TxFramesPerBlock: c.TxFramesPerBlock, + NumRxQueues: c.NumRxQueues, + NumTxQueues: c.NumTxQueues, + Flags: c.Flags, } now := time.Now() afPacketCreateRsp, err := af_packet.NewServiceClient(vppConn).AfPacketCreateV3(ctx, afPacketCreate) @@ -316,13 +322,14 @@ func createAfXDP(ctx context.Context, vppConn api.Connection, link netlink.Link) if err != nil { return 0, err } - + c := GetAfXdpValues(ctx) afXDPCreate := &af_xdp.AfXdpCreate{ HostIf: link.Attrs().Name, - RxqSize: 8192, - TxqSize: 8192, + RxqSize: c.RxqSize, + TxqSize: c.TxqSize, RxqNum: rxqNum, - Mode: af_xdp.AF_XDP_API_MODE_AUTO, + Mode: c.Mode, + Flags: c.Flags, Prog: "/bin/afxdp.o", }