diff --git a/internal/tests/suite_setup_test.go b/internal/tests/suite_setup_test.go index b0d264bd..fce5ddc0 100644 --- a/internal/tests/suite_setup_test.go +++ b/internal/tests/suite_setup_test.go @@ -1,5 +1,7 @@ // Copyright (c) 2020-2023 Cisco and/or its affiliates. // +// Copyright (c) 2024 Nordix Foundation. +// // SPDX-License-Identifier: Apache-2.0 // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -52,6 +54,7 @@ import ( "github.com/networkservicemesh/sdk/pkg/tools/token" "github.com/networkservicemesh/cmd-forwarder-vpp/internal/vppinit" + "github.com/networkservicemesh/cmd-forwarder-vpp/internal/vppinit/vppcfg" ) func (f *ForwarderTestSuite) SetupSuite() { @@ -81,14 +84,14 @@ func (f *ForwarderTestSuite) SetupSuite() { log.FromContext(f.ctx).Infof("Creating test vpp Server (time since start: %s)", time.Since(starttime)) // ******************************************************************************** f.vppServerConn, f.vppServerRoot, f.vppServerErrCh = f.createVpp(f.ctx, "vpp-server") - _, err = vppinit.LinkToSocket(f.ctx, f.vppServerConn, net.ParseIP(serverIP), vppinit.AfPacket) + _, err = vppinit.LinkToSocket(f.ctx, f.vppServerConn, net.ParseIP(serverIP), vppcfg.Config{}, vppinit.AfPacket) f.Require().NoError(err) // ******************************************************************************** log.FromContext(f.ctx).Infof("Creating test vpp Client (time since start: %s)", time.Since(starttime)) // ******************************************************************************** f.vppClientConn, f.vppClientRoot, f.vppClientErrCh = f.createVpp(f.ctx, "vpp-client") - _, err = vppinit.LinkToSocket(f.ctx, f.vppClientConn, net.ParseIP(clientIP), vppinit.AfPacket) + _, err = vppinit.LinkToSocket(f.ctx, f.vppClientConn, net.ParseIP(clientIP), vppcfg.Config{}, vppinit.AfPacket) f.Require().NoError(err) // ******************************************************************************** diff --git a/internal/vppinit/links.go b/internal/vppinit/links.go index 53012151..5323f6bc 100644 --- a/internal/vppinit/links.go +++ b/internal/vppinit/links.go @@ -1,7 +1,7 @@ -// Copyright (c) 2021-2022 Nordix Foundation. -// // Copyright (c) 2023 Cisco and/or its affiliates. // +// Copyright (c) 2021-2024 Nordix Foundation. +// // SPDX-License-Identifier: Apache-2.0 // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -31,8 +31,9 @@ import ( interfaces "github.com/networkservicemesh/govpp/binapi/interface" "github.com/networkservicemesh/govpp/binapi/interface_types" - "github.com/networkservicemesh/sdk/pkg/tools/log" + + "github.com/networkservicemesh/cmd-forwarder-vpp/internal/vppinit/vppcfg" ) func initDevice(ctx context.Context, vppConn api.Connection, tunnelIP net.IP, device string, errChan chan error) { @@ -92,7 +93,7 @@ func isTunnelLink(link netlink.Link, tunnelIP net.IP) bool { } func setupLinkVpp(ctx context.Context, vppConn api.Connection, link netlink.Link) error { - swIfIndex, err := createAfPacket(ctx, vppConn, link) + swIfIndex, err := createAfPacket(ctx, vppConn, link, vppcfg.Config{}) if err != nil { return err } diff --git a/internal/vppinit/vppcfg/config.yaml b/internal/vppinit/vppcfg/config.yaml new file mode 100644 index 00000000..8c94ebc5 --- /dev/null +++ b/internal/vppinit/vppcfg/config.yaml @@ -0,0 +1,23 @@ +--- +interfaces: + - type: AF_PACKET_V3 + mode: 1 # 1 Ethernet, 2 IP + rxFrameSize: 2048 + txFrameSize: 10240 + rxFramesPerBlock: 32 + txFramesPerBlock: 1024 + # flags: Flags + numRxQueues: 1 + numTxQueues: 1 + + - type: AF_PACKET_V2 + rxFrameSize: 10240 + txFrameSize: 10240 + rxFramesPerBlock: 1024 + txFramesPerBlock: 1024 + # flags: Flags + numRxQueues: 1 + + - type: AF_XDP + rxqSize: 8192 + txqSize: 8192 diff --git a/internal/vppinit/vppcfg/vppcfg.go b/internal/vppinit/vppcfg/vppcfg.go new file mode 100644 index 00000000..a0b0834b --- /dev/null +++ b/internal/vppinit/vppcfg/vppcfg.go @@ -0,0 +1,135 @@ +// Copyright (c) 2024 Nordix Foundation. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at: +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package vppcfg provides parsing factilty for configuration parameters +// file for vpp init +package vppcfg + +import ( + "context" + "fmt" + "strconv" + "strings" + + "github.com/pkg/errors" + + "github.com/networkservicemesh/sdk-sriov/pkg/tools/yamlhelper" + "github.com/networkservicemesh/sdk/pkg/tools/log/logruslogger" +) + +// Config contains interfaces +type Config struct { + Interfaces []*Resource `yaml:"interfaces"` +} + +func (c *Config) String() string { + sb := &strings.Builder{} + _, _ = sb.WriteString("&{") + + _, _ = sb.WriteString("Interfaces:[") + var strs []string + for _, device := range c.Interfaces { + strs = append(strs, fmt.Sprintf("%+v", device)) + } + + _, _ = sb.WriteString(strings.Join(strs, " ")) + _, _ = sb.WriteString("]") + _, _ = sb.WriteString("}") + return sb.String() +} + +// Resource contains configuration parameters for an available interface +type Resource struct { + Type string `yaml:"type,omitempty"` + Mode int `yaml:"mode"` + RxFrameSize uint32 `yaml:"rxFrameSize"` + TxFrameSize uint32 `yaml:"txFrameSize"` + RxFramesPerBlock uint32 `yaml:"rxFramesPerBlock"` + TxFramesPerBlock uint32 `yaml:"txFramesPerBlock"` + NumRxQueues uint16 `yaml:"numRxQueues"` + NumTxQueues uint16 `yaml:"numTxQueues"` + RxqSize uint16 `yaml:"rxqSize"` + TxqSize uint16 `yaml:"txqSize"` +} + +func (res *Resource) String() string { + sb := &strings.Builder{} + _, _ = sb.WriteString("&{") + + _, _ = sb.WriteString("Type:") + _, _ = sb.WriteString(res.Type) + + _, _ = sb.WriteString("Mode:") + _, _ = sb.WriteString(strconv.Itoa(res.Mode)) + + _, _ = sb.WriteString("RxFrameSize:") + _, _ = sb.WriteString(strconv.FormatUint(uint64(res.RxFrameSize), 10)) + + _, _ = sb.WriteString("TxFrameSize:") + _, _ = sb.WriteString(strconv.FormatUint(uint64(res.TxFrameSize), 10)) + + _, _ = sb.WriteString("RxFramesPerBlock:") + _, _ = sb.WriteString(strconv.FormatUint(uint64(res.RxFramesPerBlock), 10)) + + _, _ = sb.WriteString("TxFramesPerBlock:") + _, _ = sb.WriteString(strconv.FormatUint(uint64(res.TxFramesPerBlock), 10)) + + _, _ = sb.WriteString("NumRxQueues:") + _, _ = sb.WriteString(strconv.FormatUint(uint64(res.NumRxQueues), 10)) + + _, _ = sb.WriteString("NumTxQueues:") + _, _ = sb.WriteString(strconv.FormatUint(uint64(res.NumTxQueues), 10)) + + _, _ = sb.WriteString("RxqSize:") + _, _ = sb.WriteString(strconv.FormatUint(uint64(res.RxqSize), 10)) + + _, _ = sb.WriteString("TxqSize:") + _, _ = sb.WriteString(strconv.FormatUint(uint64(res.TxqSize), 10)) + + _, _ = sb.WriteString("}") + return sb.String() +} + +// ReadConfig reads configuration from file +func ReadConfig(ctx context.Context, configFile string) (*Config, error) { + logger := logruslogger.New(ctx) + + cfg := &Config{} + if err := yamlhelper.UnmarshalFile(configFile, cfg); err != nil { + return nil, err + } + for _, device := range cfg.Interfaces { + err := validateResource(device) + if err != nil { + return nil, err + } + } + logger.WithField("Config", "ReadConfig").Infof("unmarshalled Config: %+v", cfg) + return cfg, nil +} + +func validateResource(res *Resource) error { + if res == nil { + return nil + } + if res.Type == "" { + return errors.Errorf("resource type must be set") + } + if (res.Type != "AF_PACKET_V3") && (res.Type != "AF_PACKET_V2") && (res.Type != "AF_XDP") { + return errors.Errorf("unsupported resource type") + } + return nil +} diff --git a/internal/vppinit/vppcfg/vppcfg_test.go b/internal/vppinit/vppcfg/vppcfg_test.go new file mode 100644 index 00000000..afe44c4b --- /dev/null +++ b/internal/vppinit/vppcfg/vppcfg_test.go @@ -0,0 +1,66 @@ +// Copyright (c) 2024 Nordix Foundation. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at: +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package vppcfg_test + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/networkservicemesh/cmd-forwarder-vpp/internal/vppinit/vppcfg" +) + +const ( + configFileName = "config.yaml" + type1 = "AF_PACKET_V3" + type2 = "AF_PACKET_V2" + type3 = "AF_XDP" +) + +func TestReadConfigFile(t *testing.T) { + cfg, err := vppcfg.ReadConfig(context.Background(), configFileName) + require.NoError(t, err) + require.Equal(t, &vppcfg.Config{ + Interfaces: []*vppcfg.Resource{ + { + Type: type1, + Mode: 1, + RxFrameSize: 2048, + TxFrameSize: 10240, + RxFramesPerBlock: 32, + TxFramesPerBlock: 1024, + NumRxQueues: 1, + NumTxQueues: 1, + }, + { + Type: type2, + RxFrameSize: 10240, + TxFrameSize: 10240, + RxFramesPerBlock: 1024, + TxFramesPerBlock: 1024, + NumRxQueues: 1, + }, + { + Type: type3, + Mode: 0, + RxqSize: 8192, + TxqSize: 8192, + }, + }, + }, cfg) +} diff --git a/internal/vppinit/vppinit.go b/internal/vppinit/vppinit.go index 6307f7ad..86f35a68 100644 --- a/internal/vppinit/vppinit.go +++ b/internal/vppinit/vppinit.go @@ -1,5 +1,7 @@ // Copyright (c) 2020-2023 Cisco and/or its affiliates. // +// Copyright (c) 2024 Nordix Foundation. +// // SPDX-License-Identifier: Apache-2.0 // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -41,9 +43,10 @@ import ( "github.com/networkservicemesh/govpp/binapi/ip" "github.com/networkservicemesh/govpp/binapi/ip6_nd" "github.com/networkservicemesh/govpp/binapi/ip_neighbor" - "github.com/networkservicemesh/sdk-vpp/pkg/tools/types" "github.com/networkservicemesh/sdk/pkg/tools/log" + + "github.com/networkservicemesh/cmd-forwarder-vpp/internal/vppinit/vppcfg" ) // AfType represents socket address family @@ -67,12 +70,12 @@ const ( // Func - vpp initialization function type Func struct { - f func(ctx context.Context, vppConn api.Connection, tunnelIP net.IP) (net.IP, error) + f func(ctx context.Context, vppConn api.Connection, tunnelIP net.IP, cfg vppcfg.Config) (net.IP, error) } // Execute vpp initialization function -func (f *Func) Execute(ctx context.Context, vppConn api.Connection, tunnelIP net.IP) (net.IP, error) { - return f.f(ctx, vppConn, tunnelIP) +func (f *Func) Execute(ctx context.Context, vppConn api.Connection, tunnelIP net.IP, cfg vppcfg.Config) (net.IP, error) { + return f.f(ctx, vppConn, tunnelIP, cfg) } // Decode for envconfig to select correct vpp initialization function @@ -84,16 +87,16 @@ func (f *Func) Decode(value string) error { return err } if ver[0] > afXdpMajorVer || (ver[0] == afXdpMajorVer && ver[1] >= afXdpMinorVer) { - f.f = func(ctx context.Context, vppConn api.Connection, tunnelIP net.IP) (net.IP, error) { - return LinkToSocket(ctx, vppConn, tunnelIP, AfXDP) + f.f = func(ctx context.Context, vppConn api.Connection, tunnelIP net.IP, cfg vppcfg.Config) (net.IP, error) { + return LinkToSocket(ctx, vppConn, tunnelIP, cfg, AfXDP) } return nil } log.FromContext(context.Background()).Warn("AF_XDP is not supported by this linux kernel version. AF_PACKET will be used") fallthrough case "AF_PACKET": - f.f = func(ctx context.Context, vppConn api.Connection, tunnelIP net.IP) (net.IP, error) { - return LinkToSocket(ctx, vppConn, tunnelIP, AfPacket) + f.f = func(ctx context.Context, vppConn api.Connection, tunnelIP net.IP, cfg vppcfg.Config) (net.IP, error) { + return LinkToSocket(ctx, vppConn, tunnelIP, cfg, AfPacket) } case "NONE": f.f = None @@ -112,7 +115,7 @@ func Must(tunnelIP net.IP, err error) net.IP { } // None - will perform no VPP initialization -func None(_ context.Context, _ api.Connection, tunnelIP net.IP) (net.IP, error) { +func None(_ context.Context, _ api.Connection, tunnelIP net.IP, _ vppcfg.Config) (net.IP, error) { return tunnelIP, nil } @@ -147,7 +150,7 @@ func getKernelVer() ([2]int, error) { // LinkToSocket - will link vpp via af_packet or af_xdp to the interface having the tunnelIP // if tunnelIP is nil, it will find the interface for the default route and use that instead. // It returns the resulting tunnelIP -func LinkToSocket(ctx context.Context, vppConn api.Connection, tunnelIP net.IP, family AfType) (net.IP, error) { +func LinkToSocket(ctx context.Context, vppConn api.Connection, tunnelIP net.IP, cfg vppcfg.Config, family AfType) (net.IP, error) { link, addrs, routes, err := linkAddrsRoutes(ctx, tunnelIP) if err != nil { return nil, err @@ -161,7 +164,7 @@ func LinkToSocket(ctx context.Context, vppConn api.Connection, tunnelIP net.IP, afFunc = createAfXDP } - swIfIndex, err := afFunc(ctx, vppConn, link) + swIfIndex, err := afFunc(ctx, vppConn, link, cfg) if err != nil { return nil, err } @@ -284,8 +287,8 @@ func LinkToSocket(ctx context.Context, vppConn api.Connection, tunnelIP net.IP, return tunnelIP, nil } -func createAfPacket(ctx context.Context, vppConn api.Connection, link netlink.Link) (interface_types.InterfaceIndex, error) { - afPacketCreate := &af_packet.AfPacketCreateV3{ +func createAfPacket(ctx context.Context, vppConn api.Connection, link netlink.Link, cfg vppcfg.Config) (interface_types.InterfaceIndex, error) { + var afPacketCreate *af_packet.AfPacketCreateV3 = &af_packet.AfPacketCreateV3{ Mode: af_packet.AF_PACKET_API_MODE_ETHERNET, HwAddr: types.ToVppMacAddress(&link.Attrs().HardwareAddr), HostIfName: link.Attrs().Name, @@ -293,6 +296,25 @@ func createAfPacket(ctx context.Context, vppConn api.Connection, link netlink.Li TxFrameSize: 10240, Flags: af_packet.AF_PACKET_API_FLAG_VERSION_2, } + var c *vppcfg.Resource + if cfg.Interfaces != nil && len(cfg.Interfaces) != 0 { + c = cfg.Interfaces[0] + } + if c != nil { + if c.Type == "AF_PACKET_V3" { + afPacketCreate.Flags = 0 + } + if c.RxFrameSize != 0 { + afPacketCreate.RxFrameSize = c.RxFrameSize + } + if c.TxFrameSize != 0 { + afPacketCreate.TxFrameSize = c.TxFrameSize + } + afPacketCreate.RxFramesPerBlock = c.RxFramesPerBlock + afPacketCreate.TxFramesPerBlock = c.TxFramesPerBlock + afPacketCreate.NumRxQueues = c.NumRxQueues + afPacketCreate.NumTxQueues = c.NumTxQueues + } now := time.Now() afPacketCreateRsp, err := af_packet.NewServiceClient(vppConn).AfPacketCreateV3(ctx, afPacketCreate) if err != nil { @@ -310,7 +332,7 @@ func createAfPacket(ctx context.Context, vppConn api.Connection, link netlink.Li return afPacketCreateRsp.SwIfIndex, nil } -func createAfXDP(ctx context.Context, vppConn api.Connection, link netlink.Link) (interface_types.InterfaceIndex, error) { +func createAfXDP(ctx context.Context, vppConn api.Connection, link netlink.Link, _ vppcfg.Config) (interface_types.InterfaceIndex, error) { // AF_XDP requires some tweaks on the host side rxqNum, err := afxdpHostSettings(ctx, link) if err != nil { diff --git a/main.go b/main.go index e2b1fe28..e85ab0d3 100644 --- a/main.go +++ b/main.go @@ -1,5 +1,7 @@ // Copyright (c) 2020-2023 Cisco and/or its affiliates. // +// Copyright (c) 2024 Nordix Foundation. +// // SPDX-License-Identifier: Apache-2.0 // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -49,6 +51,8 @@ import ( "github.com/networkservicemesh/sdk-sriov/pkg/sriov/pci" "github.com/networkservicemesh/sdk-sriov/pkg/sriov/resource" sriovtoken "github.com/networkservicemesh/sdk-sriov/pkg/sriov/token" + "github.com/networkservicemesh/sdk-vpp/pkg/networkservice/mechanisms/vxlan" + "github.com/networkservicemesh/sdk-vpp/pkg/networkservice/stats" "github.com/networkservicemesh/sdk/pkg/networkservice/common/authorize" "github.com/networkservicemesh/sdk/pkg/networkservice/common/cleanup" registryclient "github.com/networkservicemesh/sdk/pkg/registry/chains/client" @@ -63,12 +67,10 @@ import ( "github.com/networkservicemesh/sdk/pkg/tools/token" "github.com/networkservicemesh/sdk/pkg/tools/tracing" - "github.com/networkservicemesh/sdk-vpp/pkg/networkservice/mechanisms/vxlan" - "github.com/networkservicemesh/sdk-vpp/pkg/networkservice/stats" - "github.com/networkservicemesh/cmd-forwarder-vpp/internal/config" "github.com/networkservicemesh/cmd-forwarder-vpp/internal/devicecfg" "github.com/networkservicemesh/cmd-forwarder-vpp/internal/vppinit" + "github.com/networkservicemesh/cmd-forwarder-vpp/internal/vppinit/vppcfg" "github.com/networkservicemesh/cmd-forwarder-vpp/internal/xconnectns" ) @@ -134,6 +136,7 @@ func main() { log.EnableTracing(true) log.FromContext(ctx).WithField("duration", time.Since(now)).Infof("completed phase 1: get config from environment") + vppInterfaceCfg := getInitInterfaceConfig(ctx) // ******************************************************************************** // Configure Open Telemetry // ******************************************************************************** @@ -243,7 +246,7 @@ func main() { ctx, spiffejwt.TokenGeneratorFunc(source, cfg.MaxTokenLifetime), vppConn, - vppinit.Must(cfg.VppInit.Execute(ctx, vppConn, cfg.TunnelIP)), + vppinit.Must(cfg.VppInit.Execute(ctx, vppConn, cfg.TunnelIP, *vppInterfaceCfg)), pciPool, resourcePool, sriovConfig, @@ -322,6 +325,18 @@ func main() { <-cleanupDoneCh } +func getInitInterfaceConfig(ctx context.Context) *vppcfg.Config { + vppInterfaceCfg, err := vppcfg.ReadConfig(ctx, "/var/lib/networkservicemesh/config/vppinterface.cfg") + if err != nil { + log.FromContext(ctx).Infof("vpp interface config not found, interfaces will be created with default values") + vppInterfaceCfg = &vppcfg.Config{} + } + if vppInterfaceCfg.Interfaces != nil && len(vppInterfaceCfg.Interfaces) > 1 { + log.FromContext(ctx).Warnf("First vpp interface config will be used") + } + return vppInterfaceCfg +} + func setupDeviceMap(ctx context.Context, cfg *config.Config) map[string]string { if cfg.DeviceSelectorFile == "" { return nil