Skip to content

Commit

Permalink
Incorporate SR-IOV forwarder (#300)
Browse files Browse the repository at this point in the history
Signed-off-by: Vladimir Popov <vladimir.popov@xored.com>
Signed-off-by: Ed Warnicke <hagbard@gmail.com>
  • Loading branch information
Vladimir Popov authored Sep 21, 2021
1 parent 4fa0446 commit 31e9be1
Show file tree
Hide file tree
Showing 9 changed files with 828 additions and 111 deletions.
5 changes: 5 additions & 0 deletions Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,11 @@ COPY . .
RUN go build -o /bin/forwarder .

FROM build as test
ENV NSM_DEVICE_PLUGIN_PATH=/build/run/device-plugins
RUN mkdir -p ${NSM_DEVICE_PLUGIN_PATH}
ENV NSM_POD_RESOURCES_PATH=/build/run/pod-resources
RUN mkdir -p ${NSM_POD_RESOURCES_PATH}
ENV NSM_SRIOV_CONFIG_FILE=/dev/null
CMD go test -test.v ./...

FROM test as debug
Expand Down
10 changes: 6 additions & 4 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -10,17 +10,19 @@ require (
github.com/edwarnicke/govpp v0.0.0-20210831163558-1534cf6ada42
github.com/edwarnicke/grpcfd v0.1.0
github.com/edwarnicke/vpphelper v0.0.0-20210512223648-f914b171f679
github.com/golang/protobuf v1.4.3
github.com/golang/protobuf v1.5.2
github.com/kelseyhightower/envconfig v1.4.0
github.com/networkservicemesh/api v1.0.1-0.20210907194827-9a36433d7d6e
github.com/networkservicemesh/sdk v0.5.1-0.20210920202612-c7b15bacaae5
github.com/networkservicemesh/sdk-vpp v0.0.0-20210920203239-1e2b8cf04089
github.com/networkservicemesh/sdk v0.5.1-0.20210914140353-07ced33cad48
github.com/networkservicemesh/sdk-k8s v0.0.0-20210901072442-2e531cc7f2be
github.com/networkservicemesh/sdk-sriov v0.0.0-20210914141410-e098156e4221
github.com/networkservicemesh/sdk-vpp v0.0.0-20210914141416-15df58636081
github.com/pkg/errors v0.9.1
github.com/sirupsen/logrus v1.7.0
github.com/spiffe/go-spiffe/v2 v2.0.0-beta.2
github.com/stretchr/testify v1.7.0
github.com/thanhpk/randstr v1.0.4
github.com/vishvananda/netlink v1.1.0
github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae
google.golang.org/grpc v1.35.0
google.golang.org/grpc v1.38.0
)
515 changes: 498 additions & 17 deletions go.sum

Large diffs are not rendered by default.

63 changes: 63 additions & 0 deletions internal/config/config.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
// Copyright (c) 2020-2021 Cisco and/or its affiliates.
//
// Copyright (c) 2021 Doc.ai and/or its affiliates.
//
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at:
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build linux

// Package config - contain environment variables used by cmd-forwarder-vpp
package config

import (
"net"
"net/url"
"time"

"github.com/kelseyhightower/envconfig"

"github.com/networkservicemesh/cmd-forwarder-vpp/internal/vppinit"
)

// Config - configuration for cmd-forwarder-vpp
type Config struct {
Name string `default:"forwarder" desc:"Name of Endpoint"`
NSName string `default:"xconnectns" desc:"Name of Network Service to Register with Registry"`
ConnectTo url.URL `default:"unix:///connect.to.socket" desc:"url to connect to" split_words:"true"`
ListenOn url.URL `default:"unix:///listen.on.socket" desc:"url to listen on" split_words:"true"`
MaxTokenLifetime time.Duration `default:"10m" desc:"maximum lifetime of tokens" split_words:"true"`
LogLevel string `default:"INFO" desc:"Log level" split_words:"true"`

TunnelIP net.IP `desc:"IP to use for tunnels" split_words:"true"`
VppAPISocket string `default:"" desc:"filename of socket to connect to existing VPP instance. If empty a VPP instance is run in forwarder" split_words:"true"`
VppInit vppinit.Func `default:"AF_PACKET" desc:"type of VPP initialization. Must be AF_PACKET or NONE" split_words:"true"`

ResourcePollTimeout time.Duration `default:"30s" desc:"device plugin polling timeout" split_words:"true"`
DevicePluginPath string `default:"/var/lib/kubelet/device-plugins/" desc:"path to the device plugin directory" split_words:"true"`
PodResourcesPath string `default:"/var/lib/kubelet/pod-resources/" desc:"path to the pod resources directory" split_words:"true"`
SRIOVConfigFile string `default:"pci.config" desc:"PCI resources config path" split_words:"true"`
PCIDevicesPath string `default:"/sys/bus/pci/devices" desc:"path to the PCI devices directory" split_words:"true"`
PCIDriversPath string `default:"/sys/bus/pci/drivers" desc:"path to the PCI drivers directory" split_words:"true"`
CgroupPath string `default:"/host/sys/fs/cgroup/devices" desc:"path to the host cgroup directory" split_words:"true"`
VFIOPath string `default:"/host/dev/vfio" desc:"path to the host VFIO directory" split_words:"true"`
}

// Process reads config from env
func (c *Config) Process() error {
if err := envconfig.Usage("nsm", c); err != nil {
return err
}
return envconfig.Process("nsm", c)
}
16 changes: 16 additions & 0 deletions internal/imports/imports_linux.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

47 changes: 31 additions & 16 deletions internal/tests/suite_setup_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,37 +24,44 @@ import (
"io/ioutil"
"net"
"os"
"path"
"path/filepath"
"time"

"git.fd.io/govpp.git/binapi/vpe"
nested "github.com/antonfisher/nested-logrus-formatter"
"github.com/edwarnicke/exechelper"
"github.com/edwarnicke/grpcfd"
"github.com/kelseyhightower/envconfig"
"github.com/edwarnicke/vpphelper"
"github.com/sirupsen/logrus"
"github.com/spiffe/go-spiffe/v2/spiffetls/tlsconfig"
"github.com/spiffe/go-spiffe/v2/workloadapi"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials"

"github.com/networkservicemesh/api/pkg/api/registry"
"github.com/networkservicemesh/sdk-k8s/pkg/tools/k8stest/deviceplugin"
"github.com/networkservicemesh/sdk-k8s/pkg/tools/k8stest/podresources"
"github.com/networkservicemesh/sdk-k8s/pkg/tools/socketpath"
"github.com/networkservicemesh/sdk/pkg/registry/common/expire"
"github.com/networkservicemesh/sdk/pkg/registry/common/memory"
registryrecvfd "github.com/networkservicemesh/sdk/pkg/registry/common/recvfd"
"github.com/networkservicemesh/sdk/pkg/registry/core/adapters"
registrychain "github.com/networkservicemesh/sdk/pkg/registry/core/chain"
"github.com/networkservicemesh/sdk/pkg/tools/grpcutils"
"github.com/networkservicemesh/sdk/pkg/tools/log"
"github.com/networkservicemesh/sdk/pkg/tools/log/logruslogger"
"github.com/networkservicemesh/sdk/pkg/tools/spiffejwt"
"github.com/networkservicemesh/sdk/pkg/tools/spire"
"github.com/networkservicemesh/sdk/pkg/tools/token"

"github.com/edwarnicke/vpphelper"

"github.com/networkservicemesh/cmd-forwarder-vpp/internal/vppinit"
)

const (
kubeletSocket = "kubelet.sock"
)

func (f *ForwarderTestSuite) SetupSuite() {
logrus.SetFormatter(&nested.Formatter{})
logrus.SetLevel(logrus.DebugLevel)
Expand All @@ -64,6 +71,12 @@ func (f *ForwarderTestSuite) SetupSuite() {

starttime := time.Now()

// ********************************************************************************
log.FromContext(f.ctx).Infof("Getting Config from Env (time since start: %s)", time.Since(starttime))
// ********************************************************************************
_ = os.Setenv("NSM_TUNNEL_IP", forwarderIP)
f.Require().NoError(f.config.Process())

// ********************************************************************************
log.FromContext(f.ctx).Infof("Creating test bridge (time since start: %s)", time.Since(starttime))
// ********************************************************************************
Expand All @@ -83,6 +96,21 @@ func (f *ForwarderTestSuite) SetupSuite() {
_, err = vppinit.LinkToAfPacket(f.ctx, f.vppClientConn, net.ParseIP(clientIP))
f.Require().NoError(err)

// ********************************************************************************
log.FromContext(f.ctx).Infof("Creating k8s API stubs (time since start: %s)", time.Since(starttime))
// ********************************************************************************
// Create and start device plugin server
grpcServer := grpc.NewServer()
deviceplugin.StartRegistrationServer(f.config.DevicePluginPath, grpcServer)
socketPath := socketpath.SocketPath(path.Join(f.config.DevicePluginPath, kubeletSocket))
f.Require().Len(grpcutils.ListenAndServe(f.ctx, grpcutils.AddressToURL(socketPath), grpcServer), 0)

// Create and start pod resources server
grpcServer = grpc.NewServer()
podresources.StartPodResourcesListerServer(grpcServer)
socketPath = socketpath.SocketPath(path.Join(f.config.PodResourcesPath, kubeletSocket))
f.Require().Len(grpcutils.ListenAndServe(f.ctx, grpcutils.AddressToURL(socketPath), grpcServer), 0)

// ********************************************************************************
log.FromContext(f.ctx).Infof("Running Spire (time since start: %s)", time.Since(starttime))
// ********************************************************************************
Expand Down Expand Up @@ -118,22 +146,9 @@ func (f *ForwarderTestSuite) SetupSuite() {
exechelper.WithStdout(os.Stdout),
exechelper.WithStderr(os.Stderr),
exechelper.WithGracePeriod(30*time.Second),
exechelper.WithEnvKV("NSM_TUNNEL_IP", forwarderIP),
)
f.Require().Len(f.sutErrCh, 0)

// Get config from env
// ********************************************************************************
log.FromContext(f.ctx).Infof("Getting Config from Env (time since start: %s)", time.Since(starttime))
// ********************************************************************************
f.Require().NoError(envconfig.Process("nsm", &f.config))

level, err := logrus.ParseLevel(f.config.LogLevel)
if err != nil {
logrus.Fatalf("invalid log level %s", f.config.LogLevel)
}
logrus.SetLevel(level)

// ********************************************************************************
log.FromContext(f.ctx).Infof("Creating registryServer and registryClient (time since start: %s)", time.Since(starttime))
// ********************************************************************************
Expand Down
25 changes: 7 additions & 18 deletions internal/tests/suite_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,43 +20,32 @@ package tests

import (
"context"
"net"
"net/url"
"testing"
"time"

"github.com/edwarnicke/vpphelper"
"github.com/spiffe/go-spiffe/v2/bundle/x509bundle"
"github.com/spiffe/go-spiffe/v2/svid/x509svid"
"github.com/stretchr/testify/suite"
"google.golang.org/grpc"

"github.com/networkservicemesh/api/pkg/api/registry"

"github.com/edwarnicke/vpphelper"
"github.com/networkservicemesh/cmd-forwarder-vpp/internal/config"
)

// Config - configuration for cmd-forwarder-vpp
type Config struct {
Name string `default:"forwarder" desc:"Name of Endpoint"`
NSName string `default:"xconnectns" desc:"Name of Network Service to Register with Registry"`
TunnelIP net.IP `desc:"IP to use for tunnels" split_words:"true"`
ConnectTo url.URL `default:"unix:///connect.to.socket" desc:"url to connect to" split_words:"true"`
MaxTokenLifetime time.Duration `default:"24h" desc:"maximum lifetime of tokens" split_words:"true"`
LogLevel string `default:"INFO" desc:"Log level" split_words:"true"`
TestCount int `default:"1" desc:"Number of time to run tests" split_words:"true"`
}

type ForwarderTestSuite struct {
suite.Suite
ctx context.Context
cancel context.CancelFunc
config Config
config config.Config

sutErrCh <-chan error
sutCC grpc.ClientConnInterface

// Spire stuff
spireErrCh <-chan error
sutErrCh <-chan error
x509source x509svid.Source
x509bundle x509bundle.Source
sutCC grpc.ClientConnInterface

// vppServer stuff
vppServerConn vpphelper.Connection
Expand Down
88 changes: 88 additions & 0 deletions internal/xconnectns/server.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
// Copyright (c) 2021 Doc.ai and/or its affiliates.
//
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at:
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

//+build linux

// Package xconnectns provides an endpoint implementing xconnectns
package xconnectns

import (
"context"
"net"
"net/url"

"google.golang.org/grpc"

"github.com/networkservicemesh/api/pkg/api/networkservice"
"github.com/networkservicemesh/api/pkg/api/networkservice/mechanisms/kernel"
"github.com/networkservicemesh/api/pkg/api/networkservice/mechanisms/memif"
"github.com/networkservicemesh/api/pkg/api/networkservice/mechanisms/noop"
"github.com/networkservicemesh/api/pkg/api/networkservice/mechanisms/vfio"
"github.com/networkservicemesh/api/pkg/api/networkservice/mechanisms/vxlan"
"github.com/networkservicemesh/api/pkg/api/networkservice/mechanisms/wireguard"
sriovxconnectns "github.com/networkservicemesh/sdk-sriov/pkg/networkservice/chains/xconnectns"
"github.com/networkservicemesh/sdk-sriov/pkg/networkservice/common/resourcepool"
sriovconfig "github.com/networkservicemesh/sdk-sriov/pkg/sriov/config"
sriovtokens "github.com/networkservicemesh/sdk-sriov/pkg/tools/tokens"
vppxconnectns "github.com/networkservicemesh/sdk-vpp/pkg/networkservice/chains/xconnectns"
"github.com/networkservicemesh/sdk/pkg/networkservice/chains/endpoint"
"github.com/networkservicemesh/sdk/pkg/networkservice/common/mechanisms"
"github.com/networkservicemesh/sdk/pkg/networkservice/common/switchcase"
"github.com/networkservicemesh/sdk/pkg/tools/token"
)

// NewServer - returns an implementation of the xconnectns network service
func NewServer(
ctx context.Context,
name string,
authzServer networkservice.NetworkServiceServer,
tokenGenerator token.GeneratorFunc,
vppConn vppxconnectns.Connection,
tunnelIP net.IP,
pciPool resourcepool.PCIPool,
resourcePool resourcepool.ResourcePool,
sriovConfig *sriovconfig.Config,
vfioDir, cgroupBaseDir string,
clientURL *url.URL,
clientDialOptions ...grpc.DialOption,
) endpoint.Endpoint {
return endpoint.Combine(func(servers []networkservice.NetworkServiceServer) networkservice.NetworkServiceServer {
vppForwarder := servers[0]
sriovForwarder := servers[1]
return mechanisms.NewServer(map[string]networkservice.NetworkServiceServer{
kernel.MECHANISM: switchcase.NewServer(
&switchcase.ServerCase{
Condition: func(_ context.Context, conn *networkservice.Connection) bool {
return sriovtokens.IsTokenID(kernel.ToMechanism(conn.GetMechanism()).GetDeviceTokenID())
},
Server: sriovForwarder,
},
&switchcase.ServerCase{
Condition: switchcase.Default,
Server: vppForwarder,
},
),
vfio.MECHANISM: sriovForwarder,
memif.MECHANISM: vppForwarder,
vxlan.MECHANISM: vppForwarder,
wireguard.MECHANISM: vppForwarder,
noop.MECHANISM: sriovForwarder,
})
},
vppxconnectns.NewServer(ctx, name, authzServer, tokenGenerator, clientURL, vppConn, tunnelIP, clientDialOptions...),
sriovxconnectns.NewServer(ctx, name, authzServer, tokenGenerator, pciPool, resourcePool, sriovConfig, vfioDir, cgroupBaseDir, clientURL, clientDialOptions...),
)
}
Loading

0 comments on commit 31e9be1

Please sign in to comment.