Skip to content

Commit

Permalink
Disable SR-IOV by default (#343)
Browse files Browse the repository at this point in the history 
Signed-off-by: Vladimir Popov <vladimir.popov@xored.com>
  • Loading branch information
Vladimir Popov authored Sep 23, 2021
1 parent 31e9be1 commit 5cd6707
Show file tree
Hide file tree
Showing 6 changed files with 75 additions and 81 deletions.
5 changes: 0 additions & 5 deletions Dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,11 +23,6 @@ COPY . .
RUN go build -o /bin/forwarder .

FROM build as test
ENV NSM_DEVICE_PLUGIN_PATH=/build/run/device-plugins
RUN mkdir -p ${NSM_DEVICE_PLUGIN_PATH}
ENV NSM_POD_RESOURCES_PATH=/build/run/pod-resources
RUN mkdir -p ${NSM_POD_RESOURCES_PATH}
ENV NSM_SRIOV_CONFIG_FILE=/dev/null
CMD go test -test.v ./...

FROM test as debug
Expand Down
2 changes: 1 addition & 1 deletion internal/config/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ type Config struct {
ResourcePollTimeout time.Duration `default:"30s" desc:"device plugin polling timeout" split_words:"true"`
DevicePluginPath string `default:"/var/lib/kubelet/device-plugins/" desc:"path to the device plugin directory" split_words:"true"`
PodResourcesPath string `default:"/var/lib/kubelet/pod-resources/" desc:"path to the pod resources directory" split_words:"true"`
SRIOVConfigFile string `default:"pci.config" desc:"PCI resources config path" split_words:"true"`
SRIOVConfigFile string `default:"" desc:"PCI resources config path" split_words:"true"`
PCIDevicesPath string `default:"/sys/bus/pci/devices" desc:"path to the PCI devices directory" split_words:"true"`
PCIDriversPath string `default:"/sys/bus/pci/drivers" desc:"path to the PCI drivers directory" split_words:"true"`
CgroupPath string `default:"/host/sys/fs/cgroup/devices" desc:"path to the host cgroup directory" split_words:"true"`
Expand Down
4 changes: 0 additions & 4 deletions internal/imports/imports_linux.go
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

24 changes: 0 additions & 24 deletions internal/tests/suite_setup_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@ import (
"io/ioutil"
"net"
"os"
"path"
"path/filepath"
"time"

Expand All @@ -40,15 +39,11 @@ import (
"google.golang.org/grpc/credentials"

"github.com/networkservicemesh/api/pkg/api/registry"
"github.com/networkservicemesh/sdk-k8s/pkg/tools/k8stest/deviceplugin"
"github.com/networkservicemesh/sdk-k8s/pkg/tools/k8stest/podresources"
"github.com/networkservicemesh/sdk-k8s/pkg/tools/socketpath"
"github.com/networkservicemesh/sdk/pkg/registry/common/expire"
"github.com/networkservicemesh/sdk/pkg/registry/common/memory"
registryrecvfd "github.com/networkservicemesh/sdk/pkg/registry/common/recvfd"
"github.com/networkservicemesh/sdk/pkg/registry/core/adapters"
registrychain "github.com/networkservicemesh/sdk/pkg/registry/core/chain"
"github.com/networkservicemesh/sdk/pkg/tools/grpcutils"
"github.com/networkservicemesh/sdk/pkg/tools/log"
"github.com/networkservicemesh/sdk/pkg/tools/log/logruslogger"
"github.com/networkservicemesh/sdk/pkg/tools/spiffejwt"
Expand All @@ -58,10 +53,6 @@ import (
"github.com/networkservicemesh/cmd-forwarder-vpp/internal/vppinit"
)

const (
kubeletSocket = "kubelet.sock"
)

func (f *ForwarderTestSuite) SetupSuite() {
logrus.SetFormatter(&nested.Formatter{})
logrus.SetLevel(logrus.DebugLevel)
Expand Down Expand Up @@ -96,21 +87,6 @@ func (f *ForwarderTestSuite) SetupSuite() {
_, err = vppinit.LinkToAfPacket(f.ctx, f.vppClientConn, net.ParseIP(clientIP))
f.Require().NoError(err)

// ********************************************************************************
log.FromContext(f.ctx).Infof("Creating k8s API stubs (time since start: %s)", time.Since(starttime))
// ********************************************************************************
// Create and start device plugin server
grpcServer := grpc.NewServer()
deviceplugin.StartRegistrationServer(f.config.DevicePluginPath, grpcServer)
socketPath := socketpath.SocketPath(path.Join(f.config.DevicePluginPath, kubeletSocket))
f.Require().Len(grpcutils.ListenAndServe(f.ctx, grpcutils.AddressToURL(socketPath), grpcServer), 0)

// Create and start pod resources server
grpcServer = grpc.NewServer()
podresources.StartPodResourcesListerServer(grpcServer)
socketPath = socketpath.SocketPath(path.Join(f.config.PodResourcesPath, kubeletSocket))
f.Require().Len(grpcutils.ListenAndServe(f.ctx, grpcutils.AddressToURL(socketPath), grpcServer), 0)

// ********************************************************************************
log.FromContext(f.ctx).Infof("Running Spire (time since start: %s)", time.Since(starttime))
// ********************************************************************************
Expand Down
7 changes: 6 additions & 1 deletion internal/xconnectns/server.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,11 @@ func NewServer(
clientURL *url.URL,
clientDialOptions ...grpc.DialOption,
) endpoint.Endpoint {
vppForwarder := vppxconnectns.NewServer(ctx, name, authzServer, tokenGenerator, clientURL, vppConn, tunnelIP, clientDialOptions...)
if sriovConfig == nil {
return vppForwarder
}

return endpoint.Combine(func(servers []networkservice.NetworkServiceServer) networkservice.NetworkServiceServer {
vppForwarder := servers[0]
sriovForwarder := servers[1]
Expand All @@ -82,7 +87,7 @@ func NewServer(
noop.MECHANISM: sriovForwarder,
})
},
vppxconnectns.NewServer(ctx, name, authzServer, tokenGenerator, clientURL, vppConn, tunnelIP, clientDialOptions...),
vppForwarder,
sriovxconnectns.NewServer(ctx, name, authzServer, tokenGenerator, pciPool, resourcePool, sriovConfig, vfioDir, cgroupBaseDir, clientURL, clientDialOptions...),
)
}
114 changes: 68 additions & 46 deletions main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ import (
registryapi "github.com/networkservicemesh/api/pkg/api/registry"
"github.com/networkservicemesh/sdk-k8s/pkg/tools/deviceplugin"
"github.com/networkservicemesh/sdk-k8s/pkg/tools/podresources"
"github.com/networkservicemesh/sdk-sriov/pkg/networkservice/common/resourcepool"
sriovconfig "github.com/networkservicemesh/sdk-sriov/pkg/sriov/config"
"github.com/networkservicemesh/sdk-sriov/pkg/sriov/pci"
"github.com/networkservicemesh/sdk-sriov/pkg/sriov/resource"
Expand Down Expand Up @@ -141,55 +142,13 @@ func main() {
log.FromContext(ctx).WithField("duration", time.Since(now)).Info("completed phase 2: run vpp and get a connection to it")

// ********************************************************************************
log.FromContext(ctx).Infof("executing phase 3: get SR-IOV config from file (time since start: %s)", time.Since(starttime))
// ********************************************************************************
now = time.Now()

sriovConfig, err := sriovconfig.ReadConfig(ctx, cfg.SRIOVConfigFile)
if err != nil {
log.FromContext(ctx).Fatalf("failed to get PCI resources config: %+v", err)
}

if err = pci.UpdateConfig(cfg.PCIDevicesPath, cfg.PCIDriversPath, sriovConfig); err != nil {
log.FromContext(ctx).Fatalf("failed to update PCI resources config with VFs: %+v", err)
}

log.FromContext(ctx).WithField("duration", time.Since(now)).Infof("completed phase 3: get SR-IOV config from file")

// ********************************************************************************
log.FromContext(ctx).Infof("executing phase 4: init pools (time since start: %s)", time.Since(starttime))
// ********************************************************************************
now = time.Now()

tokenPool := sriovtoken.NewPool(sriovConfig)

pciPool, err := pci.NewPool(cfg.PCIDevicesPath, cfg.PCIDriversPath, cfg.VFIOPath, sriovConfig)
if err != nil {
log.FromContext(ctx).Fatalf("failed to init PCI pool: %+v", err)
}

resourcePool := resource.NewPool(tokenPool, sriovConfig)

log.FromContext(ctx).WithField("duration", time.Since(now)).Infof("completed phase 4: init pools")

// executing phases 3-5
// ********************************************************************************
log.FromContext(ctx).Infof("executing phase 5: start device plugin server (time since start: %s)", time.Since(starttime))
// ********************************************************************************
now = time.Now()

// Start device plugin server
if err = deviceplugin.StartServers(
ctx,
tokenPool,
cfg.ResourcePollTimeout,
deviceplugin.NewClient(cfg.DevicePluginPath),
podresources.NewClient(cfg.PodResourcesPath),
); err != nil {
log.FromContext(ctx).Fatalf("failed to start a device plugin server: %+v", err)
sriovConfig, pciPool, resourcePool := setupSRIOV(ctx, cfg, starttime)
if sriovConfig == nil {
log.FromContext(ctx).Warn("SR-IOV is not enabled")
}

log.FromContext(ctx).WithField("duration", time.Since(now)).Infof("completed phase 5: start device plugin server")

// ********************************************************************************
log.FromContext(ctx).Infof("executing phase 6: retrieving svid, check spire agent logs if this is the last line you see (time since start: %s)", time.Since(starttime))
// ********************************************************************************
Expand Down Expand Up @@ -292,6 +251,69 @@ func main() {
<-vppErrCh
}

func setupSRIOV(ctx context.Context, cfg *config.Config, starttime time.Time) (*sriovconfig.Config, resourcepool.PCIPool, resourcepool.ResourcePool) {
if cfg.SRIOVConfigFile == "" {
log.FromContext(ctx).Warn("skipping phases 3-5: no PCI resources config")
return nil, nil, nil
}

// ********************************************************************************
log.FromContext(ctx).Infof("executing phase 3: get SR-IOV config from file (time since start: %s)", time.Since(starttime))
// ********************************************************************************
now := time.Now()

sriovConfig, err := sriovconfig.ReadConfig(ctx, cfg.SRIOVConfigFile)
if err != nil {
log.FromContext(ctx).Fatalf("failed to get PCI resources config: %+v", err)
}

if err = pci.UpdateConfig(cfg.PCIDevicesPath, cfg.PCIDriversPath, sriovConfig); err != nil {
log.FromContext(ctx).Fatalf("failed to update PCI resources config with VFs: %+v", err)
}

log.FromContext(ctx).WithField("duration", time.Since(now)).Infof("completed phase 3: get SR-IOV config from file")

if len(sriovConfig.PhysicalFunctions) == 0 {
log.FromContext(ctx).Warn("skipping phases 4-5: empty PF list")
return nil, nil, nil
}

// ********************************************************************************
log.FromContext(ctx).Infof("executing phase 4: init pools (time since start: %s)", time.Since(starttime))
// ********************************************************************************
now = time.Now()

tokenPool := sriovtoken.NewPool(sriovConfig)

pciPool, err := pci.NewPool(cfg.PCIDevicesPath, cfg.PCIDriversPath, cfg.VFIOPath, sriovConfig)
if err != nil {
log.FromContext(ctx).Fatalf("failed to init PCI pool: %+v", err)
}

resourcePool := resource.NewPool(tokenPool, sriovConfig)

log.FromContext(ctx).WithField("duration", time.Since(now)).Infof("completed phase 4: init pools")

// ********************************************************************************
log.FromContext(ctx).Infof("executing phase 5: start device plugin server (time since start: %s)", time.Since(starttime))
// ********************************************************************************
now = time.Now()

if err = deviceplugin.StartServers(
ctx,
tokenPool,
cfg.ResourcePollTimeout,
deviceplugin.NewClient(cfg.DevicePluginPath),
podresources.NewClient(cfg.PodResourcesPath),
); err != nil {
log.FromContext(ctx).Fatalf("failed to start a device plugin server: %+v", err)
}

log.FromContext(ctx).WithField("duration", time.Since(now)).Infof("completed phase 5: start device plugin server")

return sriovConfig, pciPool, resourcePool
}

func exitOnErrCh(ctx context.Context, cancel context.CancelFunc, errCh <-chan error) {
// If we already have an error, log it and exit
select {
Expand Down

0 comments on commit 5cd6707

Please sign in to comment.