Skip to content
This repository has been archived by the owner on Apr 3, 2018. It is now read-only.

shim: Start shims inside appropriate namespaces #637

Merged
merged 4 commits into from
Mar 2, 2018
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 25 additions & 2 deletions hyperstart_agent.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ import (

proxyClient "github.com/clearcontainers/proxy/client"
"github.com/containers/virtcontainers/pkg/hyperstart"
ns "github.com/containers/virtcontainers/pkg/nsenter"
"github.com/sirupsen/logrus"
"github.com/vishvananda/netlink"
)
Expand Down Expand Up @@ -314,7 +315,19 @@ func (h *hyper) exec(pod *Pod, c Container, cmd Cmd) (*Process, error) {
Process: *hyperProcess,
}

process, err := prepareAndStartShim(pod, h.shim, c.id, token, h.state.URL, cmd)
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I prefer passing additional arguments here rather than duplicating the code.

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fair enough ! I have removed it.

enterNSList := []ns.Namespace{
{
PID: c.process.Pid,
Type: ns.NSTypeNet,
},
{
PID: c.process.Pid,
Type: ns.NSTypePID,
},
}

process, err := prepareAndStartShim(pod, h.shim, c.id,
token, h.state.URL, cmd, []ns.NSType{}, enterNSList)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -488,7 +501,17 @@ func (h *hyper) createContainer(pod *Pod, c *Container) (*Process, error) {
return nil, err
}

return prepareAndStartShim(pod, h.shim, c.id, token, h.state.URL, c.config.Cmd)
createNSList := []ns.NSType{ns.NSTypePID}

enterNSList := []ns.Namespace{
{
Path: pod.networkNS.NetNsPath,
Type: ns.NSTypeNet,
},
}

return prepareAndStartShim(pod, h.shim, c.id, token,
h.state.URL, c.config.Cmd, createNSList, enterNSList)
}

// startContainer is the agent Container starting implementation for hyperstart.
Expand Down
27 changes: 25 additions & 2 deletions kata_agent.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ import (
"syscall"

vcAnnotations "github.com/containers/virtcontainers/pkg/annotations"
ns "github.com/containers/virtcontainers/pkg/nsenter"
"github.com/containers/virtcontainers/pkg/uuid"
kataclient "github.com/kata-containers/agent/protocols/client"
"github.com/kata-containers/agent/protocols/grpc"
Expand Down Expand Up @@ -297,7 +298,19 @@ func (k *kataAgent) exec(pod *Pod, c Container, cmd Cmd) (*Process, error) {
return nil, err
}

return prepareAndStartShim(pod, k.shim, c.id, req.ExecId, k.state.URL, cmd)
enterNSList := []ns.Namespace{
{
PID: c.process.Pid,
Type: ns.NSTypeNet,
},
{
PID: c.process.Pid,
Type: ns.NSTypePID,
},
}

return prepareAndStartShim(pod, k.shim, c.id, req.ExecId,
k.state.URL, cmd, []ns.NSType{}, enterNSList)
}

func (k *kataAgent) generateInterfacesAndRoutes(networkNS NetworkNamespace) ([]*grpc.Interface, []*grpc.Route, error) {
Expand Down Expand Up @@ -671,7 +684,17 @@ func (k *kataAgent) createContainer(pod *Pod, c *Container) (*Process, error) {
return nil, err
}

return prepareAndStartShim(pod, k.shim, c.id, req.ExecId, k.state.URL, c.config.Cmd)
createNSList := []ns.NSType{ns.NSTypePID}

enterNSList := []ns.Namespace{
{
Path: pod.networkNS.NetNsPath,
Type: ns.NSTypeNet,
},
}

return prepareAndStartShim(pod, k.shim, c.id, req.ExecId,
k.state.URL, c.config.Cmd, createNSList, enterNSList)
}

func (k *kataAgent) startContainer(pod Pod, c *Container) error {
Expand Down
194 changes: 194 additions & 0 deletions pkg/nsenter/nsenter.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,194 @@
//
// Copyright (c) 2018 Intel Corporation
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The commit mentions "Based on the CNI implementation of the
package 'ns' on the plugins repository"
, so do you need to reference some of those details here? Or is this a clean-room re-implementation?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is really a brand new re-implementation here. I have just got inspiration from what the ns package is doing for CNI.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If there's anything that was loosely copied from the ns package, we have to include the right copyrights.

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ok I understand, then I'll add them.

// Copyright 2015-2017 CNI authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

package nsenter

import (
"fmt"
"os"
"path/filepath"
"runtime"
"strconv"
"sync"
"syscall"

"golang.org/x/sys/unix"
)

// Filesystems constants.
const (
// https://github.com/torvalds/linux/blob/master/include/uapi/linux/magic.h
nsFSMagic = 0x6e736673
procFSMagic = 0x9fa0

procRootPath = "/proc"
nsDirPath = "ns"
taskDirPath = "task"
)

// NSType defines a namespace type.
type NSType string

// List of namespace types.
// Notice that neither "mnt" nor "user" are listed into this list.
// Because Golang is multithreaded, we get some errors when trying
// to switch to those namespaces, getting "invalid argument".
// The solution is to reexec the current code so that it will call
// into a C constructor, making sure the namespace can be entered
// without multithreading issues.
const (
NSTypeCGroup NSType = "cgroup"
NSTypeIPC = "ipc"
NSTypeNet = "net"
NSTypePID = "pid"
NSTypeUTS = "uts"
)

// CloneFlagsTable is exported so that consumers of this package don't need
// to define this same table again.
var CloneFlagsTable = map[NSType]int{
NSTypeCGroup: unix.CLONE_NEWCGROUP,
NSTypeIPC: unix.CLONE_NEWIPC,
NSTypeNet: unix.CLONE_NEWNET,
NSTypePID: unix.CLONE_NEWPID,
NSTypeUTS: unix.CLONE_NEWUTS,
}

// Namespace describes a namespace that will be entered.
type Namespace struct {
Path string
PID int
Type NSType
}

type nsPair struct {
targetNS *os.File
threadNS *os.File
}

func getNSPathFromPID(pid int, nsType NSType) string {
return filepath.Join(procRootPath, strconv.Itoa(pid), nsDirPath, string(nsType))
}

func getCurrentThreadNSPath(nsType NSType) string {
return filepath.Join(procRootPath, strconv.Itoa(os.Getpid()),
taskDirPath, strconv.Itoa(unix.Gettid()), nsDirPath, string(nsType))
}

func setNS(nsFile *os.File, nsType NSType) error {
if nsFile == nil {
return fmt.Errorf("File handler cannot be nil")
}

nsFlag, exist := CloneFlagsTable[nsType]
if !exist {
return fmt.Errorf("Unknown namespace type %q", nsType)
}

if err := unix.Setns(int(nsFile.Fd()), nsFlag); err != nil {
return fmt.Errorf("Error switching to ns %v: %v", nsFile.Name(), err)
}

return nil
}

// getFileFromNS checks the provided file path actually matches a real
// namespace filesystem, and then opens it to return a handler to this
// file. This is needed since the system call setns() expects a file
// descriptor to enter the given namespace.
func getFileFromNS(nsPath string) (*os.File, error) {
stat := syscall.Statfs_t{}
if err := syscall.Statfs(nsPath, &stat); err != nil {
return nil, fmt.Errorf("failed to Statfs %q: %v", nsPath, err)
}

switch stat.Type {
case nsFSMagic, procFSMagic:
break
default:
return nil, fmt.Errorf("unknown FS magic on %q: %x", nsPath, stat.Type)
}

file, err := os.Open(nsPath)
if err != nil {
return nil, err
}

return file, nil
}

// NsEnter executes the passed closure under the given namespace,
// restoring the original namespace afterwards.
func NsEnter(nsList []Namespace, toRun func() error) error {
targetNSList := make(map[NSType]*nsPair)

// Open all targeted namespaces.
for _, ns := range nsList {
targetNSPath := ns.Path
if targetNSPath == "" {
targetNSPath = getNSPathFromPID(ns.PID, ns.Type)
}

targetNS, err := getFileFromNS(targetNSPath)
if err != nil {
return fmt.Errorf("failed to open target ns: %v", err)
}
defer targetNS.Close()

targetNSList[ns.Type] = &nsPair{
targetNS: targetNS,
}
}

containedCall := func() error {
for nsType := range targetNSList {
threadNS, err := getFileFromNS(getCurrentThreadNSPath(nsType))
if err != nil {
return fmt.Errorf("failed to open current ns: %v", err)
}
defer threadNS.Close()

targetNSList[nsType].threadNS = threadNS
}

// Switch to namespaces all at once.
for nsType, pair := range targetNSList {
// Switch to targeted namespace.
if err := setNS(pair.targetNS, nsType); err != nil {
return fmt.Errorf("error switching to ns %v: %v", pair.targetNS.Name(), err)
}
// Switch back to initial namespace after closure return.
defer setNS(pair.threadNS, nsType)
}

return toRun()
}

var wg sync.WaitGroup
wg.Add(1)

var innerError error
go func() {
defer wg.Done()
runtime.LockOSThread()
defer runtime.UnlockOSThread()
innerError = containedCall()
}()
wg.Wait()

return innerError
}
Loading