Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

collect specify pod && remove nsenter cmd depend #47

Merged
merged 1 commit into from
May 16, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 22 additions & 2 deletions cmd/collector/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,10 @@ package main

import (
"flag"
"fmt"
"os"
"path"
"runtime"

pod_collector "github.com/alibaba/kubeskoop/pkg/skoop/collector/podcollector"

Expand All @@ -15,10 +17,28 @@ import (
)

func main() {
var dumpPath string
runtime.GOMAXPROCS(1)
runtime.LockOSThread()

if os.Args[0] == "nsenter" {
output, err := pod_collector.NSExec(os.Args)
if err != nil {
os.Stderr.WriteString(fmt.Sprintf("error: %v\n", err))
os.Stderr.Sync()
os.Exit(1)
}
fmt.Print(output)
return
}
var (
dumpPath, podNamespace, podName, runtimeEndpoint string
)
flag.StringVar(&dumpPath, "dump-path", "/data/collector.json", "Collector result path")
flag.StringVar(&podNamespace, "namespace", "", "pod namespace to collect")
flag.StringVar(&podName, "name", "", "pod name to collect, 'host' as host network namespace")
flag.StringVar(&runtimeEndpoint, "runtime-endpoint", "", "runtime socket addr to resolve pod info")
flag.Parse()
c, err := pod_collector.NewCollector()
c, err := pod_collector.NewCollector(podNamespace, podName, runtimeEndpoint)
if err != nil {
log.Fatalf("error init collector, %v", err)
}
Expand Down
72 changes: 60 additions & 12 deletions pkg/skoop/collector/podcollector/collector.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,9 @@ import (
"strings"
"syscall"

"github.com/samber/lo"
"golang.org/x/sys/unix"

"google.golang.org/grpc/credentials/insecure"

"github.com/alibaba/kubeskoop/pkg/skoop/collector"
Expand All @@ -31,32 +34,49 @@ import (
)

type podCollector struct {
runtimeEndpoint string
podNamespace string
podName string

dockerCli client.APIClient
runtimeCli pb.RuntimeServiceClient
}

func (a *podCollector) DumpNodeInfos() (*k8s.NodeNetworkStackDump, error) {
runtime.GOMAXPROCS(1)
runtime.LockOSThread()
defer runtime.UnlockOSThread()
dump := &k8s.NodeNetworkStackDump{}
var err error
dump.Pods, err = a.PodList()
if err != nil {
return nil, fmt.Errorf("error get pod list, %v", err)
}
dump.Netns, err = a.SandboxInfos(dump.Pods)

collectHost := true
if a.podNamespace == "" && a.podName == "host" {
// only collect for host namespace
dump.Pods = nil
}
if a.podNamespace != "" && a.podName != "" {
dump.Pods = lo.Filter(dump.Pods, func(item k8s.PodNetInfo, index int) bool {
return item.PodNamespace == a.podNamespace && item.PodName == a.podName
})
collectHost = false
}

dump.Netns, err = a.SandboxInfos(dump.Pods, collectHost)
if err != nil {
return nil, fmt.Errorf("error get sandboxs info, %v", err)
}

return dump, nil
}

func NewCollector() (collector.Collector, error) {
pc := &podCollector{}
func NewCollector(podNamespace, podName, runtimeEndpoint string) (collector.Collector, error) {
pc := &podCollector{podNamespace: podNamespace, podName: podName, runtimeEndpoint: runtimeEndpoint}

socket := os.Getenv("RUNTIME_SOCK")
if runtimeEndpoint != "" {
socket = runtimeEndpoint
}
if socket == "" {
socket = "unix:///var/run/dockershim.sock"
_, err := os.Stat("/var/run/dockershim.sock")
Expand Down Expand Up @@ -140,13 +160,15 @@ func (a *podCollector) PodList() ([]k8s.PodNetInfo, error) {
return pods, nil
}

func (a *podCollector) SandboxInfos(pods []k8s.PodNetInfo) ([]netstack.NetNSInfo, error) {
func (a *podCollector) SandboxInfos(pods []k8s.PodNetInfo, collectHostNs bool) ([]netstack.NetNSInfo, error) {
var sandboxInfos []netstack.NetNSInfo
hostNsInfo, err := a.SandboxInfo("/proc/1/ns/net", "", 1)
if err != nil {
return nil, err
if collectHostNs {
hostNsInfo, err := a.SandboxInfo("/proc/1/ns/net", "", 1)
if err != nil {
return nil, err
}
sandboxInfos = append(sandboxInfos, hostNsInfo)
}
sandboxInfos = append(sandboxInfos, hostNsInfo)
for _, p := range pods {
if p.NetworkMode == "none" {
nsInfo, err := a.SandboxInfo(p.Netns, fmt.Sprintf("%s/%s", p.PodNamespace, p.PodName), p.PID)
Expand Down Expand Up @@ -220,8 +242,34 @@ func getFileInode(path string) (string, error) {
return strconv.FormatUint(fileInfo.Ino, 10), nil
}

// NSExec nsenter {pid} {command}
func NSExec(args []string) (string, error) {
if len(args) < 3 {
return "", fmt.Errorf("command args invalid: %v", args)
}
fd, err := unix.Open(fmt.Sprintf("/proc/%s/ns/net", args[1]), unix.O_RDONLY|unix.O_CLOEXEC, 0)
if err != nil {
return "", fmt.Errorf("cannot get pid: %v", err)
}
defer func() {
unix.Close(fd)
}()

err = unix.Setns(fd, unix.CLONE_NEWNET)
if err != nil {
return "", fmt.Errorf("cannot get pid from pid: %v", err)
}
output, err := exec.Command(args[2], args[3:]...).CombinedOutput()
if err != nil {
return "", fmt.Errorf("err:%v, output: %v", err, string(output))
}
return string(output), nil
}

func namespaceCmd(pid uint32, cmd string) (string, error) {
output, err := exec.Command("nsenter", "-t", strconv.Itoa(int(pid)), "--net", "--", "sh", "-c", cmd).CombinedOutput()
cmdExec := exec.Command("nsenter", strconv.Itoa(int(pid)), "sh", "-c", cmd)
cmdExec.Path = "/proc/self/exe"
output, err := cmdExec.CombinedOutput()
if err != nil {
return "", fmt.Errorf("err:%v, output: %v", err, string(output))
}
Expand Down