diff --git a/Dockerfile b/Dockerfile index 4af4f153..e82a69ac 100644 --- a/Dockerfile +++ b/Dockerfile @@ -28,5 +28,7 @@ RUN apk add --no-cache \ COPY --from=build /go/src/github.com/alibaba/kubeskoop/bin/inspector /bin/inspector COPY --from=build /go/src/github.com/alibaba/kubeskoop/bin/pod-collector /bin/pod-collector COPY --from=build /go/src/github.com/alibaba/kubeskoop/bin/skoop /bin/skoop +COPY --from=build /go/src/github.com/alibaba/kubeskoop/bin/btfhack /bin/btfhack + COPY tools/scripts/* /bin/ COPY deploy/resource/kubeskoop-exporter-dashboard.json /etc/ diff --git a/Makefile b/Makefile index 2a8a6147..e3829ad5 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,7 @@ GIT_COMMIT=${shell git rev-parse HEAD} ldflags="-X $(VERSION_PKG).Version=$(TAG) -X $(VERSION_PKG).Commit=${GIT_COMMIT}" .PHONY: all -all: build-exporter build-skoop build-collector +all: build-exporter build-skoop build-collector build-btfhack .PHONY: fmt fmt: @@ -36,6 +36,10 @@ build-skoop: build-collector: CGO_ENABLED=0 go build -o bin/pod-collector -ldflags $(ldflags) ./cmd/collector +.PHONY: build-btfhack +build-btfhack: + CGO_ENABLED=0 go build -o bin/btfhack -ldflags $(ldflags) ./cmd/btfhack + .PHONY: image image: ## build kubeskoop image docker build -t $(SKOOP_REPO):$(TAG) . diff --git a/cmd/btfhack/main.go b/cmd/btfhack/main.go new file mode 100644 index 00000000..dcd04dfc --- /dev/null +++ b/cmd/btfhack/main.go @@ -0,0 +1,9 @@ +package main + +import ( + "github.com/alibaba/kubeskoop/pkg/exporter/btfhack" +) + +func main() { + btfhack.Execute() +} diff --git a/deploy/helm/values.yaml b/deploy/helm/values.yaml index 9ec9af5f..04cea117 100644 --- a/deploy/helm/values.yaml +++ b/deploy/helm/values.yaml @@ -11,7 +11,7 @@ image: initContainer: enabled: true - repository: registry.cn-hangzhou.aliyuncs.com/acs/btfhack + repository: kubeskoop/kubeskoop tag: latest imagePullPolicy: Always diff --git a/deploy/skoopbundle.yaml b/deploy/skoopbundle.yaml index be6c9f4f..11548b47 100644 --- a/deploy/skoopbundle.yaml +++ b/deploy/skoopbundle.yaml @@ -31,13 +31,13 @@ spec: dnsPolicy: ClusterFirstWithHostNet initContainers: - name: inspector-prepare - image: registry.cn-hangzhou.aliyuncs.com/acs/btfhack:latest + image: "kubeskoop/kubeskoop:latest" volumeMounts: - name: btf-rawdata mountPath: /etc/net-exporter/btf - mountPath: /boot/ name: boot - command: [btfhack, discover,-p ,/etc/net-exporter/btf/] + command: [btfhack, discover, -p, /etc/net-exporter/btf/] containers: - image: "kubeskoop/kubeskoop:latest" name: inspector diff --git a/pkg/exporter/bpfutil/btf.go b/pkg/exporter/bpfutil/btf.go index af7e3fd6..60a3f217 100644 --- a/pkg/exporter/bpfutil/btf.go +++ b/pkg/exporter/bpfutil/btf.go @@ -50,6 +50,15 @@ func KernelRelease() (string, error) { return unix.ByteSliceToString(uname.Release[:]), nil } +func KernelArch() (string, error) { + var uname unix.Utsname + if err := unix.Uname(&uname); err != nil { + return "", fmt.Errorf("uname failed: %w", err) + } + + return unix.ByteSliceToString(uname.Machine[:]), nil +} + // LoadBTFSpecOrNil once error occurs in load process, return nil and use system raw spec instead func LoadBTFSpecOrNil() *btf.Spec { var ( @@ -60,7 +69,7 @@ func LoadBTFSpecOrNil() *btf.Spec { btffile = kernelBTFPath } else if os.IsNotExist(err) { for _, btfPath := range []string{BTFPATH, bpfSharePath, userCustomBtfPath} { - btffile, err = findBTFFileWithPath(btfPath) + btffile, err = FindBTFFileWithPath(btfPath) if err == nil { break } @@ -85,7 +94,7 @@ func LoadBTFSpecOrNil() *btf.Spec { return spec } -func findBTFFileWithPath(path string) (string, error) { +func FindBTFFileWithPath(path string) (string, error) { path = filepath.Clean(path) v, err := KernelRelease() diff --git a/pkg/exporter/btfhack/discover.go b/pkg/exporter/btfhack/discover.go new file mode 100644 index 00000000..d03ea168 --- /dev/null +++ b/pkg/exporter/btfhack/discover.go @@ -0,0 +1,70 @@ +package btfhack + +import ( + "fmt" + "log" + "os/exec" + + "github.com/alibaba/kubeskoop/pkg/exporter/bpfutil" + + "github.com/spf13/cobra" +) + +const ( + defaultBTFDstPath = "/etc/net-exporter/btf" +) + +// cpCmd represents the cp command +var ( + cpCmd = &cobra.Command{ + Use: "discover", + Short: "copy or download appropriate btf file to dst path", + Run: func(cmd *cobra.Command, args []string) { + if btfSrcPath == "" { + btfSrcPath = defaultBTFPath + } + if btfDstPath == "" { + btfDstPath = defaultBTFDstPath + } + + btffile, err := bpfutil.FindBTFFileWithPath(btfSrcPath) + if err == nil { + err := copyBtfFile(btffile, btfDstPath) + if err != nil { + log.Fatalf("Failed copy btf file: %s\n", err) + } + log.Printf("Copy btf file %s to %s succeed\n", btffile, btfDstPath) + return + } + + btffile, err = downloadBTFOnline(btfDstPath) + if err != nil { + log.Printf("Download btf error: %s\n", err) + return + } + log.Printf("Download btf file %s succeed\n", btffile) + }, + } + + btfDstPath string +) + +func copyBtfFile(path, dstPath string) error { + cmdToExecute := exec.Command("cp", path, dstPath) + output, err := cmdToExecute.CombinedOutput() + if err != nil { + return fmt.Errorf("load btf with:%s err:%s", output, err) + } + + log.Printf("load btf %s to %s succeed", path, dstPath) + return nil +} + +func init() { + rootCmd.AddCommand(cpCmd) + + flags := cpCmd.PersistentFlags() + + flags.StringVarP(&btfSrcPath, "src", "s", "", "btf source file") + flags.StringVarP(&btfDstPath, "dst", "p", "", "btf destination directory") +} diff --git a/pkg/exporter/btfhack/download.go b/pkg/exporter/btfhack/download.go new file mode 100644 index 00000000..10837407 --- /dev/null +++ b/pkg/exporter/btfhack/download.go @@ -0,0 +1,90 @@ +package btfhack + +import ( + "fmt" + "io" + "log" + "net" + "net/http" + "net/url" + "os" + "path" + "time" + + "github.com/alibaba/kubeskoop/pkg/exporter/bpfutil" +) + +const ( + EnvBTFDownloadURL = "BTF_DOWNLOAD_URL" + OpenBTFURL = "https://mirrors.openanolis.cn/coolbpf/btf/" +) + +func downloadBTFOnline(btfDstPath string) (string, error) { + release, err := bpfutil.KernelRelease() + if err != nil { + return "", err + } + arch, err := bpfutil.KernelArch() + if err != nil { + return "", err + } + + filename := fmt.Sprintf("vmlinux-%s", release) + dst := path.Join(btfDstPath, filename) + urlPath := fmt.Sprintf("%s/%s", arch, filename) + if envURL, ok := os.LookupEnv(EnvBTFDownloadURL); ok { + downloadURL, err := url.JoinPath(envURL, urlPath) + if err != nil { + return "", err + } + err = downloadTo(downloadURL, dst) + if err == nil { + log.Printf("Downloaded btf file from %s", downloadURL) + return dst, nil + } + log.Printf("Download btf file failed from %s: %s", downloadURL, err) + } + + downloadURL, err := url.JoinPath(OpenBTFURL, urlPath) + if err != nil { + return "", err + } + err = downloadTo(downloadURL, dst) + if err != nil { + return "", fmt.Errorf("download btf file failed from %s: %w", downloadURL, err) + } + return dst, nil +} + +func downloadTo(url, dst string) error { + tr := &http.Transport{ + Dial: (&net.Dialer{ + Timeout: 1 * time.Second, + KeepAlive: 30 * time.Second, + }).Dial, + } + + client := http.Client{ + Timeout: 50 * time.Second, + Transport: tr, + } + + res, err := client.Get(url) + if err != nil { + return err + } + defer res.Body.Close() + if res.StatusCode != http.StatusOK { + return fmt.Errorf("got status code %d", res.StatusCode) + } + + f, err := os.Create(dst) + if err != nil { + return err + } + _, err = io.Copy(f, res.Body) + if err != nil { + return err + } + return nil +} diff --git a/pkg/exporter/btfhack/root.go b/pkg/exporter/btfhack/root.go new file mode 100644 index 00000000..4818a0b5 --- /dev/null +++ b/pkg/exporter/btfhack/root.go @@ -0,0 +1,23 @@ +package btfhack + +import ( + "os" + + "github.com/spf13/cobra" +) + +const ( + defaultBTFPath = "/etc/btf" +) + +var rootCmd = &cobra.Command{ + Use: "btfhack", + Short: "A tool to automatically discover btf file from local path or online", +} + +func Execute() { + err := rootCmd.Execute() + if err != nil { + os.Exit(1) + } +} diff --git a/pkg/exporter/btfhack/test.go b/pkg/exporter/btfhack/test.go new file mode 100644 index 00000000..2587d05e --- /dev/null +++ b/pkg/exporter/btfhack/test.go @@ -0,0 +1,55 @@ +package btfhack + +import ( + "log" + + "github.com/alibaba/kubeskoop/pkg/exporter/bpfutil" + "github.com/alibaba/kubeskoop/pkg/exporter/testbtf" + + "github.com/cilium/ebpf/btf" + "github.com/spf13/cobra" +) + +// testCmd represents the test command +var ( + testCmd = &cobra.Command{ + Use: "test", + Short: "test btf support locally", + Run: func(cmd *cobra.Command, args []string) { + if btfSrcPath == "" { + btfSrcPath = defaultBTFPath + } + + file, err := bpfutil.FindBTFFileWithPath(btfSrcPath) + if err != nil { + log.Printf("failed with %s", err) + return + } + + spec, err := bpfutil.LoadBTFFromFile(file) + if err != nil { + log.Printf("load btf spec faiild with %s", err) + return + } + + if err := testBTFAvailable(spec); err != nil { + log.Printf("btf test failed: %v", err) + } else { + log.Printf("btf test ok") + } + }, + } + + btfSrcPath string +) + +func init() { + rootCmd.AddCommand(testCmd) + flags := testCmd.PersistentFlags() + + flags.StringVarP(&btfSrcPath, "src", "s", "", "btf source file") +} + +func testBTFAvailable(spec *btf.Spec) error { + return testbtf.RunBTFTest(spec) +} diff --git a/pkg/exporter/btfhack/version.go b/pkg/exporter/btfhack/version.go new file mode 100644 index 00000000..bf14ae4d --- /dev/null +++ b/pkg/exporter/btfhack/version.go @@ -0,0 +1,20 @@ +package btfhack + +import ( + "github.com/alibaba/kubeskoop/version" + "github.com/spf13/cobra" +) + +var ( + versionCmd = &cobra.Command{ + Use: "version", + Short: "show version", + Run: func(_ *cobra.Command, args []string) { + version.PrintVersion() + }, + } +) + +func init() { + rootCmd.AddCommand(versionCmd) +} diff --git a/pkg/exporter/testbtf/bpf_bpfeb.go b/pkg/exporter/testbtf/bpf_bpfeb.go new file mode 100644 index 00000000..df197592 --- /dev/null +++ b/pkg/exporter/testbtf/bpf_bpfeb.go @@ -0,0 +1,119 @@ +// Code generated by bpf2go; DO NOT EDIT. +//go:build arm64be || armbe || mips || mips64 || mips64p32 || ppc64 || s390 || s390x || sparc || sparc64 +// +build arm64be armbe mips mips64 mips64p32 ppc64 s390 s390x sparc sparc64 + +package testbtf + +import ( + "bytes" + _ "embed" + "fmt" + "io" + + "github.com/cilium/ebpf" +) + +// loadBpf returns the embedded CollectionSpec for bpf. +func loadBpf() (*ebpf.CollectionSpec, error) { + reader := bytes.NewReader(_BpfBytes) + spec, err := ebpf.LoadCollectionSpecFromReader(reader) + if err != nil { + return nil, fmt.Errorf("can't load bpf: %w", err) + } + + return spec, err +} + +// loadBpfObjects loads bpf and converts it into a struct. +// +// The following types are suitable as obj argument: +// +// *bpfObjects +// *bpfPrograms +// *bpfMaps +// +// See ebpf.CollectionSpec.LoadAndAssign documentation for details. +func loadBpfObjects(obj interface{}, opts *ebpf.CollectionOptions) error { + spec, err := loadBpf() + if err != nil { + return err + } + + return spec.LoadAndAssign(obj, opts) +} + +// bpfSpecs contains maps and programs before they are loaded into the kernel. +// +// It can be passed ebpf.CollectionSpec.Assign. +type bpfSpecs struct { + bpfProgramSpecs + bpfMapSpecs +} + +// bpfSpecs contains programs before they are loaded into the kernel. +// +// It can be passed ebpf.CollectionSpec.Assign. +type bpfProgramSpecs struct { + KprobeExecve *ebpf.ProgramSpec `ebpf:"kprobe_execve"` +} + +// bpfMapSpecs contains maps before they are loaded into the kernel. +// +// It can be passed ebpf.CollectionSpec.Assign. +type bpfMapSpecs struct { + KprobeMap *ebpf.MapSpec `ebpf:"kprobe_map"` +} + +// bpfObjects contains all objects after they have been loaded into the kernel. +// +// It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign. +type bpfObjects struct { + bpfPrograms + bpfMaps +} + +func (o *bpfObjects) Close() error { + return _BpfClose( + &o.bpfPrograms, + &o.bpfMaps, + ) +} + +// bpfMaps contains all maps after they have been loaded into the kernel. +// +// It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign. +type bpfMaps struct { + KprobeMap *ebpf.Map `ebpf:"kprobe_map"` +} + +func (m *bpfMaps) Close() error { + return _BpfClose( + m.KprobeMap, + ) +} + +// bpfPrograms contains all programs after they have been loaded into the kernel. +// +// It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign. +type bpfPrograms struct { + KprobeExecve *ebpf.Program `ebpf:"kprobe_execve"` +} + +func (p *bpfPrograms) Close() error { + return _BpfClose( + p.KprobeExecve, + ) +} + +func _BpfClose(closers ...io.Closer) error { + for _, closer := range closers { + if err := closer.Close(); err != nil { + return err + } + } + return nil +} + +// Do not access this directly. +//go:embed bpf_bpfeb.o +var _BpfBytes []byte diff --git a/pkg/exporter/testbtf/bpf_bpfeb.o b/pkg/exporter/testbtf/bpf_bpfeb.o new file mode 100644 index 00000000..58fd2128 Binary files /dev/null and b/pkg/exporter/testbtf/bpf_bpfeb.o differ diff --git a/pkg/exporter/testbtf/bpf_bpfel.go b/pkg/exporter/testbtf/bpf_bpfel.go new file mode 100644 index 00000000..d5fa572c --- /dev/null +++ b/pkg/exporter/testbtf/bpf_bpfel.go @@ -0,0 +1,119 @@ +// Code generated by bpf2go; DO NOT EDIT. +//go:build 386 || amd64 || amd64p32 || arm || arm64 || mips64le || mips64p32le || mipsle || ppc64le || riscv64 +// +build 386 amd64 amd64p32 arm arm64 mips64le mips64p32le mipsle ppc64le riscv64 + +package testbtf + +import ( + "bytes" + _ "embed" + "fmt" + "io" + + "github.com/cilium/ebpf" +) + +// loadBpf returns the embedded CollectionSpec for bpf. +func loadBpf() (*ebpf.CollectionSpec, error) { + reader := bytes.NewReader(_BpfBytes) + spec, err := ebpf.LoadCollectionSpecFromReader(reader) + if err != nil { + return nil, fmt.Errorf("can't load bpf: %w", err) + } + + return spec, err +} + +// loadBpfObjects loads bpf and converts it into a struct. +// +// The following types are suitable as obj argument: +// +// *bpfObjects +// *bpfPrograms +// *bpfMaps +// +// See ebpf.CollectionSpec.LoadAndAssign documentation for details. +func loadBpfObjects(obj interface{}, opts *ebpf.CollectionOptions) error { + spec, err := loadBpf() + if err != nil { + return err + } + + return spec.LoadAndAssign(obj, opts) +} + +// bpfSpecs contains maps and programs before they are loaded into the kernel. +// +// It can be passed ebpf.CollectionSpec.Assign. +type bpfSpecs struct { + bpfProgramSpecs + bpfMapSpecs +} + +// bpfSpecs contains programs before they are loaded into the kernel. +// +// It can be passed ebpf.CollectionSpec.Assign. +type bpfProgramSpecs struct { + KprobeExecve *ebpf.ProgramSpec `ebpf:"kprobe_execve"` +} + +// bpfMapSpecs contains maps before they are loaded into the kernel. +// +// It can be passed ebpf.CollectionSpec.Assign. +type bpfMapSpecs struct { + KprobeMap *ebpf.MapSpec `ebpf:"kprobe_map"` +} + +// bpfObjects contains all objects after they have been loaded into the kernel. +// +// It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign. +type bpfObjects struct { + bpfPrograms + bpfMaps +} + +func (o *bpfObjects) Close() error { + return _BpfClose( + &o.bpfPrograms, + &o.bpfMaps, + ) +} + +// bpfMaps contains all maps after they have been loaded into the kernel. +// +// It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign. +type bpfMaps struct { + KprobeMap *ebpf.Map `ebpf:"kprobe_map"` +} + +func (m *bpfMaps) Close() error { + return _BpfClose( + m.KprobeMap, + ) +} + +// bpfPrograms contains all programs after they have been loaded into the kernel. +// +// It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign. +type bpfPrograms struct { + KprobeExecve *ebpf.Program `ebpf:"kprobe_execve"` +} + +func (p *bpfPrograms) Close() error { + return _BpfClose( + p.KprobeExecve, + ) +} + +func _BpfClose(closers ...io.Closer) error { + for _, closer := range closers { + if err := closer.Close(); err != nil { + return err + } + } + return nil +} + +// Do not access this directly. +//go:embed bpf_bpfel.o +var _BpfBytes []byte diff --git a/pkg/exporter/testbtf/bpf_bpfel.o b/pkg/exporter/testbtf/bpf_bpfel.o new file mode 100644 index 00000000..862c3277 Binary files /dev/null and b/pkg/exporter/testbtf/bpf_bpfel.o differ diff --git a/pkg/exporter/testbtf/btf.go b/pkg/exporter/testbtf/btf.go new file mode 100644 index 00000000..561c9988 --- /dev/null +++ b/pkg/exporter/testbtf/btf.go @@ -0,0 +1,56 @@ +package testbtf + +import ( + "log" + + "github.com/cilium/ebpf" + "github.com/cilium/ebpf/btf" + "github.com/cilium/ebpf/link" + "github.com/cilium/ebpf/rlimit" +) + +const mapKey uint32 = 0 + +//go:generate go run github.com/cilium/ebpf/cmd/bpf2go -cc clang bpf ../../bpf/kprobe.c -- -I../../bpf/headers + +func btfTest(btf *btf.Spec) error { + fn := "sys_execve" + + // Allow the current process to lock memory for eBPF resources. + if err := rlimit.RemoveMemlock(); err != nil { + log.Printf("set mem limit:%s", err) + return err + } + + opts := ebpf.CollectionOptions{ + Programs: ebpf.ProgramOptions{ + KernelTypes: btf, + }, + } + + objs := bpfObjects{} + if err := loadBpfObjects(&objs, &opts); err != nil { + log.Printf("loading objects: %v", err) + return err + } + defer objs.Close() + + kp, err := link.Kprobe(fn, objs.KprobeExecve, nil) + if err != nil { + log.Printf("opening kprobe: %s", err) + return err + } + defer kp.Close() + + var value uint64 + if err := objs.KprobeMap.Lookup(mapKey, &value); err != nil { + log.Printf("reading map: %v", err) + return err + } + log.Printf("%s called %d times\n", fn, value) + return nil +} + +func RunBTFTest(btf *btf.Spec) error { + return btfTest(btf) +}