add nrt collector

Signed-off-by: Garrybest <garrybest@foxmail.com>

cmd/crane-agent/app/agent.go

@@ -2,7 +2,9 @@ package app
 
 import (
 	"context"
+	"encoding/json"
 	"flag"
+	"fmt"
 	"os"
 	"time"
 
@@ -17,11 +19,15 @@ import (
 	"k8s.io/client-go/kubernetes"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	"k8s.io/klog/v2"
+	kubeletconfigv1beta1 "k8s.io/kubelet/config/v1beta1"
+	kubeletconfiginternal "k8s.io/kubernetes/pkg/kubelet/apis/config"
+	kubeletscheme "k8s.io/kubernetes/pkg/kubelet/apis/config/scheme"
 	ctrl "sigs.k8s.io/controller-runtime"
 
 	ensuranceapi "github.com/gocrane/api/ensurance/v1alpha1"
 	craneclientset "github.com/gocrane/api/pkg/generated/clientset/versioned"
 	craneinformers "github.com/gocrane/api/pkg/generated/informers/externalversions"
+
 	"github.com/gocrane/crane/cmd/crane-agent/app/options"
 	"github.com/gocrane/crane/pkg/agent"
 	"github.com/gocrane/crane/pkg/metrics"
@@ -84,6 +90,10 @@ func Run(ctx context.Context, opts *options.Options) error {
 	if err != nil {
 		return err
 	}
+	kubeletConfig, err := getKubeletConfig(ctx, kubeClient, hostname)
+	if err != nil {
+		return err
+	}
 
 	podInformerFactory := informers.NewSharedInformerFactoryWithOptions(kubeClient, informerSyncPeriod,
 		informers.WithTweakListOptions(func(options *metav1.ListOptions) {
@@ -105,7 +115,7 @@ func Run(ctx context.Context, opts *options.Options) error {
 	actionInformer := craneInformerFactory.Ensurance().V1alpha1().AvoidanceActions()
 	tspInformer := craneInformerFactory.Prediction().V1alpha1().TimeSeriesPredictions()
 
-	newAgent, err := agent.NewAgent(ctx, hostname, opts.RuntimeEndpoint, opts.CgroupDriver, kubeClient, craneClient, podInformer, nodeInformer,
+	newAgent, err := agent.NewAgent(ctx, hostname, opts.RuntimeEndpoint, opts.CgroupDriver, opts.SysPath, kubeClient, craneClient, kubeletConfig, podInformer, nodeInformer,
 		nodeQOSInformer, podQOSInformer, actionInformer, tspInformer, opts.NodeResourceReserved, opts.Ifaces, healthCheck, opts.CollectInterval, opts.ExecuteExcess)
 
 	if err != nil {
@@ -124,7 +134,7 @@ func Run(ctx context.Context, opts *options.Options) error {
 	return nil
 }
 
-func buildClient() (*kubernetes.Clientset, *craneclientset.Clientset, error) {
+func buildClient() (kubernetes.Interface, craneclientset.Interface, error) {
 	config, err := ctrl.GetConfig()
 	if err != nil {
 		klog.Errorf("Failed to get GetConfig, %v.", err)
@@ -153,3 +163,38 @@ func getHostName(override string) string {
 	}
 	return nodeName
 }
+
+func getKubeletConfig(ctx context.Context, c kubernetes.Interface, hostname string) (*kubeletconfiginternal.KubeletConfiguration, error) {
+	result, err := c.CoreV1().RESTClient().Get().
+		Resource("nodes").
+		SubResource("proxy").
+		Name(hostname).
+		Suffix("configz").
+		Do(ctx).
+		Raw()
+	if err != nil {
+		return nil, err
+	}
+
+	// This hack because /configz reports the following structure:
+	// {"kubeletconfig": {the JSON representation of kubeletconfigv1beta1.KubeletConfiguration}}
+	type configzWrapper struct {
+		ComponentConfig kubeletconfigv1beta1.KubeletConfiguration `json:"kubeletconfig"`
+	}
+	configz := configzWrapper{}
+
+	if err = json.Unmarshal(result, &configz); err != nil {
+		return nil, fmt.Errorf("failed to unmarshal json for kubelet config: %v", err)
+	}
+
+	scheme, _, err := kubeletscheme.NewSchemeAndCodecs()
+	if err != nil {
+		return nil, err
+	}
+	cfg := kubeletconfiginternal.KubeletConfiguration{}
+	if err = scheme.Convert(&configz.ComponentConfig, &cfg, nil); err != nil {
+		return nil, err
+	}
+
+	return &cfg, nil
+}

cmd/crane-agent/app/options/option.go

@@ -15,6 +15,8 @@ type Options struct {
 	RuntimeEndpoint string
 	// driver that the kubelet uses to manipulate cgroups on the host (cgroupfs or systemd)
 	CgroupDriver string
+	// SysPath is th path to /sys dir.
+	SysPath string
 	// Is debug/pprof endpoint enabled
 	EnableProfiling bool
 	// BindAddr is the address the endpoint binds to.
@@ -50,6 +52,7 @@ func (o *Options) AddFlags(flags *pflag.FlagSet) {
 	flags.StringVar(&o.HostnameOverride, "hostname-override", "", "Which is the name of k8s node be used to filtered.")
 	flags.StringVar(&o.RuntimeEndpoint, "runtime-endpoint", "", "The runtime endpoint docker: unix:///var/run/dockershim.sock, containerd: unix:///run/containerd/containerd.sock, cri-o: unix:///run/crio/crio.sock, k3s: unix:///run/k3s/containerd/containerd.sock.")
 	flags.StringVar(&o.CgroupDriver, "cgroup-driver", "cgroupfs", "Driver that the kubelet uses to manipulate cgroups on the host.  Possible values: 'cgroupfs', 'systemd'. Default to 'cgroupfs'")
+	flags.StringVar(&o.SysPath, "sys-path", "/sys", "Path to /sys dir.")
 	flags.Bool("enable-profiling", false, "Is debug/pprof endpoint enabled, default: false")
 	flags.StringVar(&o.BindAddr, "bind-address", "0.0.0.0:8081", "The address the agent binds to for metrics, health-check and pprof, default: 0.0.0.0:8081.")
 	flags.DurationVar(&o.CollectInterval, "collect-interval", 10*time.Second, "Period for the state collector to collect metrics, default: 10s")

deploy/crane-agent/daemonset.yaml

@@ -46,7 +46,7 @@ spec:
               port: 8081
       restartPolicy: Always
       priorityClassName: system-node-critical
-      serviceAccount: crane-agent
+      serviceAccountName: crane-agent
       volumes:
         - hostPath:
             path: /sys

deploy/crane-agent/rbac.yaml

@@ -23,6 +23,12 @@ rules:
       - pods/eviction
     verbs:
       - create
+  - apiGroups:
+      - ""
+    resources:
+      - nodes/proxy
+    verbs:
+      - get
   - apiGroups:
       - ""
     resources:
@@ -60,6 +66,17 @@ rules:
       - create
       - update
       - patch
+  - apiGroups:
+      - "topology.crane.io"
+    resources:
+      - "*"
+    verbs:
+      - get
+      - list
+      - watch
+      - create
+      - update
+      - patch
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding

go.mod

@@ -7,6 +7,7 @@ require (
 	github.com/go-echarts/go-echarts/v2 v2.2.4
 	github.com/gocrane/api v0.7.1-0.20220906050113-0f331eb419b0
 	github.com/google/cadvisor v0.39.2
+	github.com/jaypipes/ghw v0.9.0
 	github.com/mjibson/go-dsp v0.0.0-20180508042940-11479a337f12
 	github.com/prometheus/client_golang v1.11.0
 	github.com/prometheus/common v0.26.0
@@ -25,6 +26,7 @@ require (
 	k8s.io/component-base v0.22.3
 	k8s.io/cri-api v0.22.3
 	k8s.io/klog/v2 v2.9.0
+	k8s.io/kubelet v0.22.3
 	k8s.io/kubernetes v1.22.3
 	k8s.io/metrics v0.22.3
 	sigs.k8s.io/controller-runtime v0.10.2
@@ -71,9 +73,10 @@ require (
 	github.com/emicklei/go-restful-swagger12 v0.0.0-20201014110547-68ccff494617 // indirect
 	github.com/euank/go-kmsg-parser v2.0.0+incompatible // indirect
 	github.com/felixge/httpsnoop v1.0.1 // indirect
+	github.com/ghodss/yaml v1.0.0 // indirect
 	github.com/gin-contrib/sse v0.1.0 // indirect
 	github.com/go-logr/logr v0.4.0 // indirect
-	github.com/go-ole/go-ole v1.2.5 // indirect
+	github.com/go-ole/go-ole v1.2.6 // indirect
 	github.com/go-openapi/jsonpointer v0.19.5 // indirect
 	github.com/go-openapi/jsonreference v0.19.5 // indirect
 	github.com/go-openapi/swag v0.19.14 // indirect
@@ -93,13 +96,16 @@ require (
 	github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect
 	github.com/imdario/mergo v0.3.12 // indirect
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect
+	github.com/jaypipes/pcidb v1.0.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/karrick/godirwalk v1.16.1 // indirect
 	github.com/leodido/go-urn v1.2.0 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
 	github.com/mindprince/gonvml v0.0.0-20190828220739-9ebdce4bb989 // indirect
 	github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible // indirect
+	github.com/mitchellh/go-homedir v1.1.0 // indirect
+	github.com/moby/spdystream v0.2.0 // indirect
 	github.com/moby/sys/mountinfo v0.4.1 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
@@ -142,14 +148,20 @@ require (
 	go.uber.org/zap v1.19.0 // indirect
 	golang.org/x/text v0.3.7 // indirect
 	golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect
+	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
 	gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
 	gopkg.in/warnings.v0 v0.1.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
+	howett.net/plist v1.0.0 // indirect
 	k8s.io/apiextensions-apiserver v0.22.2 // indirect
+	k8s.io/cloud-provider v0.22.3 // indirect
+	k8s.io/component-helpers v0.22.3 // indirect
+	k8s.io/kube-scheduler v0.0.0 // indirect
+	k8s.io/mount-utils v0.22.3 // indirect
 	k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a // indirect
 	sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.22 // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.1.2 // indirect
@@ -167,7 +179,7 @@ require (
 	go.etcd.io/etcd/client/v2 v2.305.1 // indirect
 	golang.org/x/crypto v0.0.0-20220112180741-5e0467b6c7ce // indirect
 	golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect
-	golang.org/x/sys v0.0.0-20211205182925-97ca703d548d // indirect
+	golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8 // indirect
 	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
 	golang.org/x/tools v0.1.8 // indirect
 	google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa // indirect