From db32f982d1c881d011904940826523e4155d03b7 Mon Sep 17 00:00:00 2001 From: Peter Rifel Date: Mon, 1 Jan 2024 13:10:25 -0600 Subject: [PATCH 1/3] Remove unused cluster name param Signed-off-by: Peter Rifel --- cmd/kops/toolbox_dump.go | 2 +- pkg/dump/resourcedumper.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/kops/toolbox_dump.go b/cmd/kops/toolbox_dump.go index 5e35c615f1167..557c2ae66fc09 100644 --- a/cmd/kops/toolbox_dump.go +++ b/cmd/kops/toolbox_dump.go @@ -228,7 +228,7 @@ func RunToolboxDump(ctx context.Context, f commandutils.Factory, out io.Writer, return fmt.Errorf("error dumping nodes: %v", err) } if options.K8sResources { - dumper, err := dump.NewResourceDumper("docker-desktop", config, options.Output, options.Dir) + dumper, err := dump.NewResourceDumper(config, options.Output, options.Dir) if err != nil { return fmt.Errorf("error creating resource dumper: %w", err) } diff --git a/pkg/dump/resourcedumper.go b/pkg/dump/resourcedumper.go index 73ac71dbcaa21..60bb3a8a0bb8d 100644 --- a/pkg/dump/resourcedumper.go +++ b/pkg/dump/resourcedumper.go @@ -72,7 +72,7 @@ type resourceDumpResult struct { err error } -func NewResourceDumper(clusterName string, k8sConfig *rest.Config, output, artifactsDir string) (*resourceDumper, error) { +func NewResourceDumper(k8sConfig *rest.Config, output, artifactsDir string) (*resourceDumper, error) { k8sConfig.QPS = 50 k8sConfig.Burst = 100 dynamicClient, err := dynamic.NewForConfig(k8sConfig) From 8ad06619759f5923f8ba1ef40c7eeb6ff75ab8bb Mon Sep 17 00:00:00 2001 From: Peter Rifel Date: Mon, 1 Jan 2024 13:11:25 -0600 Subject: [PATCH 2/3] Dump pod logs in parallel --- cmd/kops/toolbox_dump.go | 8 ++ pkg/dump/podlogs.go | 162 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 170 insertions(+) create mode 100644 pkg/dump/podlogs.go diff --git a/cmd/kops/toolbox_dump.go b/cmd/kops/toolbox_dump.go index 557c2ae66fc09..9dac993860b1e 100644 --- a/cmd/kops/toolbox_dump.go +++ b/cmd/kops/toolbox_dump.go @@ -235,6 +235,14 @@ func RunToolboxDump(ctx context.Context, f commandutils.Factory, out io.Writer, if err := dumper.DumpResources(ctx); err != nil { return fmt.Errorf("error dumping resources: %w", err) } + + logDumper, err := dump.NewPodLogDumper(config, options.Dir) + if err != nil { + return fmt.Errorf("error creating pod log dumper: %w", err) + } + if err := logDumper.DumpLogs(ctx); err != nil { + return fmt.Errorf("error dumping pod logs: %w", err) + } } } diff --git a/pkg/dump/podlogs.go b/pkg/dump/podlogs.go new file mode 100644 index 0000000000000..70ae84d7a40d7 --- /dev/null +++ b/pkg/dump/podlogs.go @@ -0,0 +1,162 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dump + +import ( + "context" + "errors" + "fmt" + "os" + "path" + + v1 "k8s.io/api/core/v1" + k8sErrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/klog/v2" +) + +const ( + podLogDumpConcurrency = 20 +) + +type podLogDumper struct { + k8sClient *kubernetes.Clientset + artifactsDir string +} + +type podLogDumpResult struct { + err error +} + +func NewPodLogDumper(k8sConfig *rest.Config, artifactsDir string) (*podLogDumper, error) { + k8sConfig.QPS = 50 + k8sConfig.Burst = 100 + clientSet, err := kubernetes.NewForConfig(k8sConfig) + if err != nil { + return nil, fmt.Errorf("creating clientset: %w", err) + } + return &podLogDumper{ + k8sClient: clientSet, + artifactsDir: artifactsDir, + }, nil +} + +func (d *podLogDumper) DumpLogs(ctx context.Context) error { + klog.Info("Dumping k8s pod logs") + + allPods, err := d.k8sClient.CoreV1().Pods(v1.NamespaceAll).List(ctx, metav1.ListOptions{}) + if err != nil { + return fmt.Errorf("listing pods: %w", err) + } + + jobs := make(chan v1.Pod, len(allPods.Items)) + results := make(chan podLogDumpResult, len(allPods.Items)) + + for i := 0; i < podLogDumpConcurrency; i++ { + go d.getPodLogs(ctx, jobs, results) + } + + var dumpErr error + + for _, pod := range allPods.Items { + jobs <- pod + } + close(jobs) + + for i := 0; i < len(allPods.Items); i++ { + result := <-results + if result.err != nil { + errors.Join(dumpErr, result.err) + } + } + close(results) + return dumpErr +} + +func (d *podLogDumper) getPodLogs(ctx context.Context, pods chan v1.Pod, results chan podLogDumpResult) { + for pod := range pods { + for _, container := range pod.Spec.Containers { + resPath := path.Join(d.artifactsDir, "cluster-info", pod.Namespace, pod.Name, container.Name+".log") + + err := os.MkdirAll(path.Dir(resPath), 0755) + if err != nil { + results <- podLogDumpResult{ + err: fmt.Errorf("creating directory %q: %w", resPath, err), + } + continue + } + resFile, err := os.Create(resPath) + if err != nil { + results <- podLogDumpResult{ + err: fmt.Errorf("creating file %q: %w", resPath, err), + } + continue + } + + prevResp, err := d.k8sClient.CoreV1().Pods(pod.Namespace).GetLogs(pod.Name, &v1.PodLogOptions{Container: container.Name, Previous: true}).Do(ctx).Raw() + hasPrevious := true + var statusErr *k8sErrors.StatusError + if errors.As(err, &statusErr) { + if statusErr.ErrStatus.Code == 400 { + hasPrevious = false + } else { + results <- podLogDumpResult{ + err: fmt.Errorf("getting pod logs for %v/%v: %w", pod.Namespace, pod.Name, err), + } + continue + } + } + + resp, err := d.k8sClient.CoreV1().Pods(pod.Namespace).GetLogs(pod.Name, &v1.PodLogOptions{Container: container.Name}).Do(ctx).Raw() + if err != nil { + results <- podLogDumpResult{ + err: fmt.Errorf("getting pod logs for %v/%v: %w", pod.Namespace, pod.Name, err), + } + continue + } + + suffix := fmt.Sprintf("container %v of pod %v/%v ====\n", container.Name, pod.Namespace, pod.Name) + if hasPrevious { + contents := []byte(fmt.Sprintf("==== START logs for PREVIOUS %v", suffix)) + contents = append(contents, prevResp...) + contents = append(contents, []byte(fmt.Sprintf("==== END logs for PREVIOUS %v", suffix))...) + _, err = resFile.Write(contents) + if err != nil { + results <- podLogDumpResult{ + err: fmt.Errorf("writing pod logs for %v/%v: %w", pod.Namespace, pod.Name, err), + } + continue + } + } + contents := []byte(fmt.Sprintf("==== START logs for CURRENT %v", suffix)) + contents = append(contents, resp...) + contents = append(contents, []byte(fmt.Sprintf("==== END logs for CURRENT %v", suffix))...) + _, err = resFile.Write(contents) + if err != nil { + results <- podLogDumpResult{ + err: fmt.Errorf("writing pod logs for %v/%v: %w", pod.Namespace, pod.Name, err), + } + continue + } + + } + + results <- podLogDumpResult{} + } +} From f7310808677ba0046ffab5acb20137fa9a22ddcc Mon Sep 17 00:00:00 2001 From: Peter Rifel Date: Mon, 1 Jan 2024 21:51:26 -0600 Subject: [PATCH 3/3] Update podlogs.go Co-authored-by: Ciprian Hacman --- pkg/dump/podlogs.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/dump/podlogs.go b/pkg/dump/podlogs.go index 70ae84d7a40d7..572db5fd0e921 100644 --- a/pkg/dump/podlogs.go +++ b/pkg/dump/podlogs.go @@ -91,7 +91,7 @@ func (d *podLogDumper) DumpLogs(ctx context.Context) error { func (d *podLogDumper) getPodLogs(ctx context.Context, pods chan v1.Pod, results chan podLogDumpResult) { for pod := range pods { - for _, container := range pod.Spec.Containers { + for _, container := range append(pod.Spec.InitContainers, pod.Spec.Containers...) { resPath := path.Join(d.artifactsDir, "cluster-info", pod.Namespace, pod.Name, container.Name+".log") err := os.MkdirAll(path.Dir(resPath), 0755)