From 5ae50ec441dbb8ebc4a4f9d88b42c025ecaa24aa Mon Sep 17 00:00:00 2001 From: Will Thames Date: Fri, 13 Oct 2023 17:10:59 +1000 Subject: [PATCH 1/3] Don't error if log reconnection fails due to context canceled Don't retry either as the context won't become uncanceled --- pkg/workceptor/kubernetes.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/pkg/workceptor/kubernetes.go b/pkg/workceptor/kubernetes.go index 14d277b0d..7e6b32312 100644 --- a/pkg/workceptor/kubernetes.go +++ b/pkg/workceptor/kubernetes.go @@ -225,7 +225,6 @@ func (kw *kubeUnit) kubeLoggingWithReconnect(streamWait *sync.WaitGroup, stdout errMsg := fmt.Sprintf("Error getting pod %s/%s. Error: %s", podNamespace, podName, err) kw.Error(errMsg) kw.UpdateBasicStatus(WorkStateFailed, errMsg, 0) - break } @@ -239,6 +238,13 @@ func (kw *kubeUnit) kubeLoggingWithReconnect(streamWait *sync.WaitGroup, stdout for *stdinErr == nil { // check between every line read to see if we need to stop reading line, err := streamReader.ReadString('\n') if err != nil { + if kw.ctx.Err() == context.Canceled { + kw.Info( + "Context was canceled while reading logs for pod %s/%s. Assuming pod has finished", + podNamespace, + podName) + return + } kw.Info( "Detected Error: %s for pod %s/%s. Will retry %d more times.", err, From 8210706ff6aff1cf83610cae0bb46572c54f297d Mon Sep 17 00:00:00 2001 From: AaronH88 Date: Fri, 13 Oct 2023 11:52:05 +0100 Subject: [PATCH 2/3] Update kubernetes.go --- pkg/workceptor/kubernetes.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/workceptor/kubernetes.go b/pkg/workceptor/kubernetes.go index 7e6b32312..05653381e 100644 --- a/pkg/workceptor/kubernetes.go +++ b/pkg/workceptor/kubernetes.go @@ -225,6 +225,7 @@ func (kw *kubeUnit) kubeLoggingWithReconnect(streamWait *sync.WaitGroup, stdout errMsg := fmt.Sprintf("Error getting pod %s/%s. Error: %s", podNamespace, podName, err) kw.Error(errMsg) kw.UpdateBasicStatus(WorkStateFailed, errMsg, 0) + break } @@ -243,6 +244,7 @@ func (kw *kubeUnit) kubeLoggingWithReconnect(streamWait *sync.WaitGroup, stdout "Context was canceled while reading logs for pod %s/%s. Assuming pod has finished", podNamespace, podName) + return } kw.Info( From 8becd14128d05a8673ac8b0dbe3939cd0d054132 Mon Sep 17 00:00:00 2001 From: AaronH88 Date: Fri, 13 Oct 2023 11:55:05 +0100 Subject: [PATCH 3/3] Fixing linting errors --- pkg/workceptor/kubernetes.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/workceptor/kubernetes.go b/pkg/workceptor/kubernetes.go index 05653381e..14c279139 100644 --- a/pkg/workceptor/kubernetes.go +++ b/pkg/workceptor/kubernetes.go @@ -225,7 +225,7 @@ func (kw *kubeUnit) kubeLoggingWithReconnect(streamWait *sync.WaitGroup, stdout errMsg := fmt.Sprintf("Error getting pod %s/%s. Error: %s", podNamespace, podName, err) kw.Error(errMsg) kw.UpdateBasicStatus(WorkStateFailed, errMsg, 0) - + break } @@ -244,7 +244,7 @@ func (kw *kubeUnit) kubeLoggingWithReconnect(streamWait *sync.WaitGroup, stdout "Context was canceled while reading logs for pod %s/%s. Assuming pod has finished", podNamespace, podName) - + return } kw.Info(