From 4e0299e4e5ae0777a832b45115c661796f14c2d0 Mon Sep 17 00:00:00 2001 From: Anatolii Bazko Date: Wed, 13 Oct 2021 14:10:55 +0300 Subject: [PATCH 1/4] fix: wait for some time when che-operator updated status with an error Signed-off-by: Anatolii Bazko --- src/constants.ts | 2 +- src/tasks/kube.ts | 28 ++++++++++++++++------------ 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/src/constants.ts b/src/constants.ts index cafad22ea..afa6a43d2 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -78,7 +78,7 @@ export const DEFAULT_ANALYTIC_HOOK_NAME = 'analytics' // Timeouts export const DEFAULT_K8S_POD_WAIT_TIMEOUT = 600000 -export const DEFAULT_K8S_POD_ERROR_RECHECK_TIMEOUT = 15000 +export const DEFAULT_K8S_POD_ERROR_RECHECK_TIMEOUT = 60000 // Custom Resources names export const CHE_CLUSTER_CRD = 'checlusters.org.eclipse.che' diff --git a/src/tasks/kube.ts b/src/tasks/kube.ts index c899d3efc..54b662fca 100644 --- a/src/tasks/kube.ts +++ b/src/tasks/kube.ts @@ -37,17 +37,19 @@ export class KubeTasks { for (let i = 1; i <= iterations; i++) { // check cheCluster status const cheClusterFailState = await this.getCheClusterFailState(namespace) - if (cheClusterFailState) { - task.title = `${task.title}...failed` - throw new Error(`Eclipse Che operator failed, reason: ${cheClusterFailState.reason}, message: ${cheClusterFailState.message}. Consider increasing error recheck timeout with --k8spoderrorrechecktimeout flag.`) - } - // check 'PodScheduled' condition const podFailState = await this.getPodFailState(namespace, selector, 'PodScheduled') - if (podFailState) { - // for instance we need some time for pvc provisioning... + + if (cheClusterFailState || podFailState) { + // wait some time to recheck await cli.wait(this.kubeHelper.podErrorRecheckTimeout) + const cheClusterFailState = await this.getCheClusterFailState(namespace) + if (cheClusterFailState) { + task.title = `${task.title}...failed` + throw new Error(`Eclipse Che operator failed, reason: ${cheClusterFailState.reason}, message: ${cheClusterFailState.message}. Consider increasing error recheck timeout with --k8spoderrorrechecktimeout flag.`) + } + const podFailState = await this.getPodFailState(namespace, selector, 'PodScheduled') if (podFailState) { task.title = `${task.title}...failed` @@ -103,15 +105,17 @@ export class KubeTasks { for (let i = 1; i <= iterations; i++) { // check cheCluster status const cheClusterFailState = await this.getCheClusterFailState(namespace) - if (cheClusterFailState) { - task.title = `${task.title}...failed` - throw new Error(`Eclipse Che operator failed, reason: ${cheClusterFailState.reason}, message: ${cheClusterFailState.message}. Consider increasing error recheck timeout with --k8spoderrorrechecktimeout flag.`) - } const failedState = await this.getContainerFailState(namespace, selector, 'Running') - if (failedState) { + if (cheClusterFailState || failedState) { await cli.wait(this.kubeHelper.podErrorRecheckTimeout) + const cheClusterFailState = await this.getCheClusterFailState(namespace) + if (cheClusterFailState) { + task.title = `${task.title}...failed` + throw new Error(`Eclipse Che operator failed, reason: ${cheClusterFailState.reason}, message: ${cheClusterFailState.message}. Consider increasing error recheck timeout with --k8spoderrorrechecktimeout flag.`) + } + const failedState = await this.getContainerFailState(namespace, selector, 'Running') if (failedState) { task.title = `${task.title}...failed` From 0d86fc5b5f655fcc50cf75ee3c6322331b69be61 Mon Sep 17 00:00:00 2001 From: Anatolii Bazko Date: Wed, 13 Oct 2021 17:06:34 +0300 Subject: [PATCH 2/4] Update README.md Signed-off-by: Anatolii Bazko --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 42f01f1fb..f3aaa569c 100644 --- a/README.md +++ b/README.md @@ -575,7 +575,7 @@ OPTIONS [default: 600000] Waiting time for Pod downloading image (in milliseconds) --k8spoderrorrechecktimeout=k8spoderrorrechecktimeout - [default: 15000] Waiting time for Pod rechecking error (in milliseconds) + [default: 60000] Waiting time for Pod rechecking error (in milliseconds) --k8spodreadytimeout=k8spodreadytimeout [default: 600000] Waiting time for Pod Ready condition (in milliseconds) @@ -738,7 +738,7 @@ OPTIONS --k8spoddownloadimagetimeout=k8spoddownloadimagetimeout [default: 600000] Waiting time for Pod downloading image (in milliseconds) - --k8spoderrorrechecktimeout=k8spoderrorrechecktimeout [default: 15000] Waiting time for Pod rechecking error (in + --k8spoderrorrechecktimeout=k8spoderrorrechecktimeout [default: 60000] Waiting time for Pod rechecking error (in milliseconds) --k8spodreadytimeout=k8spodreadytimeout [default: 600000] Waiting time for Pod Ready condition (in From 75ab2e729692c6358f3a547d0479aed0f779bb05 Mon Sep 17 00:00:00 2001 From: Anatolii Bazko Date: Tue, 19 Oct 2021 11:10:50 +0300 Subject: [PATCH 3/4] Improvents (not to wait a minute) Signed-off-by: Anatolii Bazko --- src/tasks/kube.ts | 56 ++++++++++++++++++++++++++++++++++++----------- 1 file changed, 43 insertions(+), 13 deletions(-) diff --git a/src/tasks/kube.ts b/src/tasks/kube.ts index 54b662fca..b82a54deb 100644 --- a/src/tasks/kube.ts +++ b/src/tasks/kube.ts @@ -11,9 +11,9 @@ */ import { cli } from 'cli-ux' import * as Listr from 'listr' - import { KubeHelper } from '../api/kube' + interface FailState { reason?: string message?: string @@ -41,16 +41,26 @@ export class KubeTasks { const podFailState = await this.getPodFailState(namespace, selector, 'PodScheduled') if (cheClusterFailState || podFailState) { - // wait some time to recheck - await cli.wait(this.kubeHelper.podErrorRecheckTimeout) + const iterations = this.kubeHelper.podErrorRecheckTimeout / 1000 + let cheClusterFailState: FailState | undefined + let podFailState: FailState | undefined + + for (let j = 0; j < iterations; j++) { + await cli.wait(1000) + + cheClusterFailState = await this.getCheClusterFailState(namespace) + podFailState = await this.getPodFailState(namespace, selector, 'PodScheduled') - const cheClusterFailState = await this.getCheClusterFailState(namespace) - if (cheClusterFailState) { + if (!cheClusterFailState && !podFailState) { + break + } + } + + if (cheClusterFailState) { task.title = `${task.title}...failed` throw new Error(`Eclipse Che operator failed, reason: ${cheClusterFailState.reason}, message: ${cheClusterFailState.message}. Consider increasing error recheck timeout with --k8spoderrorrechecktimeout flag.`) } - const podFailState = await this.getPodFailState(namespace, selector, 'PodScheduled') if (podFailState) { task.title = `${task.title}...failed` throw new Error(`Failed to schedule a pod, reason: ${podFailState.reason}, message: ${podFailState.message}. Consider increasing error recheck timeout with --k8spoderrorrechecktimeout flag.`) @@ -76,9 +86,19 @@ export class KubeTasks { for (let i = 1; i <= iterations; i++) { const failedState = await this.getContainerFailState(namespace, selector, 'Pending') if (failedState) { - await cli.wait(this.kubeHelper.podErrorRecheckTimeout) + const iterations = this.kubeHelper.podErrorRecheckTimeout / 1000 + let failedState: FailState | undefined + + for (let j = 0; j < iterations; j++) { + await cli.wait(1000) + + failedState = await this.getContainerFailState(namespace, selector, 'Pending') + + if (!failedState) { + break + } + } - const failedState = await this.getContainerFailState(namespace, selector, 'Pending') if (failedState) { task.title = `${task.title}...failed` throw new Error(`Failed to download image, reason: ${failedState.reason}, message: ${failedState.message}.`) @@ -105,18 +125,28 @@ export class KubeTasks { for (let i = 1; i <= iterations; i++) { // check cheCluster status const cheClusterFailState = await this.getCheClusterFailState(namespace) - const failedState = await this.getContainerFailState(namespace, selector, 'Running') if (cheClusterFailState || failedState) { - await cli.wait(this.kubeHelper.podErrorRecheckTimeout) + const iterations = this.kubeHelper.podErrorRecheckTimeout / 1000 + let cheClusterFailState: FailState | undefined + let failedState: FailState | undefined + + for (let j = 0; j < iterations; j++) { + await cli.wait(1000) + + cheClusterFailState = await this.getCheClusterFailState(namespace) + failedState = await this.getContainerFailState(namespace, selector, 'Running') + + if (!cheClusterFailState && !failedState) { + break + } + } - const cheClusterFailState = await this.getCheClusterFailState(namespace) - if (cheClusterFailState) { + if (cheClusterFailState) { task.title = `${task.title}...failed` throw new Error(`Eclipse Che operator failed, reason: ${cheClusterFailState.reason}, message: ${cheClusterFailState.message}. Consider increasing error recheck timeout with --k8spoderrorrechecktimeout flag.`) } - const failedState = await this.getContainerFailState(namespace, selector, 'Running') if (failedState) { task.title = `${task.title}...failed` throw new Error(`Failed to start a pod, reason: ${failedState.reason}, message: ${failedState.message}`) From f311475c10104e0d5eef3ab5cfd8534c22b4e86a Mon Sep 17 00:00:00 2001 From: Anatolii Bazko Date: Tue, 19 Oct 2021 11:16:30 +0300 Subject: [PATCH 4/4] Fix indentation Signed-off-by: Anatolii Bazko --- src/tasks/kube.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/tasks/kube.ts b/src/tasks/kube.ts index b82a54deb..65d2b2bd1 100644 --- a/src/tasks/kube.ts +++ b/src/tasks/kube.ts @@ -13,7 +13,6 @@ import { cli } from 'cli-ux' import * as Listr from 'listr' import { KubeHelper } from '../api/kube' - interface FailState { reason?: string message?: string