fix(eks): sporadic broken pipe when deploying helm charts (#6522)

fix(eks): sporadic broken pipe when deploying helm charts (#6522) Retry three times if helm fails if a “broken pipe” error. Fixes #6381
aws · Mar 10, 2020 · 03df1f1 · 03df1f1
1 parent 1de2952
commit 03df1f1
Show file tree

Hide file tree

Showing 2 changed files with 50 additions and 40 deletions.
diff --git a/packages/@aws-cdk/aws-eks/lib/kubectl-handler/helm/__init__.py b/packages/@aws-cdk/aws-eks/lib/kubectl-handler/helm/__init__.py
@@ -54,24 +54,34 @@ def helm_handler(event, context):
 
 def helm(verb, release, chart = None, repo = None, file = None, namespace = None, version = None, wait = False):
     import subprocess
-    try:
-        cmnd = ['helm', verb, release]
-        if not chart is None:
-            cmnd.append(chart)
-        if verb == 'upgrade':
-            cmnd.append('--install')
-        if not repo is None:
-            cmnd.extend(['--repo', repo])
-        if not file is None:
-            cmnd.extend(['--values', file])
-        if not version is None:
-            cmnd.extend(['--version', version])
-        if not namespace is None:
-            cmnd.extend(['--namespace', namespace])
-        if wait:
-            cmnd.append('--wait')
-        cmnd.extend(['--kubeconfig', kubeconfig])
-        output = subprocess.check_output(cmnd, stderr=subprocess.STDOUT, cwd=outdir)
-        logger.info(output)
-    except subprocess.CalledProcessError as exc:
-        raise Exception(exc.output)
+
+    cmnd = ['helm', verb, release]
+    if not chart is None:
+        cmnd.append(chart)
+    if verb == 'upgrade':
+        cmnd.append('--install')
+    if not repo is None:
+        cmnd.extend(['--repo', repo])
+    if not file is None:
+        cmnd.extend(['--values', file])
+    if not version is None:
+        cmnd.extend(['--version', version])
+    if not namespace is None:
+        cmnd.extend(['--namespace', namespace])
+    if wait:
+        cmnd.append('--wait')
+    cmnd.extend(['--kubeconfig', kubeconfig])
+
+    retry = 3
+    while retry > 0:
+        try:
+            output = subprocess.check_output(cmnd, stderr=subprocess.STDOUT, cwd=outdir)
+            logger.info(output)
+            return
+        except subprocess.CalledProcessError as exc:
+            output = exc.output
+            if b'Broken pipe' in output:
+                logger.info("Broken pipe, retries left: %s" % retry)
+                retry = retry - 1
+            else:
+                raise Exception(output)
diff --git a/packages/@aws-cdk/aws-eks/test/integ.eks-cluster.expected.json b/packages/@aws-cdk/aws-eks/test/integ.eks-cluster.expected.json
@@ -1754,7 +1754,7 @@
               },
               "/",
               {
-                "Ref": "AssetParameters1e3bad756cb56d3cc78162a37a1a4b0226bdc7176cf3ed9128b0241f2799d03eS3BucketA606F9D6"
+                "Ref": "AssetParameters5736fa6dc98806541544f2c33f17a2495fe0723bd1e59ec62991d68ac3a6e690S3BucketE6BD216D"
               },
               "/",
               {
@@ -1764,7 +1764,7 @@
                     "Fn::Split": [
                       "||",
                       {
-                        "Ref": "AssetParameters1e3bad756cb56d3cc78162a37a1a4b0226bdc7176cf3ed9128b0241f2799d03eS3VersionKeyFE7BFE80"
+                        "Ref": "AssetParameters5736fa6dc98806541544f2c33f17a2495fe0723bd1e59ec62991d68ac3a6e690S3VersionKey27D33001"
                       }
                     ]
                   }
@@ -1777,7 +1777,7 @@
                     "Fn::Split": [
                       "||",
                       {
-                        "Ref": "AssetParameters1e3bad756cb56d3cc78162a37a1a4b0226bdc7176cf3ed9128b0241f2799d03eS3VersionKeyFE7BFE80"
+                        "Ref": "AssetParameters5736fa6dc98806541544f2c33f17a2495fe0723bd1e59ec62991d68ac3a6e690S3VersionKey27D33001"
                       }
                     ]
                   }
@@ -1787,11 +1787,11 @@
           ]
         },
         "Parameters": {
-          "referencetoawscdkeksclustertestAssetParameters7885982e43353796b16117389f19d46f133d2b866759b8b87c714aa408d6a25aS3Bucket6D4B3C92Ref": {
-            "Ref": "AssetParameters7885982e43353796b16117389f19d46f133d2b866759b8b87c714aa408d6a25aS3BucketF848983B"
+          "referencetoawscdkeksclustertestAssetParameters809b8ac7e88704d37fac32bbd5cfa56be7ea4d3e9ddb682d216c4b6868cd8fa2S3Bucket5C1311C2Ref": {
+            "Ref": "AssetParameters809b8ac7e88704d37fac32bbd5cfa56be7ea4d3e9ddb682d216c4b6868cd8fa2S3Bucket8A1A4BE8"
           },
-          "referencetoawscdkeksclustertestAssetParameters7885982e43353796b16117389f19d46f133d2b866759b8b87c714aa408d6a25aS3VersionKey1728B833Ref": {
-            "Ref": "AssetParameters7885982e43353796b16117389f19d46f133d2b866759b8b87c714aa408d6a25aS3VersionKeyAE869E56"
+          "referencetoawscdkeksclustertestAssetParameters809b8ac7e88704d37fac32bbd5cfa56be7ea4d3e9ddb682d216c4b6868cd8fa2S3VersionKey33922910Ref": {
+            "Ref": "AssetParameters809b8ac7e88704d37fac32bbd5cfa56be7ea4d3e9ddb682d216c4b6868cd8fa2S3VersionKeyB580A234"
           },
           "referencetoawscdkeksclustertestAssetParameters6c3e21f76e4ba0bc4b901f71bfa9c1eaf7929edcfd9a1591690d12b024100044S3Bucket24E1CF9DRef": {
             "Ref": "AssetParameters6c3e21f76e4ba0bc4b901f71bfa9c1eaf7929edcfd9a1591690d12b024100044S3Bucket75CDEB48"
@@ -1899,29 +1899,29 @@
       "Type": "String",
       "Description": "Artifact hash for asset \"6c3e21f76e4ba0bc4b901f71bfa9c1eaf7929edcfd9a1591690d12b024100044\""
     },
-    "AssetParameters7885982e43353796b16117389f19d46f133d2b866759b8b87c714aa408d6a25aS3BucketF848983B": {
+    "AssetParameters809b8ac7e88704d37fac32bbd5cfa56be7ea4d3e9ddb682d216c4b6868cd8fa2S3Bucket8A1A4BE8": {
       "Type": "String",
-      "Description": "S3 bucket for asset \"7885982e43353796b16117389f19d46f133d2b866759b8b87c714aa408d6a25a\""
+      "Description": "S3 bucket for asset \"809b8ac7e88704d37fac32bbd5cfa56be7ea4d3e9ddb682d216c4b6868cd8fa2\""
     },
-    "AssetParameters7885982e43353796b16117389f19d46f133d2b866759b8b87c714aa408d6a25aS3VersionKeyAE869E56": {
+    "AssetParameters809b8ac7e88704d37fac32bbd5cfa56be7ea4d3e9ddb682d216c4b6868cd8fa2S3VersionKeyB580A234": {
       "Type": "String",
-      "Description": "S3 key for asset version \"7885982e43353796b16117389f19d46f133d2b866759b8b87c714aa408d6a25a\""
+      "Description": "S3 key for asset version \"809b8ac7e88704d37fac32bbd5cfa56be7ea4d3e9ddb682d216c4b6868cd8fa2\""
     },
-    "AssetParameters7885982e43353796b16117389f19d46f133d2b866759b8b87c714aa408d6a25aArtifactHash741C1231": {
+    "AssetParameters809b8ac7e88704d37fac32bbd5cfa56be7ea4d3e9ddb682d216c4b6868cd8fa2ArtifactHash5CE7C76A": {
       "Type": "String",
-      "Description": "Artifact hash for asset \"7885982e43353796b16117389f19d46f133d2b866759b8b87c714aa408d6a25a\""
+      "Description": "Artifact hash for asset \"809b8ac7e88704d37fac32bbd5cfa56be7ea4d3e9ddb682d216c4b6868cd8fa2\""
     },
-    "AssetParameters1e3bad756cb56d3cc78162a37a1a4b0226bdc7176cf3ed9128b0241f2799d03eS3BucketA606F9D6": {
+    "AssetParameters5736fa6dc98806541544f2c33f17a2495fe0723bd1e59ec62991d68ac3a6e690S3BucketE6BD216D": {
       "Type": "String",
-      "Description": "S3 bucket for asset \"1e3bad756cb56d3cc78162a37a1a4b0226bdc7176cf3ed9128b0241f2799d03e\""
+      "Description": "S3 bucket for asset \"5736fa6dc98806541544f2c33f17a2495fe0723bd1e59ec62991d68ac3a6e690\""
     },
-    "AssetParameters1e3bad756cb56d3cc78162a37a1a4b0226bdc7176cf3ed9128b0241f2799d03eS3VersionKeyFE7BFE80": {
+    "AssetParameters5736fa6dc98806541544f2c33f17a2495fe0723bd1e59ec62991d68ac3a6e690S3VersionKey27D33001": {
       "Type": "String",
-      "Description": "S3 key for asset version \"1e3bad756cb56d3cc78162a37a1a4b0226bdc7176cf3ed9128b0241f2799d03e\""
+      "Description": "S3 key for asset version \"5736fa6dc98806541544f2c33f17a2495fe0723bd1e59ec62991d68ac3a6e690\""
     },
-    "AssetParameters1e3bad756cb56d3cc78162a37a1a4b0226bdc7176cf3ed9128b0241f2799d03eArtifactHashFB2BBA58": {
+    "AssetParameters5736fa6dc98806541544f2c33f17a2495fe0723bd1e59ec62991d68ac3a6e690ArtifactHash19392521": {
       "Type": "String",
-      "Description": "Artifact hash for asset \"1e3bad756cb56d3cc78162a37a1a4b0226bdc7176cf3ed9128b0241f2799d03e\""
+      "Description": "Artifact hash for asset \"5736fa6dc98806541544f2c33f17a2495fe0723bd1e59ec62991d68ac3a6e690\""
     },
     "AssetParameters6348c4414dfcbc19ed407c51ecc75d12faf4ee3219e972437d4ceed53e5b79a0S3BucketEF51ACE0": {
       "Type": "String",