From 782f15bd4580e707aa6db82aa643a52bc8dd764e Mon Sep 17 00:00:00 2001 From: vela Date: Tue, 16 Jul 2024 14:22:30 -0700 Subject: [PATCH 1/3] add retry to avoid fetching token failed with some transient issues --- nodeadm/internal/aws/ecr/ecr.go | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/nodeadm/internal/aws/ecr/ecr.go b/nodeadm/internal/aws/ecr/ecr.go index 74fcef3fe..fc7ec4d55 100644 --- a/nodeadm/internal/aws/ecr/ecr.go +++ b/nodeadm/internal/aws/ecr/ecr.go @@ -19,12 +19,15 @@ func GetAuthorizationToken(awsRegion string) (string, error) { return "", err } ecrClient := ecr.NewFromConfig(awsConfig) - token, err := ecrClient.GetAuthorizationToken(context.Background(), &ecr.GetAuthorizationTokenInput{}) - if err != nil { - return "", err + + for i := 0; i < 3; i++ { + token, err := ecrClient.GetAuthorizationToken(context.Background(), &ecr.GetAuthorizationTokenInput{}) + if err == nil { + authData := token.AuthorizationData[0].AuthorizationToken + return *authData, nil + } } - authData := token.AuthorizationData[0].AuthorizationToken - return *authData, nil + return "", err } func (r *ECRRegistry) GetSandboxImage() string { From b82d0a774cb93a4c52cd2a33f8b01e3d68dde9c9 Mon Sep 17 00:00:00 2001 From: vela Date: Thu, 18 Jul 2024 20:11:32 -0700 Subject: [PATCH 2/3] add retry to avoid ECR get auth token failed for some transient issues --- nodeadm/internal/aws/ecr/ecr.go | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/nodeadm/internal/aws/ecr/ecr.go b/nodeadm/internal/aws/ecr/ecr.go index fc7ec4d55..4a9d444f0 100644 --- a/nodeadm/internal/aws/ecr/ecr.go +++ b/nodeadm/internal/aws/ecr/ecr.go @@ -5,11 +5,13 @@ import ( "fmt" "net" "strings" + "time" "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/service/ecr" "github.com/awslabs/amazon-eks-ami/nodeadm/internal/aws/imds" "github.com/awslabs/amazon-eks-ami/nodeadm/internal/system" + "github.com/awslabs/amazon-eks-ami/nodeadm/internal/util" ) // Returns the base64 encoded authorization token string for ECR of the format "AWS:XXXXX" @@ -19,15 +21,19 @@ func GetAuthorizationToken(awsRegion string) (string, error) { return "", err } ecrClient := ecr.NewFromConfig(awsConfig) - - for i := 0; i < 3; i++ { - token, err := ecrClient.GetAuthorizationToken(context.Background(), &ecr.GetAuthorizationTokenInput{}) - if err == nil { - authData := token.AuthorizationData[0].AuthorizationToken - return *authData, nil + var token *ecr.GetAuthorizationTokenOutput + err = util.RetryExponentialBackoff(3, 2*time.Second, func() error { + token, err = ecrClient.GetAuthorizationToken(context.Background(), &ecr.GetAuthorizationTokenInput{}) + if err != nil { + return err } + return nil + }) + if err != nil { + return "", err } - return "", err + authData := token.AuthorizationData[0].AuthorizationToken + return *authData, nil } func (r *ECRRegistry) GetSandboxImage() string { From 34d9939c9f50acab86f0cb2b048e480ec7a88d9c Mon Sep 17 00:00:00 2001 From: vela Date: Thu, 18 Jul 2024 21:00:50 -0700 Subject: [PATCH 3/3] add retry to avoid ECR get auth token failed for some transient issues --- nodeadm/internal/aws/ecr/ecr.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/nodeadm/internal/aws/ecr/ecr.go b/nodeadm/internal/aws/ecr/ecr.go index 4a9d444f0..2b113e28a 100644 --- a/nodeadm/internal/aws/ecr/ecr.go +++ b/nodeadm/internal/aws/ecr/ecr.go @@ -24,10 +24,7 @@ func GetAuthorizationToken(awsRegion string) (string, error) { var token *ecr.GetAuthorizationTokenOutput err = util.RetryExponentialBackoff(3, 2*time.Second, func() error { token, err = ecrClient.GetAuthorizationToken(context.Background(), &ecr.GetAuthorizationTokenInput{}) - if err != nil { - return err - } - return nil + return err }) if err != nil { return "", err