Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fix(storage): wrap error when MaxAttempts is hit #9767

Merged
merged 4 commits into from
Apr 15, 2024
Merged

fix(storage): wrap error when MaxAttempts is hit #9767

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
06ef849
fix(storage): wrap error when MaxAttempts is hit
tritone Apr 15, 2024
5c3944c
fix wrapping on last attempt and invoke test
tritone Apr 15, 2024
614732e
Merge branch 'main' into max-attempts-wrap
tritone Apr 15, 2024
885f188
test cleanup, fix typos
tritone Apr 15, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
105 changes: 85 additions & 20 deletions storage/client_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ import (

"cloud.google.com/go/iam/apiv1/iampb"
"github.com/google/go-cmp/cmp"
"github.com/googleapis/gax-go/v2"
"github.com/googleapis/gax-go/v2/apierror"
"github.com/googleapis/gax-go/v2/callctx"
"google.golang.org/api/iterator"
Expand Down Expand Up @@ -1351,41 +1352,105 @@ func TestObjectConditionsEmulated(t *testing.T) {
func TestRetryNeverEmulated(t *testing.T) {
transportClientTest(t, func(t *testing.T, project, bucket string, client storageClient) {
ctx := context.Background()
instructions := map[string][]string{"storage.buckets.get": {"return-503"}}
testID := createRetryTest(t, project, bucket, client, instructions)
ctx = callctx.SetHeaders(ctx, "x-retry-test-id", testID)
_, err := client.GetBucket(ctx, bucket, nil, withRetryConfig(&retryConfig{policy: RetryNever}))

attrs, err := client.CreateBucket(ctx, project, bucket, &BucketAttrs{}, nil)
if err != nil {
t.Fatalf("creating bucket: %v", err)
var ae *apierror.APIError
if errors.As(err, &ae) {
// We expect a 503/UNAVAILABLE error. For anything else including a nil
// error, the test should fail.
if ae.GRPCStatus().Code() != codes.Unavailable && ae.HTTPCode() != 503 {
t.Errorf("GetBucket: got unexpected error %v; want 503", err)
}
}
})
}

// Need the HTTP hostname to set up a retry test, as well as knowledge of
// underlying transport to specify instructions.
host := os.Getenv("STORAGE_EMULATOR_HOST")
endpoint, err := url.Parse(host)
if err != nil {
t.Fatalf("parsing endpoint: %v", err)
// Test that errors are wrapped correctly if retry happens until a timeout.
func TestRetryTimeoutEmulated(t *testing.T) {
transportClientTest(t, func(t *testing.T, project, bucket string, client storageClient) {
ctx := context.Background()
instructions := map[string][]string{"storage.buckets.get": {"return-503", "return-503", "return-503", "return-503", "return-503"}}
testID := createRetryTest(t, project, bucket, client, instructions)
ctx = callctx.SetHeaders(ctx, "x-retry-test-id", testID)
ctx, cancel := context.WithTimeout(ctx, 100*time.Millisecond)
defer cancel()
_, err := client.GetBucket(ctx, bucket, nil, idempotent(true))

var ae *apierror.APIError
if errors.As(err, &ae) {
// We expect a 503/UNAVAILABLE error. For anything else including a nil
// error, the test should fail.
if ae.GRPCStatus().Code() != codes.Unavailable && ae.HTTPCode() != 503 {
t.Errorf("GetBucket: got unexpected error: %v; want 503", err)
}
}
var transport string
if _, ok := client.(*httpStorageClient); ok {
transport = "http"
} else {
transport = "grpc"
// Error should be wrapped so it's also equivalent to a context timeout.
if !errors.Is(err, context.DeadlineExceeded) {
t.Errorf("GetBucket: got unexpected error %v, want to match DeadlineExceeded.", err)
}
})
}

et := emulatorTest{T: t, name: "testRetryNever", resources: resources{},
host: endpoint}
et.create(map[string][]string{"storage.buckets.get": {"return-503"}}, transport)
ctx = callctx.SetHeaders(ctx, "x-retry-test-id", et.id)
_, err = client.GetBucket(ctx, attrs.Name, nil, withRetryConfig(&retryConfig{policy: RetryNever}))
// Test that errors are wrapped correctly if retry happens until max attempts.
func TestRetryMaxAttemptsEmulated(t *testing.T) {
transportClientTest(t, func(t *testing.T, project, bucket string, client storageClient) {
ctx := context.Background()
instructions := map[string][]string{"storage.buckets.get": {"return-503", "return-503", "return-503", "return-503", "return-503"}}
testID := createRetryTest(t, project, bucket, client, instructions)
ctx = callctx.SetHeaders(ctx, "x-retry-test-id", testID)
config := &retryConfig{maxAttempts: expectedAttempts(3), backoff: &gax.Backoff{Initial: 10 * time.Millisecond}}
_, err := client.GetBucket(ctx, bucket, nil, idempotent(true), withRetryConfig(config))

var ae *apierror.APIError
if errors.As(err, &ae) {
// We espect a 503/UNAVAILABLE error. For anything else including a nil
// We expect a 503/UNAVAILABLE error. For anything else including a nil
// error, the test should fail.
if ae.GRPCStatus().Code() != codes.Unavailable && ae.HTTPCode() != 503 {
t.Errorf("GetBucket: got unexpected error %v; want 503", err)
}
}
// Error should be wrapped so it indicates that MaxAttempts has been reached.
if got, want := err.Error(), "retry failed after 3 attempts"; !strings.Contains(got, want) {
t.Errorf("got error: %q, want to contain: %q", got, want)
}
tritone marked this conversation as resolved.
Show resolved Hide resolved
})
}

// createRetryTest creates a bucket in the emulator and sets up a test using the
// Retry Test API for the given instructions. This is intended for emulator tests
// of retry behavior that are not covered by conformance tests.
func createRetryTest(t *testing.T, project, bucket string, client storageClient, instructions map[string][]string) string {
t.Helper()
ctx := context.Background()

_, err := client.CreateBucket(ctx, project, bucket, &BucketAttrs{}, nil)
if err != nil {
t.Fatalf("creating bucket: %v", err)
}

// Need the HTTP hostname to set up a retry test, as well as knowledge of
// underlying transport to specify instructions.
host := os.Getenv("STORAGE_EMULATOR_HOST")
endpoint, err := url.Parse(host)
if err != nil {
t.Fatalf("parsing endpoint: %v", err)
}
var transport string
if _, ok := client.(*httpStorageClient); ok {
transport = "http"
} else {
transport = "grpc"
}

et := emulatorTest{T: t, name: t.Name(), resources: resources{}, host: endpoint}
et.create(instructions, transport)
t.Cleanup(func() {
et.delete()
})
return et.id
}

// createObject creates an object in the emulator and returns its name, generation, and
Expand Down
4 changes: 2 additions & 2 deletions storage/invoke.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,8 +70,8 @@ func run(ctx context.Context, call func(ctx context.Context) error, retry *retry
return internal.Retry(ctx, bo, func() (stop bool, err error) {
ctxWithHeaders := setInvocationHeaders(ctx, invocationID, attempts)
err = call(ctxWithHeaders)
if retry.maxAttempts != nil && attempts >= *retry.maxAttempts {
return true, err
if err != nil && retry.maxAttempts != nil && attempts >= *retry.maxAttempts {
return true, fmt.Errorf("storage: retry failed after %v attempts; last error: %w", *retry.maxAttempts, err)
}
attempts++
return !errorFunc(err), err
Expand Down
4 changes: 2 additions & 2 deletions storage/invoke_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -251,9 +251,9 @@ func TestInvoke(t *testing.T) {
return test.finalErr
}
got := run(ctx, call, test.retry, test.isIdempotentValue)
if test.expectFinalErr && got != test.finalErr {
if test.expectFinalErr && !errors.Is(got, test.finalErr) {
s.Errorf("got %v, want %v", got, test.finalErr)
} else if !test.expectFinalErr && got != test.initialErr {
} else if !test.expectFinalErr && !errors.Is(got, test.initialErr) {
s.Errorf("got %v, want %v", got, test.initialErr)
}
wantAttempts := 1 + test.count
Expand Down
Loading