From c37c553e54e4cc22738c79931259d49ce283c928 Mon Sep 17 00:00:00 2001 From: Tulsi Shah Date: Sun, 27 Oct 2024 08:59:02 +0000 Subject: [PATCH 1/9] support error for single shot upload --- .idea/workspace.xml | 281 ++++++++++++++++++++++++++++++++++ testbench/rest_server.py | 16 ++ tests/test_testbench_retry.py | 106 +++++++++++++ 3 files changed, 403 insertions(+) create mode 100644 .idea/workspace.xml diff --git a/.idea/workspace.xml b/.idea/workspace.xml new file mode 100644 index 00000000..c0be75b2 --- /dev/null +++ b/.idea/workspace.xml @@ -0,0 +1,281 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + { + "keyToString": { + "RunOnceActivity.OpenProjectViewOnStart": "true", + "RunOnceActivity.ShowReadmeOnStart": "true", + "RunOnceActivity.go.formatter.settings.were.checked": "true", + "RunOnceActivity.go.migrated.go.modules.settings": "true", + "WebServerToolWindowFactoryState": "false", + "go.import.settings.migrated": "true", + "last_opened_file_path": "/usr/local/google/home/tulsishah/fork-testbench/storage-testbench", + "node.js.detected.package.eslint": "true", + "node.js.detected.package.tslint": "true", + "node.js.selected.package.eslint": "(autodetect)", + "node.js.selected.package.tslint": "(autodetect)", + "nodejs_package_manager_path": "npm", + "project.structure.last.edited": "Project", + "project.structure.proportion": "0.15", + "project.structure.side.proportion": "0.2", + "vue.rearranger.settings.migration": "true" + } +} + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 1729861767767 + + + + + + + true + + + + + \ No newline at end of file diff --git a/testbench/rest_server.py b/testbench/rest_server.py index 72a0ae88..f05ebfd9 100644 --- a/testbench/rest_server.py +++ b/testbench/rest_server.py @@ -979,6 +979,22 @@ def object_insert(bucket_name): blob, projection = gcs_type.object.Object.init_media(flask.request, bucket) elif upload_type == "multipart": blob, projection = gcs_type.object.Object.init_multipart(flask.request, bucket) + testbench.common.extract_instruction(request, context=None) + ( + error_code, + after_bytes, + test_id, + ) = testbench.common.get_retry_uploads_error_after_bytes(db, request) + if error_code and len(blob.media) >= after_bytes: + if test_id: + db.dequeue_next_instruction(test_id, "storage.objects.insert") + testbench.error.generic( + "Fault injected during a resumable upload", + rest_code=error_code, + grpc_code=None, + context=None, + ) + # Handle stall for full uploads. testbench.common.extract_instruction(request, context=None) ( diff --git a/tests/test_testbench_retry.py b/tests/test_testbench_retry.py index 0030993e..5a6fc4f6 100644 --- a/tests/test_testbench_retry.py +++ b/tests/test_testbench_retry.py @@ -880,6 +880,112 @@ def test_write_retry_test_stall_single_shot_while_upload_size_less_than_stall_si self.assertEqual(response.status_code, 200) self.assertLess(elapsed_time, 1) + def test_retry_test_return_error_after_bytes_for_single_shot_upload(self): + # Create a new bucket + response = self.client.post( + "/storage/v1/b", data=json.dumps({"name": "bucket-name"}) + ) + self.assertEqual(response.status_code, 200) + + # Setup a stall for reading back the object. + response = self.client.post( + "/retry_test", + data=json.dumps( + { + "instructions": { + "storage.objects.insert": [ + "return-503-after-250K", + ] + } + } + ), + content_type="application/json", + ) + self.assertEqual(response.status_code, 200) + self.assertTrue( + response.headers.get("content-type").startswith("application/json") + ) + + create_rest = json.loads(response.data) + self.assertIn("id", create_rest) + test_id = create_rest.get("id") + + # Upload the 256KiB of data and trigger the stall. + data = self._create_block(UPLOAD_QUANTUM) + self.assertEqual(len(data), UPLOAD_QUANTUM) + + boundary, payload = format_multipart_upload({}, data) + response = self.client.post( + "/upload/storage/v1/b/bucket-name/o", + query_string={"uploadType": "multipart", "name": "stall"}, + content_type="multipart/related; boundary=" + boundary, + headers={ + "x-retry-test-id": test_id, + }, + data=payload, + ) + self.assertEqual(response.status_code, 503) + + # Upload the data again and check that stall not happen. + response = self.client.post( + "/upload/storage/v1/b/bucket-name/o", + query_string={"uploadType": "multipart", "name": "stall"}, + content_type="multipart/related; boundary=" + boundary, + headers={ + "x-retry-test-id": test_id, + }, + data=payload, + ) + self.assertEqual(response.status_code, 200) + + def test_write_retry_error_single_shot_while_upload_size_less_than_size( + self, + ): + # Create a new bucket + response = self.client.post( + "/storage/v1/b", data=json.dumps({"name": "bucket-name"}) + ) + self.assertEqual(response.status_code, 200) + + # Setup a error for reading back the object. + response = self.client.post( + "/retry_test", + data=json.dumps( + { + "instructions": { + "storage.objects.insert": [ + "return-503-after-250K", + ] + } + } + ), + content_type="application/json", + ) + self.assertEqual(response.status_code, 200) + self.assertTrue( + response.headers.get("content-type").startswith("application/json") + ) + + create_rest = json.loads(response.data) + self.assertIn("id", create_rest) + test_id = create_rest.get("id") + + # Upload the 200KiB of data and check error not happen. + data = self._create_block(200 * 1024) + self.assertEqual(len(data), 200 * 1024) + + boundary, payload = format_multipart_upload({}, data) + response = self.client.post( + "/upload/storage/v1/b/bucket-name/o", + query_string={"uploadType": "multipart", "name": "error"}, + content_type="multipart/related; boundary=" + boundary, + headers={ + "x-retry-test-id": test_id, + }, + data=payload, + ) + self.assertEqual(response.status_code, 200) + class TestTestbenchRetryGrpc(unittest.TestCase): def setUp(self): From 0ef2f833597ddc2fe625243b0d777c3701d003e9 Mon Sep 17 00:00:00 2001 From: Tulsi Shah Date: Sun, 27 Oct 2024 08:59:22 +0000 Subject: [PATCH 2/9] remove .idea files --- .idea/workspace.xml | 281 -------------------------------------------- 1 file changed, 281 deletions(-) delete mode 100644 .idea/workspace.xml diff --git a/.idea/workspace.xml b/.idea/workspace.xml deleted file mode 100644 index c0be75b2..00000000 --- a/.idea/workspace.xml +++ /dev/null @@ -1,281 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - { - "keyToString": { - "RunOnceActivity.OpenProjectViewOnStart": "true", - "RunOnceActivity.ShowReadmeOnStart": "true", - "RunOnceActivity.go.formatter.settings.were.checked": "true", - "RunOnceActivity.go.migrated.go.modules.settings": "true", - "WebServerToolWindowFactoryState": "false", - "go.import.settings.migrated": "true", - "last_opened_file_path": "/usr/local/google/home/tulsishah/fork-testbench/storage-testbench", - "node.js.detected.package.eslint": "true", - "node.js.detected.package.tslint": "true", - "node.js.selected.package.eslint": "(autodetect)", - "node.js.selected.package.tslint": "(autodetect)", - "nodejs_package_manager_path": "npm", - "project.structure.last.edited": "Project", - "project.structure.proportion": "0.15", - "project.structure.side.proportion": "0.2", - "vue.rearranger.settings.migration": "true" - } -} - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 1729861767767 - - - - - - - true - - - - - \ No newline at end of file From 3b726b1e4e10cf632c6f077748f3917bb690caf4 Mon Sep 17 00:00:00 2001 From: Tulsi Shah Date: Sun, 27 Oct 2024 09:06:22 +0000 Subject: [PATCH 3/9] lint fix --- testbench/rest_server.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/testbench/rest_server.py b/testbench/rest_server.py index f05ebfd9..0e87fc90 100644 --- a/testbench/rest_server.py +++ b/testbench/rest_server.py @@ -981,9 +981,9 @@ def object_insert(bucket_name): blob, projection = gcs_type.object.Object.init_multipart(flask.request, bucket) testbench.common.extract_instruction(request, context=None) ( - error_code, - after_bytes, - test_id, + error_code, + after_bytes, + test_id, ) = testbench.common.get_retry_uploads_error_after_bytes(db, request) if error_code and len(blob.media) >= after_bytes: if test_id: From 1845f39defa63da756fa238fbf47d350f76ea1b6 Mon Sep 17 00:00:00 2001 From: Tulsi Shah Date: Mon, 28 Oct 2024 03:21:32 +0000 Subject: [PATCH 4/9] review comments --- .idea/workspace.xml | 280 +++++++++++++++++++++++++++++++++++++++ testbench/rest_server.py | 4 +- 2 files changed, 283 insertions(+), 1 deletion(-) create mode 100644 .idea/workspace.xml diff --git a/.idea/workspace.xml b/.idea/workspace.xml new file mode 100644 index 00000000..3692bd32 --- /dev/null +++ b/.idea/workspace.xml @@ -0,0 +1,280 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 1729861767767 + + + + + + + true + + + + + \ No newline at end of file diff --git a/testbench/rest_server.py b/testbench/rest_server.py index 0e87fc90..dcdd8b91 100644 --- a/testbench/rest_server.py +++ b/testbench/rest_server.py @@ -979,6 +979,8 @@ def object_insert(bucket_name): blob, projection = gcs_type.object.Object.init_media(flask.request, bucket) elif upload_type == "multipart": blob, projection = gcs_type.object.Object.init_multipart(flask.request, bucket) + + if upload_type == "media" or upload_type == "multipart": testbench.common.extract_instruction(request, context=None) ( error_code, @@ -989,7 +991,7 @@ def object_insert(bucket_name): if test_id: db.dequeue_next_instruction(test_id, "storage.objects.insert") testbench.error.generic( - "Fault injected during a resumable upload", + "Fault injected during a single-shot upload", rest_code=error_code, grpc_code=None, context=None, From ae03841fdaceb9417257e59307deb3da00114a3b Mon Sep 17 00:00:00 2001 From: Tulsi Shah Date: Mon, 28 Oct 2024 03:22:50 +0000 Subject: [PATCH 5/9] review comments --- .idea/workspace.xml | 280 -------------------------------------------- 1 file changed, 280 deletions(-) delete mode 100644 .idea/workspace.xml diff --git a/.idea/workspace.xml b/.idea/workspace.xml deleted file mode 100644 index 3692bd32..00000000 --- a/.idea/workspace.xml +++ /dev/null @@ -1,280 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 1729861767767 - - - - - - - true - - - - - \ No newline at end of file From 4bb8db465ad7ab8b206b33477b061583f54431ff Mon Sep 17 00:00:00 2001 From: Tulsi Shah Date: Mon, 28 Oct 2024 03:43:39 +0000 Subject: [PATCH 6/9] review comments --- testbench/rest_server.py | 52 ++++++++++++++++++++-------------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/testbench/rest_server.py b/testbench/rest_server.py index dcdd8b91..44b265a7 100644 --- a/testbench/rest_server.py +++ b/testbench/rest_server.py @@ -980,34 +980,34 @@ def object_insert(bucket_name): elif upload_type == "multipart": blob, projection = gcs_type.object.Object.init_multipart(flask.request, bucket) - if upload_type == "media" or upload_type == "multipart": - testbench.common.extract_instruction(request, context=None) - ( - error_code, - after_bytes, - test_id, - ) = testbench.common.get_retry_uploads_error_after_bytes(db, request) - if error_code and len(blob.media) >= after_bytes: - if test_id: - db.dequeue_next_instruction(test_id, "storage.objects.insert") - testbench.error.generic( - "Fault injected during a single-shot upload", - rest_code=error_code, - grpc_code=None, - context=None, - ) + testbench.common.extract_instruction(request, context=None) + ( + error_code, + after_bytes, + test_id, + ) = testbench.common.get_retry_uploads_error_after_bytes(db, request) + + if error_code and len(blob.media) >= after_bytes: + if test_id: + db.dequeue_next_instruction(test_id, "storage.objects.insert") + testbench.error.generic( + "Fault injected during a single-shot upload", + rest_code=error_code, + grpc_code=None, + context=None, + ) # Handle stall for full uploads. - testbench.common.extract_instruction(request, context=None) - ( - stall_time, - after_bytes, - test_id, - ) = testbench.common.get_stall_uploads_after_bytes(db, request) - if stall_time and len(blob.media) >= after_bytes: - if test_id: - db.dequeue_next_instruction(test_id, "storage.objects.insert") - time.sleep(stall_time) + testbench.common.extract_instruction(request, context=None) + ( + stall_time, + after_bytes, + test_id, + ) = testbench.common.get_stall_uploads_after_bytes(db, request) + if stall_time and len(blob.media) >= after_bytes: + if test_id: + db.dequeue_next_instruction(test_id, "storage.objects.insert") + time.sleep(stall_time) db.insert_object( bucket_name, From 9902857e7b8079d72cde892022f099163a97e1ff Mon Sep 17 00:00:00 2001 From: Tulsi Shah Date: Mon, 28 Oct 2024 03:45:16 +0000 Subject: [PATCH 7/9] lint fix --- testbench/rest_server.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/testbench/rest_server.py b/testbench/rest_server.py index 44b265a7..800eabf9 100644 --- a/testbench/rest_server.py +++ b/testbench/rest_server.py @@ -989,12 +989,12 @@ def object_insert(bucket_name): if error_code and len(blob.media) >= after_bytes: if test_id: - db.dequeue_next_instruction(test_id, "storage.objects.insert") + db.dequeue_next_instruction(test_id, "storage.objects.insert") testbench.error.generic( - "Fault injected during a single-shot upload", - rest_code=error_code, - grpc_code=None, - context=None, + "Fault injected during a single-shot upload", + rest_code=error_code, + grpc_code=None, + context=None, ) # Handle stall for full uploads. From e02deaeea571c6cdba852da4bc44a16224a754c7 Mon Sep 17 00:00:00 2001 From: Tulsi Shah Date: Mon, 28 Oct 2024 03:47:21 +0000 Subject: [PATCH 8/9] lint fix --- testbench/rest_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testbench/rest_server.py b/testbench/rest_server.py index 800eabf9..9e396244 100644 --- a/testbench/rest_server.py +++ b/testbench/rest_server.py @@ -991,7 +991,7 @@ def object_insert(bucket_name): if test_id: db.dequeue_next_instruction(test_id, "storage.objects.insert") testbench.error.generic( - "Fault injected during a single-shot upload", + "Fault injected during a single-shot upload", rest_code=error_code, grpc_code=None, context=None, From 73a1b65f6a9f97ba513547873c585087056c9d49 Mon Sep 17 00:00:00 2001 From: Tulsi Shah Date: Mon, 28 Oct 2024 03:53:43 +0000 Subject: [PATCH 9/9] lint fix --- testbench/rest_server.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/testbench/rest_server.py b/testbench/rest_server.py index 9e396244..5e58296f 100644 --- a/testbench/rest_server.py +++ b/testbench/rest_server.py @@ -980,6 +980,7 @@ def object_insert(bucket_name): elif upload_type == "multipart": blob, projection = gcs_type.object.Object.init_multipart(flask.request, bucket) + # Handle errors for single-shot uploads. testbench.common.extract_instruction(request, context=None) ( error_code, @@ -997,7 +998,7 @@ def object_insert(bucket_name): context=None, ) - # Handle stall for full uploads. + # Handle stall for single-shot uploads. testbench.common.extract_instruction(request, context=None) ( stall_time,