From 697662b0bbbe5d7177976248a1dbd9b526bbadc1 Mon Sep 17 00:00:00 2001 From: James Saryerwinnie Date: Tue, 3 Feb 2015 10:30:34 -0800 Subject: [PATCH] Add class for merging converting s3 runtime config --- awscli/customizations/s3/transferconfig.py | 70 ++++++++++++++++++- awscli/customizations/s3/utils.py | 29 ++++++++ .../customizations/s3/test_transferconfig.py | 61 ++++++++++++++++ tests/unit/customizations/s3/test_utils.py | 15 ++++ 4 files changed, 174 insertions(+), 1 deletion(-) create mode 100644 tests/unit/customizations/s3/test_transferconfig.py diff --git a/awscli/customizations/s3/transferconfig.py b/awscli/customizations/s3/transferconfig.py index 53bdc5cb4174..01b314518ee7 100644 --- a/awscli/customizations/s3/transferconfig.py +++ b/awscli/customizations/s3/transferconfig.py @@ -1,4 +1,4 @@ -# Copyright 2013 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# Copyright 2013-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"). You # may not use this file except in compliance with the License. A copy of @@ -10,8 +10,76 @@ # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. +from awscli.customizations.s3.utils import human_readable_to_bytes +# If the user does not specify any overrides, +# these are the default values we use for the s3 transfer +# commands. +DEFAULTS = { + 'multipart_threshold': 8 * (1024 ** 2), + 'multipart_chunksize': 8 * (1024 ** 2), + 'max_concurrent_requests': 10, + 'max_queue_size': 1000, + # TODO: do we need this now that we have sentinels? Need to double check. + # I think this was exposed before because you'd have a 0.2s delay when + # running some of the S3 tests. + 'queue_timeout_wait': 0.2, +} MULTI_THRESHOLD = 8 * (1024 ** 2) CHUNKSIZE = 7 * (1024 ** 2) NUM_THREADS = 10 QUEUE_TIMEOUT_WAIT = 0.2 MAX_QUEUE_SIZE = 1000 + + +class InvalidConfigError(Exception): + pass + + +class RuntimeConfig(object): + + POSITIVE_INTEGERS = ['multipart_chunksize', 'multipart_threshold', + 'max_concurrent_requests', 'max_queue_size'] + HUMAN_READABLE_SIZES = ['multipart_chunksize', 'multipart_threshold'] + + @staticmethod + def defaults(): + return DEFAULTS.copy() + + def build_config(self, **kwargs): + """Create and convert a runtime config dictionary. + + This method will merge and convert S3 runtime configuration + data into a single dictionary that can then be passed to classes + that use this runtime config. + + :param kwargs: Any key in the ``DEFAULTS`` dict. + :return: A dictionar of the merged and converted values. + + """ + runtime_config = DEFAULTS.copy() + if kwargs: + runtime_config.update(kwargs) + self._convert_human_readable_sizes(runtime_config) + self._validate_config(runtime_config) + return runtime_config + + def _convert_human_readable_sizes(self, runtime_config): + for attr in self.HUMAN_READABLE_SIZES: + value = runtime_config.get(attr) + if value is not None and not isinstance(value, int): + runtime_config[attr] = human_readable_to_bytes(value) + + def _validate_config(self, runtime_config): + for attr in self.POSITIVE_INTEGERS: + value = runtime_config.get(attr) + if value is not None: + try: + runtime_config[attr] = int(value) + if not runtime_config[attr] > 0: + self._error_positive_value(attr, value) + except ValueError: + self._error_positive_value(attr, value) + + def _error_positive_value(self, name, value): + raise InvalidConfigError( + "Value for %s must be a positive integer: %s" % (name, value)) diff --git a/awscli/customizations/s3/utils.py b/awscli/customizations/s3/utils.py index c9d929b2a4d1..a191235ce2a3 100644 --- a/awscli/customizations/s3/utils.py +++ b/awscli/customizations/s3/utils.py @@ -35,6 +35,13 @@ # See: http://docs.aws.amazon.com/AmazonS3/latest/dev/UploadingObjects.html # and: http://docs.aws.amazon.com/AmazonS3/latest/dev/qfacts.html MAX_SINGLE_UPLOAD_SIZE = 5 * (1024 ** 3) +SIZE_SUFFIX = { + 'kb': 1024, + 'mb': 1024 ** 2, + 'gb': 1024 ** 3, + 'tb': 1024 ** 4, +} + def human_readable_size(value): @@ -70,6 +77,28 @@ def human_readable_size(value): return '%.1f %s' % ((base * bytes_int / unit), suffix) +def human_readable_to_bytes(value): + """Converts a human readable size to bytes. + + :param value: A string such as "10MB". If a suffix is not included, + then the value is assumed to be an integer representing the size + in bytes. + :returns: The converted value in bytes as an integer + + """ + suffix = value[-2:].lower() + has_size_identifier = ( + len(value) >= 2 and suffix in SIZE_SUFFIX) + if not has_size_identifier: + try: + return int(value) + except ValueError: + raise ValueError("Invalid size value: %s" % value) + else: + multiplier = SIZE_SUFFIX[suffix] + return int(value[:-2]) * multiplier + + class AppendFilter(argparse.Action): """ This class is used as an action when parsing the parameters. diff --git a/tests/unit/customizations/s3/test_transferconfig.py b/tests/unit/customizations/s3/test_transferconfig.py new file mode 100644 index 000000000000..cc9be90a8ce7 --- /dev/null +++ b/tests/unit/customizations/s3/test_transferconfig.py @@ -0,0 +1,61 @@ +# Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +from awscli.testutils import unittest + +from awscli.customizations.s3 import transferconfig + + +class TestTransferConfig(unittest.TestCase): + + def build_config_with(self, **config_from_user): + return transferconfig.RuntimeConfig().build_config(**config_from_user) + + def test_user_provides_no_config_uses_default(self): + # If the user does not provide any config overrides, + # we should just use the default values defined in + # the module. + config = transferconfig.RuntimeConfig() + runtime_config = config.build_config() + self.assertEqual(runtime_config, transferconfig.DEFAULTS) + + def test_user_provides_partial_overrides(self): + config_from_user = { + 'max_concurrent_requests': '20', + 'multipart_threshold': str(64 * (1024 ** 2)), + } + runtime_config = self.build_config_with(**config_from_user) + # Our overrides were accepted. + self.assertEqual(runtime_config['multipart_threshold'], + int(config_from_user['multipart_threshold'])) + self.assertEqual(runtime_config['max_concurrent_requests'], + int(config_from_user['max_concurrent_requests'])) + # And defaults were used for values not specified. + self.assertEqual(runtime_config['max_queue_size'], + int(transferconfig.DEFAULTS['max_queue_size'])) + + def test_validates_integer_types(self): + with self.assertRaises(transferconfig.InvalidConfigError): + self.build_config_with(max_concurrent_requests="not an int") + + def test_validates_positive_integers(self): + with self.assertRaises(transferconfig.InvalidConfigError): + self.build_config_with(max_concurrent_requests="-10") + + def test_min_value(self): + with self.assertRaises(transferconfig.InvalidConfigError): + self.build_config_with(max_concurrent_requests="0") + + def test_human_readable_sizes_converted_to_bytes(self): + runtime_config = self.build_config_with(multipart_threshold="10MB") + self.assertEqual(runtime_config['multipart_threshold'], + 10 * 1024 * 1024) diff --git a/tests/unit/customizations/s3/test_utils.py b/tests/unit/customizations/s3/test_utils.py index fde73e07a276..41c021ff22fc 100644 --- a/tests/unit/customizations/s3/test_utils.py +++ b/tests/unit/customizations/s3/test_utils.py @@ -22,6 +22,7 @@ from awscli.customizations.s3.utils import AppendFilter from awscli.customizations.s3.utils import create_warning from awscli.customizations.s3.utils import human_readable_size +from awscli.customizations.s3.utils import human_readable_to_bytes from awscli.customizations.s3.utils import MAX_SINGLE_UPLOAD_SIZE @@ -46,6 +47,20 @@ def _test_human_size_matches(bytes_int, expected): assert_equal(human_readable_size(bytes_int), expected) +def test_convert_human_readable_to_bytes(): + yield _test_convert_human_readable_to_bytes, "1", 1 + yield _test_convert_human_readable_to_bytes, "1024", 1024 + yield _test_convert_human_readable_to_bytes, "1KB", 1024 + yield _test_convert_human_readable_to_bytes, "1kb", 1024 + yield _test_convert_human_readable_to_bytes, "1MB", 1024 ** 2 + yield _test_convert_human_readable_to_bytes, "1GB", 1024 ** 3 + yield _test_convert_human_readable_to_bytes, "1TB", 1024 ** 4 + + +def _test_convert_human_readable_to_bytes(size_str, expected): + assert_equal(human_readable_to_bytes(size_str), expected) + + class AppendFilterTest(unittest.TestCase): def test_call(self): parser = argparse.ArgumentParser()