From f17e91c37a025d24d0dd396160a5588fd2d88cf5 Mon Sep 17 00:00:00 2001 From: Pawel Polewicz Date: Fri, 29 Apr 2022 13:55:29 +0200 Subject: [PATCH 01/13] Add a unit test --- test/unit/v_all/test_replication.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/test/unit/v_all/test_replication.py b/test/unit/v_all/test_replication.py index 8640f40e4..72a94c860 100644 --- a/test/unit/v_all/test_replication.py +++ b/test/unit/v_all/test_replication.py @@ -17,7 +17,7 @@ from apiver_deps import InMemoryCache from apiver_deps import InMemoryAccountInfo from apiver_deps import RawSimulator -from apiver_deps import ReplicationRule, ReplicationDestinationConfiguration, ReplicationSourceConfiguration +from apiver_deps import ReplicationConfiguration, ReplicationDestinationConfiguration, ReplicationRule, ReplicationSourceConfiguration from ..test_base import TestBase from b2sdk.replication.setup import ReplicationSetupHelper @@ -146,3 +146,8 @@ def test_setup_both(self): new_source_application_key.id_: destination_application_key.id_ } ) + + @pytest.mark.apiver(from_ver=2) + def test_factory(self): + replication = ReplicationConfiguration.from_dict({}) + assert replication == ReplicationConfiguration() From 94c2ed9a7c465a58407f2bf88885417a988f2d5e Mon Sep 17 00:00:00 2001 From: Pawel Polewicz Date: Sat, 30 Apr 2022 20:28:18 +0200 Subject: [PATCH 02/13] Small style tweaks in b2sdk/replication --- b2sdk/replication/setting.py | 2 +- b2sdk/replication/setup.py | 83 ++++++++++++++++------------- test/unit/v_all/test_replication.py | 3 +- 3 files changed, 49 insertions(+), 39 deletions(-) diff --git a/b2sdk/replication/setting.py b/b2sdk/replication/setting.py index 3ea52cba4..7db2bf3b9 100644 --- a/b2sdk/replication/setting.py +++ b/b2sdk/replication/setting.py @@ -33,7 +33,7 @@ class ReplicationRule: REPLICATION_RULE_REGEX: ClassVar = re.compile(r'^[a-zA-Z0-9_\-]{1,64}$') MIN_PRIORITY: ClassVar[int] = 1 - MAX_PRIORITY: ClassVar[int] = 2147483647 + MAX_PRIORITY: ClassVar[int] = 2**31 - 1 def __post_init__(self): if not self.destination_bucket_id: diff --git a/b2sdk/replication/setup.py b/b2sdk/replication/setup.py index e374e38c7..aa94558dd 100644 --- a/b2sdk/replication/setup.py +++ b/b2sdk/replication/setup.py @@ -17,6 +17,7 @@ # b2 replication-accept destinationBucketName sourceKeyId [destinationKeyId] # b2 replication-deny destinationBucketName sourceKeyId +from collections.abc import Iterable from typing import ClassVar, List, Optional, Tuple import itertools import logging @@ -31,7 +32,7 @@ class ReplicationSetupHelper(metaclass=B2TraceMeta): - """ class with various methods that help with repliction management """ + """ class with various methods that help with setting up repliction """ PRIORITY_OFFSET: ClassVar[int] = 5 #: how far to to put the new rule from the existing rules DEFAULT_PRIORITY: ClassVar[ int @@ -58,15 +59,15 @@ def __init__(self, source_b2api: B2Api = None, destination_b2api: B2Api = None): def setup_both( self, source_bucket_name: str, - destination_bucket: Bucket, + destination_bucket_name: str, name: Optional[str] = None, #: name for the new replication rule priority: int = None, #: priority for the new replication rule prefix: Optional[str] = None, ) -> Tuple[Bucket, Bucket]: source_bucket = self.setup_source( source_bucket_name, + destination_bucket_name, prefix, - destination_bucket, name, priority, ) @@ -79,10 +80,10 @@ def setup_both( def setup_destination( self, source_key_id: str, - destination_bucket: Bucket, + destination_bucket_name: str, ) -> Bucket: api: B2Api = destination_bucket.api - destination_bucket = api.list_buckets(destination_bucket.name)[0] # fresh! + destination_bucket = api.list_buckets(destination_bucket_name)[0] # fresh! if destination_bucket.replication is None or destination_bucket.replication.as_replication_source is None: source_configuration = None else: @@ -153,20 +154,20 @@ def _get_destination_key( def setup_source( self, - source_bucket_name, - prefix, + source_bucket_name: str, destination_bucket: Bucket, - name, - priority, + prefix: Optional[str] = None, + name: Optional[str] = None, #: name for the new replication rule + priority: int = None, #: priority for the new replication rule ) -> Bucket: source_bucket: Bucket = self.source_b2api.list_buckets(source_bucket_name)[0] # fresh! if prefix is None: prefix = "" try: - current_source_rrs = source_bucket.replication.as_replication_source.rules + current_source_rules = source_bucket.replication.as_replication_source.rules except (NameError, AttributeError): - current_source_rrs = [] + current_source_rules = [] try: destination_configuration = source_bucket.replication.as_replication_destination except (NameError, AttributeError): @@ -176,16 +177,16 @@ def setup_source( source_bucket, prefix, source_bucket.replication, - current_source_rrs, + current_source_rules, ) priority = self._get_priority_for_new_rule( priority, - current_source_rrs, + current_source_rules, ) - name = self._get_name_for_new_rule( + name = self._get_new_rule_name( + current_source_rules, + destination_bucket, name, - current_source_rrs, - destination_bucket.name, ) new_rr = ReplicationRule( name=name, @@ -196,7 +197,7 @@ def setup_source( new_replication_configuration = ReplicationConfiguration( ReplicationSourceConfiguration( source_application_key_id=source_key.id_, - rules=current_source_rrs + [new_rr], + rules=current_source_rules + [new_rr], ), destination_configuration, ) @@ -208,10 +209,10 @@ def setup_source( @classmethod def _get_source_key( cls, - source_bucket, - prefix, + source_bucket: Bucket, + prefix: str, current_replication_configuration: ReplicationConfiguration, - current_source_rrs, + current_source_rules: Iterable[ReplicationRule], ) -> ApplicationKey: api = source_bucket.api @@ -229,7 +230,8 @@ def _get_source_key( name=source_bucket.name[:91] + '-replisrc', api=api, bucket_id=source_bucket.id_, - ) # no prefix! + prefix=prefix, + ) return new_key @classmethod @@ -273,6 +275,12 @@ def _create_source_key( bucket_id: str, prefix: Optional[str] = None, ) -> ApplicationKey: + # in this implementation we ignore the prefix and create a full key, because + # if someone would need a different (wider) key later, all replication + # destinations would have to start using new keys and it's not feasible + # from organizational perspective, while the prefix of uploaded files can be + # restricted on the rule level + prefix = None capabilities = cls.DEFAULT_SOURCE_CAPABILITIES return cls._create_key(name, api, bucket_id, prefix, capabilities) @@ -304,32 +312,35 @@ def _create_key( ) @classmethod - def _get_narrowest_common_prefix(cls, widen_to: List[str]) -> str: - for path in widen_to: - pass # TODO - return '' - - @classmethod - def _get_priority_for_new_rule(cls, priority, current_source_rrs): - # if there is no priority hint, look into current rules to determine the last priority and add a constant to it + def _get_priority_for_new_rule( + cls, + current_rules: Iterable[ReplicationRule], + priority: Optional[int] = None, + ): if priority is not None: return priority - if current_source_rrs: - # TODO: maybe handle a case where the existing rrs need to have their priorities decreased to make space - existing_priority = max(rr.priority for rr in current_source_rrs) + if current_rules: + # ignore a case where the existing rrs need to have their priorities decreased to make space (max is 2**31-1) + existing_priority = max(rr.priority for rr in current_rules) return min(existing_priority + cls.PRIORITY_OFFSET, cls.MAX_PRIORITY) return cls.DEFAULT_PRIORITY @classmethod - def _get_name_for_new_rule( - cls, name: Optional[str], current_source_rrs, destination_bucket_name + def _get_new_rule_name( + cls, + current_rules: Iterable[ReplicationRule], + destination_bucket: Bucket, + name: Optional[str] = None, ): if name is not None: return name - existing_names = set(rr.name for rr in current_source_rrs) + existing_names = set(rr.name for rr in current_rules) suffixes = cls._get_rule_name_candidate_suffixes() while True: - candidate = '%s%s' % (destination_bucket_name, next(suffixes)) + candidate = '%s%s' % ( + destination_bucket_name, + next(suffixes), + ) # use := after dropping 3.7 if candidate not in existing_names: return candidate diff --git a/test/unit/v_all/test_replication.py b/test/unit/v_all/test_replication.py index 72a94c860..fc0800063 100644 --- a/test/unit/v_all/test_replication.py +++ b/test/unit/v_all/test_replication.py @@ -115,7 +115,6 @@ def test_setup_both(self): source_bucket, destination_bucket = rsh.setup_both( source_bucket_name="bucket1", destination_bucket=destination_bucket, - name='ac', prefix='ad', ) @@ -132,7 +131,7 @@ def test_setup_both(self): ), ReplicationRule( destination_bucket_id='bucket_1', - name='ac', + name='bucket2', file_name_prefix='ad', is_enabled=True, priority=133, From 9d3c880344830095dab507d1b89ac81ca73b2255 Mon Sep 17 00:00:00 2001 From: Pawel Polewicz Date: Mon, 2 May 2022 16:11:29 +0200 Subject: [PATCH 03/13] Change interface of ReplicationSetupHelper --- b2sdk/replication/setup.py | 61 ++++++++++++++--------------- test/unit/v_all/test_replication.py | 20 ++-------- 2 files changed, 33 insertions(+), 48 deletions(-) diff --git a/b2sdk/replication/setup.py b/b2sdk/replication/setup.py index aa94558dd..5281ef185 100644 --- a/b2sdk/replication/setup.py +++ b/b2sdk/replication/setup.py @@ -30,6 +30,11 @@ logger = logging.getLogger(__name__) +try: + Iterable[str] +except TypeError: + Iterable = List # Remove after dropping Python 3.8 + class ReplicationSetupHelper(metaclass=B2TraceMeta): """ class with various methods that help with setting up repliction """ @@ -51,39 +56,36 @@ class ReplicationSetupHelper(metaclass=B2TraceMeta): 'deleteFiles', ) - def __init__(self, source_b2api: B2Api = None, destination_b2api: B2Api = None): - assert source_b2api is not None or destination_b2api is not None - self.source_b2api = source_b2api - self.destination_b2api = destination_b2api - def setup_both( self, - source_bucket_name: str, - destination_bucket_name: str, + source_bucket: Bucket, + destination_bucket: Bucket, name: Optional[str] = None, #: name for the new replication rule priority: int = None, #: priority for the new replication rule prefix: Optional[str] = None, ) -> Tuple[Bucket, Bucket]: - source_bucket = self.setup_source( - source_bucket_name, - destination_bucket_name, + + new_source_bucket = self.setup_source( + source_bucket, + destination_bucket, prefix, name, priority, ) - destination_bucket = self.setup_destination( - source_bucket.replication.as_replication_source.source_application_key_id, + + new_destination_bucket = self.setup_destination( + new_source_bucket.replication.as_replication_source.source_application_key_id, destination_bucket, ) - return source_bucket, destination_bucket + + return new_source_bucket, new_destination_bucket def setup_destination( self, source_key_id: str, - destination_bucket_name: str, + destination_bucket: Bucket, ) -> Bucket: api: B2Api = destination_bucket.api - destination_bucket = api.list_buckets(destination_bucket_name)[0] # fresh! if destination_bucket.replication is None or destination_bucket.replication.as_replication_source is None: source_configuration = None else: @@ -146,21 +148,19 @@ def _get_destination_key( logger.debug("no matching key found, making a new one") key = cls._create_destination_key( name=destination_bucket.name[:91] + '-replidst', - api=api, - bucket_id=destination_bucket.id_, + bucket=destination_bucket, prefix=None, ) return keys_to_purge, key def setup_source( self, - source_bucket_name: str, + source_bucket: Bucket, destination_bucket: Bucket, prefix: Optional[str] = None, name: Optional[str] = None, #: name for the new replication rule priority: int = None, #: priority for the new replication rule ) -> Bucket: - source_bucket: Bucket = self.source_b2api.list_buckets(source_bucket_name)[0] # fresh! if prefix is None: prefix = "" @@ -180,8 +180,8 @@ def setup_source( current_source_rules, ) priority = self._get_priority_for_new_rule( - priority, current_source_rules, + priority, ) name = self._get_new_rule_name( current_source_rules, @@ -228,8 +228,7 @@ def _get_source_key( new_key = cls._create_source_key( name=source_bucket.name[:91] + '-replisrc', - api=api, - bucket_id=source_bucket.id_, + bucket=source_bucket, prefix=prefix, ) return new_key @@ -271,8 +270,7 @@ def _should_make_new_source_key( def _create_source_key( cls, name: str, - api: B2Api, - bucket_id: str, + bucket: Bucket, prefix: Optional[str] = None, ) -> ApplicationKey: # in this implementation we ignore the prefix and create a full key, because @@ -282,32 +280,31 @@ def _create_source_key( # restricted on the rule level prefix = None capabilities = cls.DEFAULT_SOURCE_CAPABILITIES - return cls._create_key(name, api, bucket_id, prefix, capabilities) + return cls._create_key(name, bucket, prefix, capabilities) @classmethod def _create_destination_key( cls, name: str, - api: B2Api, - bucket_id: str, + bucket: Bucket, prefix: Optional[str] = None, ) -> ApplicationKey: capabilities = cls.DEFAULT_DESTINATION_CAPABILITIES - return cls._create_key(name, api, bucket_id, prefix, capabilities) + return cls._create_key(name, bucket, prefix, capabilities) @classmethod def _create_key( cls, name: str, - api: B2Api, - bucket_id: str, + bucket: Bucket, prefix: Optional[str] = None, capabilities=tuple(), ) -> ApplicationKey: + api: B2Api = bucket.api return api.create_key( capabilities=capabilities, key_name=name, - bucket_id=bucket_id, + bucket_id=bucket.id_, name_prefix=prefix, ) @@ -338,7 +335,7 @@ def _get_new_rule_name( suffixes = cls._get_rule_name_candidate_suffixes() while True: candidate = '%s%s' % ( - destination_bucket_name, + destination_bucket.name, next(suffixes), ) # use := after dropping 3.7 if candidate not in existing_names: diff --git a/test/unit/v_all/test_replication.py b/test/unit/v_all/test_replication.py index fc0800063..d1bf838aa 100644 --- a/test/unit/v_all/test_replication.py +++ b/test/unit/v_all/test_replication.py @@ -17,11 +17,9 @@ from apiver_deps import InMemoryCache from apiver_deps import InMemoryAccountInfo from apiver_deps import RawSimulator -from apiver_deps import ReplicationConfiguration, ReplicationDestinationConfiguration, ReplicationRule, ReplicationSourceConfiguration +from apiver_deps import ReplicationConfiguration, ReplicationDestinationConfiguration, ReplicationRule, ReplicationSetupHelper, ReplicationSourceConfiguration from ..test_base import TestBase -from b2sdk.replication.setup import ReplicationSetupHelper - logger = logging.getLogger(__name__) @@ -41,22 +39,12 @@ def _authorize_account(self): @pytest.mark.apiver(from_ver=2) def test_setup_both(self): self._authorize_account() - #with pytest.raises(BucketIdNotFound): - # self.api.get_bucket_by_id("this id doesn't even exist") source_bucket = self.api.create_bucket('bucket1', 'allPrivate') destination_bucket = self.api.create_bucket('bucket2', 'allPrivate') - #read_bucket = self.api.get_bucket_by_id(source_bucket.id_) - #assert source_bucket.id_ == read_bucket.id_ - #self.cache.save_bucket(Bucket(api=self.api, name='bucket_name', id_='bucket_id')) - #read_bucket = self.api.get_bucket_by_id('bucket_id') - #assert read_bucket.name == 'bucket_name' logger.info('preparations complete, starting the test') - rsh = ReplicationSetupHelper( - source_b2api=self.api, - destination_b2api=self.api, - ) + rsh = ReplicationSetupHelper() source_bucket, destination_bucket = rsh.setup_both( - source_bucket_name="bucket1", + source_bucket=source_bucket, destination_bucket=destination_bucket, name='aa', prefix='ab', @@ -113,7 +101,7 @@ def test_setup_both(self): old_source_application_key = source_application_key source_bucket, destination_bucket = rsh.setup_both( - source_bucket_name="bucket1", + source_bucket=source_bucket, destination_bucket=destination_bucket, prefix='ad', ) From c3e9f0b8486a6bd3fbae24c57dfbccf94a894aa4 Mon Sep 17 00:00:00 2001 From: Pawel Polewicz Date: Tue, 3 May 2022 22:33:59 +0200 Subject: [PATCH 04/13] Re-enable accidentally disabled download integration tests --- test/integration/test_download.py | 114 +++++++++++++++--------------- 1 file changed, 57 insertions(+), 57 deletions(-) diff --git a/test/integration/test_download.py b/test/integration/test_download.py index 39f2f510c..d950dd7c9 100644 --- a/test/integration/test_download.py +++ b/test/integration/test_download.py @@ -21,60 +21,60 @@ from .helpers import GENERAL_BUCKET_NAME_PREFIX from .base import IntegrationTestBase -#class TestDownload(IntegrationTestBase): -# def test_large_file(self): -# bucket = self.create_bucket() -# with mock.patch.object( -# self.info, '_recommended_part_size', new=self.info.get_absolute_minimum_part_size() -# ): -# download_manager = self.b2_api.services.download_manager -# with mock.patch.object( -# download_manager, -# 'strategies', -# new=[ -# ParallelDownloader( -# min_part_size=self.info.get_absolute_minimum_part_size(), -# min_chunk_size=download_manager.MIN_CHUNK_SIZE, -# max_chunk_size=download_manager.MAX_CHUNK_SIZE, -# thread_pool=download_manager._thread_pool, -# ) -# ] -# ): -# -# # let's check that small file downloads fail with these settings -# bucket.upload_bytes(b'0', 'a_single_zero') -# with pytest.raises(ValueError) as exc_info: -# with io.BytesIO() as io_: -# bucket.download_file_by_name('a_single_zero').save(io_) -# assert exc_info.value.args == ('no strategy suitable for download was found!',) -# f = self._file_helper(bucket) -# assert f.download_version.content_sha1_verified -# -# def _file_helper(self, bucket, sha1_sum=None): -# bytes_to_write = int(self.info.get_absolute_minimum_part_size() * 2.5) -# with TempDir() as temp_dir: -# temp_dir = pathlib.Path(temp_dir) -# source_large_file = pathlib.Path(temp_dir) / 'source_large_file' -# with open(source_large_file, 'wb') as large_file: -# self.write_zeros(large_file, bytes_to_write) -# bucket.upload_local_file( -# source_large_file, -# 'large_file', -# sha1_sum='do_not_verify', -# ) -# target_large_file = pathlib.Path(temp_dir) / 'target_large_file' -# -# f = bucket.download_file_by_name('large_file') -# f.save_to(target_large_file) -# assert hex_sha1_of_file(source_large_file) == hex_sha1_of_file(target_large_file) -# return f -# -# def test_small(self): -# bucket = self.create_bucket() -# f = self._file_helper(bucket) -# assert not f.download_version.content_sha1_verified -# -# def test_small_unverified(self): -# bucket = self.create_bucket() -# f = self._file_helper(bucket, sha1_sum='do_not_verify') -# assert not f.download_version.content_sha1_verified +class TestDownload(IntegrationTestBase): + def test_large_file(self): + bucket = self.create_bucket() + with mock.patch.object( + self.info, '_recommended_part_size', new=self.info.get_absolute_minimum_part_size() + ): + download_manager = self.b2_api.services.download_manager + with mock.patch.object( + download_manager, + 'strategies', + new=[ + ParallelDownloader( + min_part_size=self.info.get_absolute_minimum_part_size(), + min_chunk_size=download_manager.MIN_CHUNK_SIZE, + max_chunk_size=download_manager.MAX_CHUNK_SIZE, + thread_pool=download_manager._thread_pool, + ) + ] + ): + + # let's check that small file downloads fail with these settings + bucket.upload_bytes(b'0', 'a_single_zero') + with pytest.raises(ValueError) as exc_info: + with io.BytesIO() as io_: + bucket.download_file_by_name('a_single_zero').save(io_) + assert exc_info.value.args == ('no strategy suitable for download was found!',) + f = self._file_helper(bucket) + assert f.download_version.content_sha1_verified + + def _file_helper(self, bucket, sha1_sum=None): + bytes_to_write = int(self.info.get_absolute_minimum_part_size() * 2.5) + with TempDir() as temp_dir: + temp_dir = pathlib.Path(temp_dir) + source_large_file = pathlib.Path(temp_dir) / 'source_large_file' + with open(source_large_file, 'wb') as large_file: + self.write_zeros(large_file, bytes_to_write) + bucket.upload_local_file( + source_large_file, + 'large_file', + sha1_sum='do_not_verify', + ) + target_large_file = pathlib.Path(temp_dir) / 'target_large_file' + + f = bucket.download_file_by_name('large_file') + f.save_to(target_large_file) + assert hex_sha1_of_file(source_large_file) == hex_sha1_of_file(target_large_file) + return f + + def test_small(self): + bucket = self.create_bucket() + f = self._file_helper(bucket) + assert not f.download_version.content_sha1_verified + + def test_small_unverified(self): + bucket = self.create_bucket() + f = self._file_helper(bucket, sha1_sum='do_not_verify') + assert not f.download_version.content_sha1_verified From e364a6eeb7c40c7cd109c2e7bea717a2e81266bc Mon Sep 17 00:00:00 2001 From: Pawel Polewicz Date: Mon, 18 Apr 2022 00:09:12 +0200 Subject: [PATCH 05/13] Tweak integration download test --- test/integration/test_download.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/integration/test_download.py b/test/integration/test_download.py index d950dd7c9..a0dabbbb4 100644 --- a/test/integration/test_download.py +++ b/test/integration/test_download.py @@ -51,7 +51,7 @@ def test_large_file(self): assert f.download_version.content_sha1_verified def _file_helper(self, bucket, sha1_sum=None): - bytes_to_write = int(self.info.get_absolute_minimum_part_size() * 2.5) + bytes_to_write = int(self.info.get_absolute_minimum_part_size()) + 1 with TempDir() as temp_dir: temp_dir = pathlib.Path(temp_dir) source_large_file = pathlib.Path(temp_dir) / 'source_large_file' @@ -60,7 +60,7 @@ def _file_helper(self, bucket, sha1_sum=None): bucket.upload_local_file( source_large_file, 'large_file', - sha1_sum='do_not_verify', + sha1_sum=sha1_sum, ) target_large_file = pathlib.Path(temp_dir) / 'target_large_file' From 59052a3da27783e00d4c37837d9645414a37168e Mon Sep 17 00:00:00 2001 From: Pawel Polewicz Date: Tue, 3 May 2022 22:18:09 +0200 Subject: [PATCH 06/13] Fix account info is_master_key() --- CHANGELOG.md | 3 +++ b2sdk/account_info/abstract.py | 11 ++++++++++- test/unit/account_info/test_account_info.py | 21 +++++++++++++++++++++ 3 files changed, 34 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4f5beefa5..05f0e06d4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Fixed +* Fix account info `is_master_key()` + ## [1.16.0] - 2022-04-27 This release contains a preview of replication support. It allows for basic diff --git a/b2sdk/account_info/abstract.py b/b2sdk/account_info/abstract.py index 81803e356..f144c6a68 100644 --- a/b2sdk/account_info/abstract.py +++ b/b2sdk/account_info/abstract.py @@ -127,7 +127,16 @@ def is_same_account(self, account_id: str, realm: str) -> bool: return False def is_master_key(self) -> bool: - return self.get_account_id() == self.get_application_key_id() + application_key_id = self.get_application_key_id() + account_id = self.get_account_id() + new_style_master_key_suffix = '0000000000' + if account_id == application_key_id: + return True # old style + if len(application_key_id) == (3 + len(account_id) + len(new_style_master_key_suffix)): # 3 for cluster id + # new style + if application_key_id.endswith(account_id + new_style_master_key_suffix): + return True + return False @abstractmethod def get_account_id(self): diff --git a/test/unit/account_info/test_account_info.py b/test/unit/account_info/test_account_info.py index 27e96e504..62df7ed12 100644 --- a/test/unit/account_info/test_account_info.py +++ b/test/unit/account_info/test_account_info.py @@ -63,6 +63,27 @@ def test_is_same_key(self, application_key_id, realm, expected): assert account_info.is_same_key(application_key_id, realm) is expected + @pytest.mark.parametrize( + 'account_id,application_key_id,expected', + ( + ('account_id', 'account_id', True), + ('account_id', 'ACCOUNT_ID', False), + ('account_id', '123account_id0000000000', True), + ('account_id', '234account_id0000000000', True), + ('account_id', '123account_id000000000', False), + ('account_id', '123account_id0000000001', False), + ('account_id', '123account_id00000000000', False), + ), + ) + def test_is_master_key(self, account_id, application_key_id, expected): + account_info = self.account_info_factory() + account_data = self.account_info_default_data.copy() + account_data['account_id'] = account_id + account_data['application_key_id'] = application_key_id + account_info.set_auth_data(**account_data) + + assert account_info.is_master_key() is expected, (account_id, application_key_id, expected) + @pytest.mark.parametrize( 'account_id,realm,expected', ( From cf46c5aefd183a075344579beffa06fe2bf3b521 Mon Sep 17 00:00:00 2001 From: Pawel Polewicz Date: Wed, 4 May 2022 15:50:23 +0200 Subject: [PATCH 07/13] yapf --- b2sdk/account_info/abstract.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/b2sdk/account_info/abstract.py b/b2sdk/account_info/abstract.py index f144c6a68..5b4fe1fde 100644 --- a/b2sdk/account_info/abstract.py +++ b/b2sdk/account_info/abstract.py @@ -132,7 +132,8 @@ def is_master_key(self) -> bool: new_style_master_key_suffix = '0000000000' if account_id == application_key_id: return True # old style - if len(application_key_id) == (3 + len(account_id) + len(new_style_master_key_suffix)): # 3 for cluster id + if len(application_key_id + ) == (3 + len(account_id) + len(new_style_master_key_suffix)): # 3 for cluster id # new style if application_key_id.endswith(account_id + new_style_master_key_suffix): return True From 9979a363dc80ca1f64574ee119332ed604dc6e2d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Polewicz?= Date: Tue, 24 May 2022 19:22:57 +0200 Subject: [PATCH 08/13] Add 3.11.0-beta.1 to test matrix --- .github/workflows/ci.yml | 2 +- CHANGELOG.md | 3 +++ noxfile.py | 9 ++++++++- 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 652bab4fc..4d7d70d0f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -80,7 +80,7 @@ jobs: fail-fast: false matrix: os: ["ubuntu-latest", "macos-latest", "windows-latest"] - python-version: ["3.7", "3.8", "3.9", "3.10", "pypy-3.7", "pypy-3.8"] + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11.0-beta.1", "pypy-3.7", "pypy-3.8"] exclude: - os: "macos-latest" python-version: "pypy-3.7" diff --git a/CHANGELOG.md b/CHANGELOG.md index 4f5beefa5..26849d4bd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Infrastructure +* Add 3.11.0-beta.1 to CI + ## [1.16.0] - 2022-04-27 This release contains a preview of replication support. It allows for basic diff --git a/noxfile.py b/noxfile.py index 3e306c17d..6fac81a29 100644 --- a/noxfile.py +++ b/noxfile.py @@ -18,7 +18,14 @@ NOX_PYTHONS = os.environ.get('NOX_PYTHONS') SKIP_COVERAGE = os.environ.get('SKIP_COVERAGE') == 'true' -PYTHON_VERSIONS = ['3.7', '3.8', '3.9', '3.10'] if NOX_PYTHONS is None else NOX_PYTHONS.split(',') +PYTHON_VERSIONS = [ + '3.7', + '3.8', + '3.9', + '3.10', + '3.11', +] if NOX_PYTHONS is None else NOX_PYTHONS.split(',') + PYTHON_DEFAULT_VERSION = PYTHON_VERSIONS[-1] PY_PATHS = ['b2sdk', 'test', 'noxfile.py', 'setup.py'] From a6c02a0974937a39559783d7fe89255b6e256418 Mon Sep 17 00:00:00 2001 From: Malwina Date: Wed, 1 Jun 2022 09:33:22 +0200 Subject: [PATCH 09/13] Fix IllegalEnum crashing tests on 3.11 --- test/unit/sync/test_sync.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/test/unit/sync/test_sync.py b/test/unit/sync/test_sync.py index 887d61520..fc9de10f4 100644 --- a/test/unit/sync/test_sync.py +++ b/test/unit/sync/test_sync.py @@ -2,7 +2,7 @@ # # File: test/unit/sync/test_sync.py # -# Copyright 2020 Backblaze Inc. All Rights Reserved. +# Copyright 2022 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # @@ -22,10 +22,11 @@ TODAY = DAY * 100 # an arbitrary reference time for testing -class TestSynchronizer: - class IllegalEnum(Enum): - ILLEGAL = 5100 +class IllegalEnum(Enum): + ILLEGAL = 5100 + +class TestSynchronizer: @pytest.fixture(autouse=True) def setup(self, folder_factory, mocker, apiver): self.folder_factory = folder_factory From 744f97108e2ad0421e1010c5f316c2642c41a4e5 Mon Sep 17 00:00:00 2001 From: Malwina Date: Wed, 1 Jun 2022 09:59:26 +0200 Subject: [PATCH 10/13] Add `include_existing_files` parameter to `ReplicationSetupHelper` --- CHANGELOG.md | 3 +++ b2sdk/replication/setup.py | 4 ++++ test/unit/v_all/test_replication.py | 4 +++- 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4f5beefa5..983255cd8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added +* Add `include_existing_files` parameter to `ReplicationSetupHelper` + ## [1.16.0] - 2022-04-27 This release contains a preview of replication support. It allows for basic diff --git a/b2sdk/replication/setup.py b/b2sdk/replication/setup.py index 5281ef185..037531e8d 100644 --- a/b2sdk/replication/setup.py +++ b/b2sdk/replication/setup.py @@ -63,6 +63,7 @@ def setup_both( name: Optional[str] = None, #: name for the new replication rule priority: int = None, #: priority for the new replication rule prefix: Optional[str] = None, + include_existing_files: bool = False, ) -> Tuple[Bucket, Bucket]: new_source_bucket = self.setup_source( @@ -71,6 +72,7 @@ def setup_both( prefix, name, priority, + include_existing_files, ) new_destination_bucket = self.setup_destination( @@ -160,6 +162,7 @@ def setup_source( prefix: Optional[str] = None, name: Optional[str] = None, #: name for the new replication rule priority: int = None, #: priority for the new replication rule + include_existing_files: bool = False, ) -> Bucket: if prefix is None: prefix = "" @@ -193,6 +196,7 @@ def setup_source( priority=priority, destination_bucket_id=destination_bucket.id_, file_name_prefix=prefix, + include_existing_files=include_existing_files, ) new_replication_configuration = ReplicationConfiguration( ReplicationSourceConfiguration( diff --git a/test/unit/v_all/test_replication.py b/test/unit/v_all/test_replication.py index d1bf838aa..d7f95b569 100644 --- a/test/unit/v_all/test_replication.py +++ b/test/unit/v_all/test_replication.py @@ -2,7 +2,7 @@ # # File: test/unit/v_all/test_replication.py # -# Copyright 2021 Backblaze Inc. All Rights Reserved. +# Copyright 2022 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # @@ -104,6 +104,7 @@ def test_setup_both(self): source_bucket=source_bucket, destination_bucket=destination_bucket, prefix='ad', + include_existing_files=True, ) keymap = {k.key_name: k for k in self.api.list_keys()} @@ -123,6 +124,7 @@ def test_setup_both(self): file_name_prefix='ad', is_enabled=True, priority=133, + include_existing_files=True, ), ], source_application_key_id=old_source_application_key.id_, From db15b2b85408f0fe1b046d9de52d279e3eb58c68 Mon Sep 17 00:00:00 2001 From: Malwina Date: Wed, 1 Jun 2022 10:18:16 +0200 Subject: [PATCH 11/13] lint --- test/integration/test_download.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/integration/test_download.py b/test/integration/test_download.py index a0dabbbb4..5fade9849 100644 --- a/test/integration/test_download.py +++ b/test/integration/test_download.py @@ -21,6 +21,7 @@ from .helpers import GENERAL_BUCKET_NAME_PREFIX from .base import IntegrationTestBase + class TestDownload(IntegrationTestBase): def test_large_file(self): bucket = self.create_bucket() From 3c4c525f658546a633771048da7e044a91888ec1 Mon Sep 17 00:00:00 2001 From: Malwina Date: Wed, 1 Jun 2022 10:37:52 +0200 Subject: [PATCH 12/13] Fix download integration test --- test/integration/test_download.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/integration/test_download.py b/test/integration/test_download.py index 5fade9849..8af40f216 100644 --- a/test/integration/test_download.py +++ b/test/integration/test_download.py @@ -52,7 +52,7 @@ def test_large_file(self): assert f.download_version.content_sha1_verified def _file_helper(self, bucket, sha1_sum=None): - bytes_to_write = int(self.info.get_absolute_minimum_part_size()) + 1 + bytes_to_write = int(self.info.get_absolute_minimum_part_size()) * 2 + 1 with TempDir() as temp_dir: temp_dir = pathlib.Path(temp_dir) source_large_file = pathlib.Path(temp_dir) / 'source_large_file' @@ -73,7 +73,7 @@ def _file_helper(self, bucket, sha1_sum=None): def test_small(self): bucket = self.create_bucket() f = self._file_helper(bucket) - assert not f.download_version.content_sha1_verified + assert f.download_version.content_sha1_verified def test_small_unverified(self): bucket = self.create_bucket() From 207cc8147eb2e98a527dbf91ab065cc9a75c7f29 Mon Sep 17 00:00:00 2001 From: Aleksandr Goncharov Date: Wed, 13 Apr 2022 14:07:22 +0300 Subject: [PATCH 13/13] Fix docstring for SqliteAccountInfo.__init__ --- CHANGELOG.md | 1 + b2sdk/account_info/sqlite_account_info.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index deb40ee72..83ebd7c33 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed * Fix `AccountInfo.is_master_key()` +* Fix docstring of `SqliteAccountInfo` ### Infrastructure * Add 3.11.0-beta.1 to CI diff --git a/b2sdk/account_info/sqlite_account_info.py b/b2sdk/account_info/sqlite_account_info.py index 2a0d81d13..c2e7d4506 100644 --- a/b2sdk/account_info/sqlite_account_info.py +++ b/b2sdk/account_info/sqlite_account_info.py @@ -53,10 +53,12 @@ def __init__(self, file_name=None, last_upgrade_to_run=None, profile: Optional[s SqliteAccountInfo currently checks locations in the following order: If ``profile`` arg is provided: + * ``{XDG_CONFIG_HOME_ENV_VAR}/b2/db-.sqlite``, if ``{XDG_CONFIG_HOME_ENV_VAR}`` env var is set * ``{B2_ACCOUNT_INFO_PROFILE_FILE}`` Otherwise: + * ``file_name``, if truthy * ``{B2_ACCOUNT_INFO_ENV_VAR}`` env var's value, if set * ``{B2_ACCOUNT_INFO_DEFAULT_FILE}``, if it exists