Merge branch 'master' into dependabot/pip/sphinx-lt-6.0

.github/workflows/ci.yml

@@ -80,7 +80,7 @@ jobs:
       fail-fast: false
       matrix:
         os: ["ubuntu-latest", "macos-latest", "windows-latest"]
-        python-version: ["3.7", "3.8", "3.9", "3.10", "pypy-3.7", "pypy-3.8"]
+        python-version: ["3.7", "3.8", "3.9", "3.10", "3.11.0-beta.1", "pypy-3.7", "pypy-3.8"]
         exclude:
           - os: "macos-latest"
             python-version: "pypy-3.7"

CHANGELOG.md

@@ -6,7 +6,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+* Add `include_existing_files` parameter to `ReplicationSetupHelper`
+
+### Fixed
+* Fix `AccountInfo.is_master_key()`
+* Fix docstring of `SqliteAccountInfo`
+
 ### Infrastructure
+* Add 3.11.0-beta.1 to CI
 * Change Sphinx major version from 5 to 6
 
 ## [1.16.0] - 2022-04-27

b2sdk/account_info/abstract.py

@@ -127,7 +127,17 @@ def is_same_account(self, account_id: str, realm: str) -> bool:
             return False
 
     def is_master_key(self) -> bool:
-        return self.get_account_id() == self.get_application_key_id()
+        application_key_id = self.get_application_key_id()
+        account_id = self.get_account_id()
+        new_style_master_key_suffix = '0000000000'
+        if account_id == application_key_id:
+            return True  # old style
+        if len(application_key_id
+              ) == (3 + len(account_id) + len(new_style_master_key_suffix)):  # 3 for cluster id
+            # new style
+            if application_key_id.endswith(account_id + new_style_master_key_suffix):
+                return True
+        return False
 
     @abstractmethod
     def get_account_id(self):

b2sdk/account_info/sqlite_account_info.py

@@ -53,10 +53,12 @@ def __init__(self, file_name=None, last_upgrade_to_run=None, profile: Optional[s
         SqliteAccountInfo currently checks locations in the following order:
 
         If ``profile`` arg is provided:
+
         * ``{XDG_CONFIG_HOME_ENV_VAR}/b2/db-<profile>.sqlite``, if ``{XDG_CONFIG_HOME_ENV_VAR}`` env var is set
         * ``{B2_ACCOUNT_INFO_PROFILE_FILE}``
 
         Otherwise:
+
         * ``file_name``, if truthy
         * ``{B2_ACCOUNT_INFO_ENV_VAR}`` env var's value, if set
         * ``{B2_ACCOUNT_INFO_DEFAULT_FILE}``, if it exists

b2sdk/replication/setting.py

@@ -33,7 +33,7 @@ class ReplicationRule:
 
     REPLICATION_RULE_REGEX: ClassVar = re.compile(r'^[a-zA-Z0-9_\-]{1,64}$')
     MIN_PRIORITY: ClassVar[int] = 1
-    MAX_PRIORITY: ClassVar[int] = 2147483647
+    MAX_PRIORITY: ClassVar[int] = 2**31 - 1
 
     def __post_init__(self):
         if not self.destination_bucket_id:

b2sdk/replication/setup.py

@@ -17,6 +17,7 @@
 # b2 replication-accept destinationBucketName sourceKeyId [destinationKeyId]
 # b2 replication-deny destinationBucketName sourceKeyId
 
+from collections.abc import Iterable
 from typing import ClassVar, List, Optional, Tuple
 import itertools
 import logging
@@ -29,9 +30,14 @@
 
 logger = logging.getLogger(__name__)
 
+try:
+    Iterable[str]
+except TypeError:
+    Iterable = List  # Remove after dropping Python 3.8
+
 
 class ReplicationSetupHelper(metaclass=B2TraceMeta):
-    """ class with various methods that help with repliction management """
+    """ class with various methods that help with setting up repliction """
     PRIORITY_OFFSET: ClassVar[int] = 5  #: how far to to put the new rule from the existing rules
     DEFAULT_PRIORITY: ClassVar[
         int
@@ -50,39 +56,38 @@ class ReplicationSetupHelper(metaclass=B2TraceMeta):
         'deleteFiles',
     )
 
-    def __init__(self, source_b2api: B2Api = None, destination_b2api: B2Api = None):
-        assert source_b2api is not None or destination_b2api is not None
-        self.source_b2api = source_b2api
-        self.destination_b2api = destination_b2api
-
     def setup_both(
         self,
-        source_bucket_name: str,
+        source_bucket: Bucket,
         destination_bucket: Bucket,
         name: Optional[str] = None,  #: name for the new replication rule
         priority: int = None,  #: priority for the new replication rule
         prefix: Optional[str] = None,
+        include_existing_files: bool = False,
     ) -> Tuple[Bucket, Bucket]:
-        source_bucket = self.setup_source(
-            source_bucket_name,
-            prefix,
+
+        new_source_bucket = self.setup_source(
+            source_bucket,
             destination_bucket,
+            prefix,
             name,
             priority,
+            include_existing_files,
         )
-        destination_bucket = self.setup_destination(
-            source_bucket.replication.as_replication_source.source_application_key_id,
+
+        new_destination_bucket = self.setup_destination(
+            new_source_bucket.replication.as_replication_source.source_application_key_id,
             destination_bucket,
         )
-        return source_bucket, destination_bucket
+
+        return new_source_bucket, new_destination_bucket
 
     def setup_destination(
         self,
         source_key_id: str,
         destination_bucket: Bucket,
     ) -> Bucket:
         api: B2Api = destination_bucket.api
-        destination_bucket = api.list_buckets(destination_bucket.name)[0]  # fresh!
         if destination_bucket.replication is None or destination_bucket.replication.as_replication_source is None:
             source_configuration = None
         else:
@@ -145,28 +150,27 @@ def _get_destination_key(
             logger.debug("no matching key found, making a new one")
             key = cls._create_destination_key(
                 name=destination_bucket.name[:91] + '-replidst',
-                api=api,
-                bucket_id=destination_bucket.id_,
+                bucket=destination_bucket,
                 prefix=None,
             )
         return keys_to_purge, key
 
     def setup_source(
         self,
-        source_bucket_name,
-        prefix,
+        source_bucket: Bucket,
         destination_bucket: Bucket,
-        name,
-        priority,
+        prefix: Optional[str] = None,
+        name: Optional[str] = None,  #: name for the new replication rule
+        priority: int = None,  #: priority for the new replication rule
+        include_existing_files: bool = False,
     ) -> Bucket:
-        source_bucket: Bucket = self.source_b2api.list_buckets(source_bucket_name)[0]  # fresh!
         if prefix is None:
             prefix = ""
 
         try:
-            current_source_rrs = source_bucket.replication.as_replication_source.rules
+            current_source_rules = source_bucket.replication.as_replication_source.rules
         except (NameError, AttributeError):
-            current_source_rrs = []
+            current_source_rules = []
         try:
             destination_configuration = source_bucket.replication.as_replication_destination
         except (NameError, AttributeError):
@@ -176,27 +180,28 @@ def setup_source(
             source_bucket,
             prefix,
             source_bucket.replication,
-            current_source_rrs,
+            current_source_rules,
         )
         priority = self._get_priority_for_new_rule(
+            current_source_rules,
             priority,
-            current_source_rrs,
         )
-        name = self._get_name_for_new_rule(
+        name = self._get_new_rule_name(
+            current_source_rules,
+            destination_bucket,
             name,
-            current_source_rrs,
-            destination_bucket.name,
         )
         new_rr = ReplicationRule(
             name=name,
             priority=priority,
             destination_bucket_id=destination_bucket.id_,
             file_name_prefix=prefix,
+            include_existing_files=include_existing_files,
         )
         new_replication_configuration = ReplicationConfiguration(
             ReplicationSourceConfiguration(
                 source_application_key_id=source_key.id_,
-                rules=current_source_rrs + [new_rr],
+                rules=current_source_rules + [new_rr],
             ),
             destination_configuration,
         )
@@ -208,10 +213,10 @@ def setup_source(
     @classmethod
     def _get_source_key(
         cls,
-        source_bucket,
-        prefix,
+        source_bucket: Bucket,
+        prefix: str,
         current_replication_configuration: ReplicationConfiguration,
-        current_source_rrs,
+        current_source_rules: Iterable[ReplicationRule],
     ) -> ApplicationKey:
         api = source_bucket.api
 
@@ -227,9 +232,9 @@ def _get_source_key(
 
         new_key = cls._create_source_key(
             name=source_bucket.name[:91] + '-replisrc',
-            api=api,
-            bucket_id=source_bucket.id_,
-        )  # no prefix!
+            bucket=source_bucket,
+            prefix=prefix,
+        )
         return new_key
 
     @classmethod
@@ -269,67 +274,74 @@ def _should_make_new_source_key(
     def _create_source_key(
         cls,
         name: str,
-        api: B2Api,
-        bucket_id: str,
+        bucket: Bucket,
         prefix: Optional[str] = None,
     ) -> ApplicationKey:
+        # in this implementation we ignore the prefix and create a full key, because
+        # if someone would need a different (wider) key later, all replication
+        # destinations would have to start using new keys and it's not feasible
+        # from organizational perspective, while the prefix of uploaded files can be
+        # restricted on the rule level
+        prefix = None
         capabilities = cls.DEFAULT_SOURCE_CAPABILITIES
-        return cls._create_key(name, api, bucket_id, prefix, capabilities)
+        return cls._create_key(name, bucket, prefix, capabilities)
 
     @classmethod
     def _create_destination_key(
         cls,
         name: str,
-        api: B2Api,
-        bucket_id: str,
+        bucket: Bucket,
         prefix: Optional[str] = None,
     ) -> ApplicationKey:
         capabilities = cls.DEFAULT_DESTINATION_CAPABILITIES
-        return cls._create_key(name, api, bucket_id, prefix, capabilities)
+        return cls._create_key(name, bucket, prefix, capabilities)
 
     @classmethod
     def _create_key(
         cls,
         name: str,
-        api: B2Api,
-        bucket_id: str,
+        bucket: Bucket,
         prefix: Optional[str] = None,
         capabilities=tuple(),
     ) -> ApplicationKey:
+        api: B2Api = bucket.api
         return api.create_key(
             capabilities=capabilities,
             key_name=name,
-            bucket_id=bucket_id,
+            bucket_id=bucket.id_,
             name_prefix=prefix,
         )
 
     @classmethod
-    def _get_narrowest_common_prefix(cls, widen_to: List[str]) -> str:
-        for path in widen_to:
-            pass  # TODO
-        return ''
-
-    @classmethod
-    def _get_priority_for_new_rule(cls, priority, current_source_rrs):
-        # if there is no priority hint, look into current rules to determine the last priority and add a constant to it
+    def _get_priority_for_new_rule(
+        cls,
+        current_rules: Iterable[ReplicationRule],
+        priority: Optional[int] = None,
+    ):
         if priority is not None:
             return priority
-        if current_source_rrs:
-            # TODO: maybe handle a case where the existing rrs need to have their priorities decreased to make space
-            existing_priority = max(rr.priority for rr in current_source_rrs)
+        if current_rules:
+            # ignore a case where the existing rrs need to have their priorities decreased to make space (max is 2**31-1)
+            existing_priority = max(rr.priority for rr in current_rules)
             return min(existing_priority + cls.PRIORITY_OFFSET, cls.MAX_PRIORITY)
         return cls.DEFAULT_PRIORITY
 
     @classmethod
-    def _get_name_for_new_rule(
-        cls, name: Optional[str], current_source_rrs, destination_bucket_name
+    def _get_new_rule_name(
+        cls,
+        current_rules: Iterable[ReplicationRule],
+        destination_bucket: Bucket,
+        name: Optional[str] = None,
     ):
         if name is not None:
             return name
-        existing_names = set(rr.name for rr in current_source_rrs)
+        existing_names = set(rr.name for rr in current_rules)
         suffixes = cls._get_rule_name_candidate_suffixes()
         while True:
-            candidate = '%s%s' % (destination_bucket_name, next(suffixes))
+            candidate = '%s%s' % (
+                destination_bucket.name,
+                next(suffixes),
+            )  # use := after dropping 3.7
             if candidate not in existing_names:
                 return candidate
 

noxfile.py

@@ -18,7 +18,14 @@
 NOX_PYTHONS = os.environ.get('NOX_PYTHONS')
 SKIP_COVERAGE = os.environ.get('SKIP_COVERAGE') == 'true'
 
-PYTHON_VERSIONS = ['3.7', '3.8', '3.9', '3.10'] if NOX_PYTHONS is None else NOX_PYTHONS.split(',')
+PYTHON_VERSIONS = [
+    '3.7',
+    '3.8',
+    '3.9',
+    '3.10',
+    '3.11',
+] if NOX_PYTHONS is None else NOX_PYTHONS.split(',')
+
 PYTHON_DEFAULT_VERSION = PYTHON_VERSIONS[-1]
 
 PY_PATHS = ['b2sdk', 'test', 'noxfile.py', 'setup.py']