diff --git a/src/dvc_objects/db.py b/src/dvc_objects/db.py index ae1d667..6ed8cde 100644 --- a/src/dvc_objects/db.py +++ b/src/dvc_objects/db.py @@ -433,6 +433,7 @@ def oids_exist(self, oids, jobs=None, progress=noop): traverse_weight = traverse_pages * self.fs.TRAVERSE_WEIGHT_MULTIPLIER else: traverse_weight = traverse_pages + traverse_weight *= self.fs.EXTRA_TRAVERSE_WEIGHT_MULTIPLIER if len(oids) < traverse_weight and not always_traverse: logger.debug( "Large remote (%r oids < %r traverse weight), " diff --git a/src/dvc_objects/fs/base.py b/src/dvc_objects/fs/base.py index 6ad1b07..10fe29b 100644 --- a/src/dvc_objects/fs/base.py +++ b/src/dvc_objects/fs/base.py @@ -69,6 +69,9 @@ class FileSystem: HASH_JOBS = max(1, min(4, cpu_count() // 2)) LIST_OBJECT_PAGE_SIZE = 1000 TRAVERSE_WEIGHT_MULTIPLIER = 5 + EXTRA_TRAVERSE_WEIGHT_MULTIPLIER = int( + os.getenv("EXTRA_TRAVERSE_WEIGHT_MULTIPLIER", "1") + ) TRAVERSE_PREFIX_LEN = 2 TRAVERSE_THRESHOLD_SIZE = 500000 CAN_TRAVERSE = True