feat: allow setting max_concurrent_downloads

src/stac_asset/_cli.py

@@ -141,6 +141,11 @@ def cli() -> None:
     is_flag=True,
     show_default=True,
 )
+@click.option(
+    "--max-concurrent-downloads",
+    help="The maximum number of downloads that can be active at one time",
+    default=_functions.DEFAULT_MAX_CONCURRENT_DOWNLOADS,
+)
 # TODO add option to disable content type checking
 def download(
     href: Optional[str],
@@ -159,6 +164,7 @@ def download(
     keep: bool,
     fail_fast: bool,
     overwrite: bool,
+    max_concurrent_downloads: int,
 ) -> None:
     """Download STAC assets from an item or item collection.
 
@@ -197,6 +203,7 @@ def download(
             keep=keep,
             fail_fast=fail_fast,
             overwrite=overwrite,
+            max_concurrent_downloads=max_concurrent_downloads,
         )
     )
 
@@ -218,6 +225,7 @@ async def download_async(
     keep: bool,
     fail_fast: bool,
     overwrite: bool,
+    max_concurrent_downloads: int,
 ) -> None:
     config = Config(
         alternate_assets=alternate_assets,
@@ -263,6 +271,7 @@ async def download() -> Union[Item, ItemCollection]:
                 infer_file_name=False,
                 config=config,
                 messages=messages,
+                max_concurrent_downloads=max_concurrent_downloads,
             )
 
     elif type_ == "FeatureCollection":
@@ -276,6 +285,7 @@ async def download() -> Union[Item, ItemCollection]:
                 file_name=file_name,
                 config=config,
                 messages=messages,
+                max_concurrent_downloads=max_concurrent_downloads,
             )
 
     else:

src/stac_asset/_functions.py

@@ -34,6 +34,9 @@
 from .strategy import ErrorStrategy, FileNameStrategy
 from .types import MessageQueue, PathLikeObject
 
+DEFAULT_MAX_CONCURRENT_DOWNLOADS: int = 500
+"""The default number of downloads that can be active at once."""
+
 
 @dataclass
 class Download:
@@ -81,7 +84,7 @@ def __init__(
         self,
         config: Config,
         clients: Optional[List[Client]] = None,
-        max_concurrent_downloads: int = 500,
+        max_concurrent_downloads: int = DEFAULT_MAX_CONCURRENT_DOWNLOADS,
     ) -> None:
         config.validate()
         self.config = config
@@ -217,6 +220,7 @@ async def download_item(
     messages: Optional[MessageQueue] = None,
     clients: Optional[List[Client]] = None,
     keep_non_downloaded: bool = False,
+    max_concurrent_downloads: int = DEFAULT_MAX_CONCURRENT_DOWNLOADS,
 ) -> Item:
     """Downloads an item to the local filesystem.
 
@@ -232,6 +236,8 @@ async def download_item(
         clients: Pre-configured clients to use for access
         keep_non_downloaded: Keep all assets on the item, even if they're not
             downloaded.
+        max_concurrent_downloads: The maximum number of downloads that can be
+            active at one time.
 
     Returns:
         Item: The `~pystac.Item`, with the updated asset hrefs and self href.
@@ -245,6 +251,7 @@ async def download_item(
     async with Downloads(
         config=config or Config(),
         clients=clients,
+        max_concurrent_downloads=max_concurrent_downloads,
     ) as downloads:
         await downloads.add(item, Path(directory), file_name, keep_non_downloaded)
         await downloads.download(messages)
@@ -268,6 +275,7 @@ async def download_collection(
     messages: Optional[MessageQueue] = None,
     clients: Optional[List[Client]] = None,
     keep_non_downloaded: bool = False,
+    max_concurrent_downloads: int = DEFAULT_MAX_CONCURRENT_DOWNLOADS,
 ) -> Collection:
     """Downloads a collection to the local filesystem.
 
@@ -284,14 +292,20 @@ async def download_collection(
         clients: Pre-configured clients to use for access
         keep_non_downloaded: Keep all assets on the item, even if they're not
             downloaded.
+        max_concurrent_downloads: The maximum number of downloads that can be
+            active at one time.
 
     Returns:
         Collection: The collection, with updated asset hrefs
 
     Raises:
         CantIncludeAndExclude: Raised if both include and exclude are not None.
     """
-    async with Downloads(config=config or Config(), clients=clients) as downloads:
+    async with Downloads(
+        config=config or Config(),
+        clients=clients,
+        max_concurrent_downloads=max_concurrent_downloads,
+    ) as downloads:
         await downloads.add(collection, Path(directory), file_name, keep_non_downloaded)
         await downloads.download(messages)
 
@@ -314,6 +328,7 @@ async def download_item_collection(
     messages: Optional[MessageQueue] = None,
     clients: Optional[List[Client]] = None,
     keep_non_downloaded: bool = False,
+    max_concurrent_downloads: int = DEFAULT_MAX_CONCURRENT_DOWNLOADS,
 ) -> ItemCollection:
     """Downloads an item collection to the local filesystem.
 
@@ -329,6 +344,8 @@ async def download_item_collection(
         clients: Pre-configured clients to use for access
         keep_non_downloaded: Keep all assets on the item, even if they're not
             downloaded.
+        max_concurrent_downloads: The maximum number of downloads that can be
+            active at one time.
 
     Returns:
         ItemCollection: The item collection, with updated asset hrefs
@@ -339,7 +356,11 @@ async def download_item_collection(
     layout_template = LayoutTemplate(
         path_template if path_template is not None else "${id}"
     )
-    async with Downloads(config=config or Config(), clients=clients) as downloads:
+    async with Downloads(
+        config=config or Config(),
+        clients=clients,
+        max_concurrent_downloads=max_concurrent_downloads,
+    ) as downloads:
         for item in item_collection.items:
             item.set_self_href(None)
             root = Path(directory) / layout_template.substitute(item)

src/stac_asset/blocking.py

@@ -25,6 +25,7 @@ def download_item(
     messages: Optional[MessageQueue] = None,
     clients: Optional[List[Client]] = None,
     keep_non_downloaded: bool = False,
+    max_concurrent_downloads: int = _functions.DEFAULT_MAX_CONCURRENT_DOWNLOADS,
 ) -> Item:
     """Downloads an item to the local filesystem, synchronously.
 
@@ -40,6 +41,8 @@ def download_item(
         clients: Pre-configured clients to use for access
         keep_non_downloaded: Keep all assets on the item, even if they're not
             downloaded.
+        max_concurrent_downloads: The maximum number of downloads that can be
+            active at one time.
 
     Returns:
         Item: The `~pystac.Item`, with the updated asset hrefs and self href.
@@ -57,6 +60,7 @@ def download_item(
             messages=messages,
             clients=clients,
             keep_non_downloaded=keep_non_downloaded,
+            max_concurrent_downloads=max_concurrent_downloads,
         )
     )
 
@@ -69,6 +73,7 @@ def download_collection(
     messages: Optional[MessageQueue] = None,
     clients: Optional[List[Client]] = None,
     keep_non_downloaded: bool = False,
+    max_concurrent_downloads: int = _functions.DEFAULT_MAX_CONCURRENT_DOWNLOADS,
 ) -> Collection:
     """Downloads a collection to the local filesystem, synchronously.
 
@@ -85,6 +90,8 @@ def download_collection(
         clients: Pre-configured clients to use for access
         keep_non_downloaded: Keep all assets on the item, even if they're not
             downloaded.
+        max_concurrent_downloads: The maximum number of downloads that can be
+            active at one time.
 
     Returns:
         Collection: The collection, with updated asset hrefs
@@ -101,6 +108,7 @@ def download_collection(
             messages=messages,
             clients=clients,
             keep_non_downloaded=keep_non_downloaded,
+            max_concurrent_downloads=max_concurrent_downloads,
         )
     )
 
@@ -114,6 +122,7 @@ def download_item_collection(
     messages: Optional[MessageQueue] = None,
     clients: Optional[List[Client]] = None,
     keep_non_downloaded: bool = False,
+    max_concurrent_downloads: int = _functions.DEFAULT_MAX_CONCURRENT_DOWNLOADS,
 ) -> ItemCollection:
     """Downloads an item collection to the local filesystem, synchronously.
 
@@ -129,6 +138,8 @@ def download_item_collection(
         clients: Pre-configured clients to use for access
         keep_non_downloaded: Keep all assets on the item, even if they're not
             downloaded.
+        max_concurrent_downloads: The maximum number of downloads that can be
+            active at one time.
 
     Returns:
         ItemCollection: The item collection, with updated asset hrefs
@@ -146,6 +157,7 @@ def download_item_collection(
             clients=clients,
             keep_non_downloaded=keep_non_downloaded,
             path_template=path_template,
+            max_concurrent_downloads=max_concurrent_downloads,
         )
     )