Skip to content

Commit

Permalink
feat: add blocking interface
Browse files Browse the repository at this point in the history
  • Loading branch information
gadomski committed Aug 22, 2023
1 parent e2b8415 commit bc34c28
Show file tree
Hide file tree
Showing 7 changed files with 274 additions and 10 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
- `ErrorStrategy` ([#69](https://github.com/stac-utils/stac-asset/pull/69))
- `fail_fast` ([#69](https://github.com/stac-utils/stac-asset/pull/69))
- `assert_asset_exists`, `asset_exists`, `Client.assert_href_exists`, `Client.href_exists` ([#81](https://github.com/stac-utils/stac-asset/pull/81), [#85](https://github.com/stac-utils/stac-asset/pull/85))
- Blocking interface ([#86](https://github.com/stac-utils/stac-asset/pull/86))

### Changed

Expand Down
11 changes: 10 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,16 @@ import stac_asset

href = "https://raw.githubusercontent.com/radiantearth/stac-spec/master/examples/simple-item.json"
item = pystac.from_file(href)
await stac_asset.download_item(item, ".")
item = await stac_asset.download_item(item, ".")
```

If you're working in a fully synchronous application, you can use our blocking interface:

```python
import stac_asset.blocking
href = "https://raw.githubusercontent.com/radiantearth/stac-spec/master/examples/simple-item.json"
item = pystac.from_file(href)
item = stac_asset.blocking.download_item(item, ".")
```

### CLI
Expand Down
6 changes: 6 additions & 0 deletions docs/api.rst
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,12 @@ API documentation
.. automodule:: stac_asset
:members:

stac_asset.blocking
-------------------

.. automodule:: stac_asset.blocking
:members:

stac_asset.validate
-------------------

Expand Down
11 changes: 2 additions & 9 deletions src/stac_asset/_functions.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,6 @@
from pathlib import Path
from types import TracebackType
from typing import (
TYPE_CHECKING,
Any,
List,
Optional,
Set,
Expand All @@ -33,13 +31,7 @@
StartAssetDownload,
)
from .strategy import ErrorStrategy, FileNameStrategy
from .types import PathLikeObject

# Needed until we drop Python 3.8
if TYPE_CHECKING:
AnyQueue = Queue[Any]
else:
AnyQueue = Queue
from .types import AnyQueue, PathLikeObject


@dataclass
Expand Down Expand Up @@ -387,6 +379,7 @@ async def download_asset(
)
raise error

asset.href = str(path)
if messages:
await messages.put(FinishAssetDownload(key=key, href=href, path=path))
return asset
Expand Down
211 changes: 211 additions & 0 deletions src/stac_asset/blocking.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,211 @@
"""Blocking interfaces for functions.
These should only be used from fully synchronous code. If you have _any_ async
code in your application, prefer the top-level functions.
"""

import asyncio
from asyncio import Queue
from pathlib import Path
from typing import List, Optional

from pystac import Asset, Collection, Item, ItemCollection

from . import _functions
from .client import Client, Clients
from .config import Config
from .messages import Message
from .types import AnyQueue, PathLikeObject


def download_item(
item: Item,
directory: PathLikeObject,
file_name: Optional[str] = None,
infer_file_name: bool = True,
config: Optional[Config] = None,
queue: Optional[AnyQueue] = None,
clients: Optional[List[Client]] = None,
) -> Item:
"""Downloads an item to the local filesystem, synchronously.
Args:
item: The :py:class:`pystac.Item`.
directory: The output directory that will hold the items and assets.
file_name: The name of the item file to save. If not provided, will not
be saved.
infer_file_name: If ``file_name`` is None, infer the file name from the
item's id. This argument is unused if ``file_name`` is not None.
config: The download configuration
queue: An optional queue to use for progress reporting
clients: Pre-configured clients to use for access
Returns:
Item: The `~pystac.Item`, with the updated asset hrefs and self href.
Raises:
ValueError: Raised if the item doesn't have any assets.
"""
return asyncio.run(
_functions.download_item(
item=item,
directory=directory,
file_name=file_name,
infer_file_name=infer_file_name,
config=config,
queue=queue,
clients=clients,
)
)


def download_collection(
collection: Collection,
directory: PathLikeObject,
file_name: Optional[str] = "collection.json",
config: Optional[Config] = None,
queue: Optional[AnyQueue] = None,
clients: Optional[List[Client]] = None,
) -> Collection:
"""Downloads a collection to the local filesystem, synchronously.
Does not download the collection's items' assets -- use
:py:func:`download_item_collection` to download multiple items.
Args:
collection: A pystac collection
directory: The destination directory
file_name: The name of the collection file to save. If not provided,
will not be saved.
config: The download configuration
queue: An optional queue to use for progress reporting
clients: Pre-configured clients to use for access
Returns:
Collection: The collection, with updated asset hrefs
Raises:
CantIncludeAndExclude: Raised if both include and exclude are not None.
"""
return asyncio.run(
_functions.download_collection(
collection=collection,
directory=directory,
file_name=file_name,
config=config,
queue=queue,
clients=clients,
)
)


def download_item_collection(
item_collection: ItemCollection,
directory: PathLikeObject,
file_name: Optional[str] = "item-collection.json",
config: Optional[Config] = None,
queue: Optional[AnyQueue] = None,
clients: Optional[List[Client]] = None,
) -> ItemCollection:
"""Downloads an item collection to the local filesystem, synchronously.
Args:
item_collection: The item collection to download
directory: The destination directory
file_name: The name of the item collection file to save. If not
provided, will not be saved.
config: The download configuration
queue: An optional queue to use for progress reporting
clients: Pre-configured clients to use for access
Returns:
ItemCollection: The item collection, with updated asset hrefs
Raises:
CantIncludeAndExclude: Raised if both include and exclude are not None.
"""
return asyncio.run(
_functions.download_item_collection(
item_collection=item_collection,
directory=directory,
file_name=file_name,
config=config,
queue=queue,
clients=clients,
)
)


def download_asset(
key: str,
asset: Asset,
path: Path,
config: Config,
messages: Optional[Queue[Message]] = None,
clients: Optional[Clients] = None,
) -> Asset:
"""Downloads an asset, synchronously.
Args:
key: The asset key
asset: The asset
path: The path to which the asset will be downloaded
config: The download configuration
messages: An optional queue to use for progress reporting
clients: A async-safe cache of clients. If not provided, a new one
will be created.
Returns:
Asset: The asset with an updated href
Raises:
ValueError: Raised if the asset does not have an absolute href
"""
return asyncio.run(
_functions.download_asset(
key=key,
asset=asset,
path=path,
config=config,
messages=messages,
clients=clients,
)
)


def assert_asset_exists(
asset: Asset,
config: Optional[Config] = None,
clients: Optional[List[Client]] = None,
) -> None:
"""Asserts that an asset exists, synchronously.
Raises the source error if it does not.
Args:
asset: The asset the check for existence
config: The download configuration to use for the existence check
clients: Any pre-configured clients to use for the existence check
Raises:
Exception: An exception from the underlying client.
"""
asyncio.run(_functions.assert_asset_exists(asset, config, clients))


def asset_exists(
asset: Asset,
config: Optional[Config] = None,
clients: Optional[List[Client]] = None,
) -> bool:
"""Returns true if an asset exists, synchronously.
Args:
asset: The asset the check for existence
config: The download configuration to use for the existence check
clients: Any pre-configured clients to use for the existence check
Returns:
bool: Whether the asset exists or not
"""
return asyncio.run(_functions.asset_exists(asset, config, clients))
4 changes: 4 additions & 0 deletions src/stac_asset/types.py
Original file line number Diff line number Diff line change
@@ -1,12 +1,16 @@
from asyncio import Queue
from os import PathLike
from typing import TYPE_CHECKING, Union

if TYPE_CHECKING:
from typing import Any

_PathLike = PathLike[Any]
# Needed until we drop Python 3.8
AnyQueue = Queue[Any]
else:
_PathLike = PathLike
AnyQueue = Queue

PathLikeObject = Union[_PathLike, str]
"""An object representing a file system path, except we exclude `bytes` because
Expand Down
40 changes: 40 additions & 0 deletions tests/test_blocking.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
from pathlib import Path

import stac_asset.blocking
from pystac import Collection, Item, ItemCollection
from stac_asset import Config


def test_download_item(tmp_path: Path, item: Item) -> None:
item = stac_asset.blocking.download_item(item, tmp_path)
item.validate()


def test_download_collection(tmp_path: Path, collection: Collection) -> None:
collection = stac_asset.blocking.download_collection(collection, tmp_path)
collection.validate()


def test_download_item_collection(
tmp_path: Path, item_collection: ItemCollection
) -> None:
item_collection = stac_asset.blocking.download_item_collection(
item_collection, tmp_path
)
for item in item_collection:
item.validate()


def test_download_asset(tmp_path: Path, item: Item) -> None:
asset = stac_asset.blocking.download_asset(
"data", item.assets["data"], tmp_path / "image.jpg", Config()
)
assert asset.href == tmp_path / "image.jpg"


def test_assert_asset_exists(tmp_path: Path, item: Item) -> None:
stac_asset.blocking.assert_asset_exists(item.assets["data"])


def test_asset_exists(tmp_path: Path, item: Item) -> None:
assert stac_asset.blocking.asset_exists(item.assets["data"])

0 comments on commit bc34c28

Please sign in to comment.