Skip to content

Commit

Permalink
feat: add assert_asset_exists, assert_href_exists
Browse files Browse the repository at this point in the history
  • Loading branch information
gadomski committed Aug 22, 2023
1 parent 61be760 commit b1962c5
Show file tree
Hide file tree
Showing 8 changed files with 72 additions and 29 deletions.
2 changes: 2 additions & 0 deletions src/stac_asset/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@


from ._functions import (
assert_asset_exists,
asset_exists,
download_asset,
download_collection,
Expand Down Expand Up @@ -53,6 +54,7 @@
"HttpClient",
"PlanetaryComputerClient",
"S3Client",
"assert_asset_exists",
"asset_exists",
"download_asset",
"download_collection",
Expand Down
37 changes: 31 additions & 6 deletions src/stac_asset/_functions.py
Original file line number Diff line number Diff line change
Expand Up @@ -392,30 +392,55 @@ async def download_asset(
return asset


async def asset_exists(
async def assert_asset_exists(
asset: Asset,
config: Optional[Config] = None,
clients: Optional[List[Client]] = None,
) -> bool:
"""Returns true if the asset exists, false if not.
) -> None:
"""Asserts that an asset exists.
Raises the source error if it does not.
Args:
asset: The asset the check for existence
config: The download configuration to use for the existence check
clients: Any pre-configured clients to use for the existence check
Returns:
bool: Whether the assets href exists or not
Raises:
Exception: An exception from the underlying client.
"""
if config is None:
config = Config()
clients_ = Clients(config, clients=clients)
href = get_absolute_asset_href(asset, config.alternate_assets)
if href:
client = await clients_.get_client(href)
return await client.href_exists(href)
await client.assert_href_exists(href)
else:
raise ValueError("asset does not have an absolute href")


async def asset_exists(
asset: Asset,
config: Optional[Config] = None,
clients: Optional[List[Client]] = None,
) -> bool:
"""Returns true if an asset exists.
Args:
asset: The asset the check for existence
config: The download configuration to use for the existence check
clients: Any pre-configured clients to use for the existence check
Returns:
bool: Whether the asset exists or not
"""
try:
await assert_asset_exists(asset, config, clients)
except Exception:
return False
else:
return True


def make_asset_hrefs_relative(
Expand Down
21 changes: 18 additions & 3 deletions src/stac_asset/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -133,11 +133,26 @@ async def href_exists(self, href: str) -> bool:
bool: Whether the href exists
"""
try:
async for _ in self.open_href(href):
break
await self.assert_href_exists(href)
except Exception:
return False
return True
else:
return True

async def assert_href_exists(self, href: str) -> None:
"""Asserts that a href exists.
The default implementation naïvely opens the href and reads one chunk.
Clients may implement specialized behavior.
Args:
href: The href to open
Raises:
Exception: The underlying error when trying to open the file.
"""
async for _ in self.open_href(href):
break

async def close(self) -> None:
"""Close this client."""
Expand Down
5 changes: 5 additions & 0 deletions src/stac_asset/filesystem_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,11 @@ async def open_url(
async for chunk in f:
yield chunk

async def assert_href_exists(self, href: str) -> None:
"""Asserts that an href exists."""
if not os.path.exists(href):
raise ValueError(f"href does not exist on the filesystem: {href}")

async def __aenter__(self) -> FilesystemClient:
return self

Expand Down
12 changes: 4 additions & 8 deletions src/stac_asset/http_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -74,17 +74,13 @@ async def open_url(
async for chunk, _ in response.content.iter_chunks():
yield chunk

async def href_exists(self, href: str) -> bool:
"""Returns true if the href exists.
async def assert_href_exists(self, href: str) -> None:
"""Asserts that the href exists.
Uses a HEAD request.
"""
try:
async with self.session.head(href) as response:
# For some reason, mypy can't see that this is a bool
return response.status >= 200 and response.status < 300 # type: ignore
except Exception:
return False
async with self.session.head(href) as response:
response.raise_for_status()

async def close(self) -> None:
"""Close this http client.
Expand Down
7 changes: 3 additions & 4 deletions src/stac_asset/planetary_computer_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -99,15 +99,14 @@ async def open_url(
):
yield chunk

async def href_exists(self, href: str) -> bool:
"""Returns true if the href exists.
async def assert_href_exists(self, href: str) -> None:
"""Asserts that the href exists.
Uses a HEAD request on a signed url.
"""
href = await self._maybe_sign_href(href)
async with self.session.head(href) as response:
# For some reason, mypy can't see that this is a bool
return response.status >= 200 and response.status < 300 # type: ignore
response.raise_for_status()

async def _sign(self, url: URL) -> URL:
assert url.host
Expand Down
11 changes: 3 additions & 8 deletions src/stac_asset/s3_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -114,18 +114,13 @@ async def has_credentials(self) -> bool:
"""Returns true if the sessions has credentials."""
return await self.session.get_credentials() is not None

async def href_exists(self, href: str) -> bool:
"""Return true if the href exists.
async def assert_href_exists(self, href: str) -> None:
"""Asserts that the href exists.
Uses ``head_object``
"""
async with self._create_client() as client:
try:
await client.head_object(**self._params(URL(href)))
except Exception:
return False
else:
return True
await client.head_object(**self._params(URL(href)))

def _create_client(self) -> ClientCreatorContext:
retries = {
Expand Down
6 changes: 6 additions & 0 deletions tests/test_functions.py
Original file line number Diff line number Diff line change
Expand Up @@ -189,3 +189,9 @@ async def test_queue(tmp_path: Path, item: Item) -> None:
async def test_asset_exists(item: Item) -> None:
assert await stac_asset.asset_exists(item.assets["data"])
assert not await stac_asset.asset_exists(Asset(href="not-a-file"))


async def test_asert_asset_exists(item: Item) -> None:
await stac_asset.assert_asset_exists(item.assets["data"])
with pytest.raises(ValueError):
await stac_asset.assert_asset_exists(Asset(href="not-a-file"))

0 comments on commit b1962c5

Please sign in to comment.