diff --git a/src/stac_asset/__init__.py b/src/stac_asset/__init__.py index 8a52c41..91c8c36 100644 --- a/src/stac_asset/__init__.py +++ b/src/stac_asset/__init__.py @@ -14,7 +14,6 @@ from .config import Config from .earthdata_client import EarthdataClient from .errors import ( - AssetDownloadError, AssetOverwriteError, CannotIncludeAndExclude, DownloadError, @@ -24,7 +23,7 @@ from .functions import ( download_item, download_item_collection, - guess_client, + guess_client_class, ) from .http_client import HttpClient from .planetary_computer_client import PlanetaryComputerClient @@ -33,7 +32,6 @@ __all__ = [ "DownloadWarning", - "AssetDownloadError", "AssetOverwriteError", "CannotIncludeAndExclude", "Client", @@ -47,5 +45,5 @@ "S3Client", "download_item", "download_item_collection", - "guess_client", + "guess_client_class", ] diff --git a/src/stac_asset/_cli.py b/src/stac_asset/_cli.py index 47668ea..9cca0ee 100644 --- a/src/stac_asset/_cli.py +++ b/src/stac_asset/_cli.py @@ -23,6 +23,12 @@ def cli() -> None: @cli.command() @click.argument("href", required=False) @click.argument("directory", required=False) +@click.option( + "-a", + "--alternate-assets", + help="Alternate asset hrefs to prefer, if available", + multiple=True, +) @click.option("-i", "--include", help="Asset keys to include", multiple=True) @click.option( "-x", @@ -57,6 +63,7 @@ def cli() -> None: def download( href: Optional[str], directory: Optional[str], + alternate_assets: List[str], include: List[str], exclude: List[str], file_name: Optional[str], @@ -83,14 +90,8 @@ def download( $ stac-asset download -i asset-key-to-include item.json """ - if href is None or href == "-": - input_dict = json.load(sys.stdin) - else: - input_dict = json.loads(asyncio.run(read_file(href))) - if directory is None: - directory = os.getcwd() - config = Config( + alternate_assets=alternate_assets, include=include, exclude=exclude, file_name=file_name, @@ -98,6 +99,13 @@ def download( warn=warn, ) + if href is None or href == "-": + input_dict = json.load(sys.stdin) + else: + input_dict = json.loads(asyncio.run(read_file(href, config))) + if directory is None: + directory = os.getcwd() + type_ = input_dict.get("type") if type_ is None: print("ERROR: missing 'type' field on input dictionary", file=sys.stderr) @@ -131,10 +139,10 @@ def download( json.dump(output.to_dict(transform_hrefs=False), sys.stdout) -async def read_file( - href: str, -) -> bytes: - async with await functions.guess_client(href) as client: +async def read_file(href: str, config: Config) -> bytes: + async with await functions.guess_client_class_from_href(href).from_config( + config + ) as client: data = b"" async for chunk in client.open_href(href): data += chunk diff --git a/src/stac_asset/client.py b/src/stac_asset/client.py index 57470ba..38200a6 100644 --- a/src/stac_asset/client.py +++ b/src/stac_asset/client.py @@ -1,27 +1,15 @@ from __future__ import annotations -import asyncio -import os.path -import warnings from abc import ABC, abstractmethod -from asyncio import Task from pathlib import Path from types import TracebackType -from typing import Any, AsyncIterator, Dict, List, Optional, Type, TypeVar +from typing import AsyncIterator, Optional, Type, TypeVar import aiofiles -import pystac.utils -from pystac import Asset, Item, ItemCollection +from pystac import Asset from yarl import URL from .config import Config -from .errors import ( - AssetDownloadError, - AssetOverwriteError, - DownloadError, - DownloadWarning, -) -from .strategy import FileNameStrategy from .types import PathLikeObject T = TypeVar("T", bound="Client") @@ -31,25 +19,31 @@ class Client(ABC): """An abstract base class for all clients.""" @classmethod - async def default(cls: Type[T]) -> T: - """Creates the default version of this client. + async def from_config(cls: Type[T], config: Config) -> T: + """Creates a client using the provided configuration. - ``__init__`` isn't enough because some clients need to do asynchronous - actions during setup. + Needed because some client setups require async operations. Returns: - T: The default version of this Client + T: A new client Client """ return cls() + def __init__(self) -> None: + pass + @abstractmethod - async def open_url(self, url: URL) -> AsyncIterator[bytes]: + async def open_url( + self, url: URL, content_type: Optional[str] = None + ) -> AsyncIterator[bytes]: """Opens a url and yields an iterator over its bytes. This is the core method that all clients must implement. Args: url: The input url + content_type: The expected content type, to be checked by the client + implementations Yields: AsyncIterator[bytes]: An iterator over chunks of the read file @@ -58,20 +52,27 @@ async def open_url(self, url: URL) -> AsyncIterator[bytes]: if False: # pragma: no cover yield - async def open_href(self, href: str) -> AsyncIterator[bytes]: + async def open_href( + self, href: str, content_type: Optional[str] = None + ) -> AsyncIterator[bytes]: """Opens a href and yields an iterator over its bytes. Args: href: The input href + content_type: The expected content type Yields: AsyncIterator[bytes]: An iterator over chunks of the read file """ - async for chunk in self.open_url(URL(href)): + async for chunk in self.open_url(URL(href), content_type=content_type): yield chunk async def download_href( - self, href: str, path: PathLikeObject, clean: bool = True + self, + href: str, + path: PathLikeObject, + clean: bool = True, + content_type: Optional[str] = None, ) -> None: """Downloads a file to the local filesystem. @@ -79,10 +80,11 @@ async def download_href( href: The input href path: The output file path clean: If an error occurs, delete the output file if it exists + content_type: The expected content type """ try: async with aiofiles.open(path, mode="wb") as f: - async for chunk in self.open_href(href): + async for chunk in self.open_href(href, content_type=content_type): await f.write(chunk) except Exception as err: path_as_path = Path(path) @@ -93,191 +95,35 @@ async def download_href( pass raise err - async def download_asset(self, key: str, asset: Asset, path: Path) -> None: + async def download_asset( + self, key: str, asset: Asset, path: Path, clean: bool = True + ) -> Asset: """Downloads an asset. Args: key: The asset key asset: The asset + clean: If an error occurs, delete the output file if it exists path: The path to which the asset will be downloaded + Returns: + Asset: The asset with an updated href + Raises: - AssetDownloadError: If any exception is raised during the - download, it is wrapped in an :py:class:`AssetDownloadError` + ValueError: Raised if the asset does not have an absolute href """ href = asset.get_absolute_href() if href is None: - raise AssetDownloadError( - key, - asset, - ValueError( - f"asset '{key}' does not have an absolute href: {asset.href}" - ), + raise ValueError( + f"asset '{key}' does not have an absolute href: {asset.href}" ) - try: - await self.download_href(href, path) - except Exception as e: - raise AssetDownloadError(key, asset, e) - - async def download_item( - self, - item: Item, - directory: PathLikeObject, - config: Optional[Config] = None, - ) -> Item: - """Downloads an item and all of its assets to the given directory. + await self.download_href(href, path, clean=clean, content_type=asset.media_type) + asset.href = str(path) + return asset - Args: - item: The item to download - directory: The root location of the downloaded files - config: Configuration for downloading the item - - Returns: - Item: The :py:class:`~pystac.Item`, with updated asset hrefs - """ - if config is None: - config = Config() - else: - config.validate() - - directory_as_path = Path(directory) - if not directory_as_path.exists(): - if config.make_directory: - directory_as_path.mkdir() - else: - raise FileNotFoundError(f"output directory does not exist: {directory}") - - if config.file_name: - item_path = directory_as_path / config.file_name - else: - self_href = item.get_self_href() - if self_href: - item_path = directory_as_path / os.path.basename(self_href) - else: - item_path = None - - tasks: List[Task[Any]] = list() - file_names: Dict[str, str] = dict() - item.make_asset_hrefs_absolute() - for key, asset in ( - (k, a) - for k, a in item.assets.items() - if (not config.include or k in config.include) - and (not config.exclude or k not in config.exclude) - ): - # TODO strategy should be auto-guessable - if config.asset_file_name_strategy == FileNameStrategy.FILE_NAME: - file_name = os.path.basename(URL(asset.href).path) - elif config.asset_file_name_strategy == FileNameStrategy.KEY: - file_name = key + Path(asset.href).suffix - path = directory_as_path / file_name - if file_name in file_names: - for task in tasks: - task.cancel() - raise AssetOverwriteError(list(file_names.values())) - else: - file_names[file_name] = str(path) - - tasks.append( - asyncio.create_task(self.download_asset(key, asset.clone(), path)) - ) - if item_path: - item.assets[key].href = pystac.utils.make_relative_href( - str(path), str(item_path) - ) - else: - item.assets[key].href = str(path.absolute()) - - results = await asyncio.gather(*tasks, return_exceptions=True) - exceptions = list() - for result in results: - if isinstance(result, Exception): - exceptions.append(result) - if exceptions: - if config.warn: - for exception in exceptions: - warnings.warn(str(exception), DownloadWarning) - if isinstance(exception, AssetDownloadError): - del item.assets[exception.key] - else: - raise DownloadError(exceptions) - - new_links = list() - for link in item.links: - link_href = link.get_href(transform_href=False) - if link_href and not pystac.utils.is_absolute_href(link_href): - link.target = pystac.utils.make_absolute_href(link.href, item.self_href) - new_links.append(link) - item.links = new_links - - if item_path: - item.set_self_href(str(item_path)) - item.save_object(include_self_link=True) - else: - item.set_self_href(None) - - return item - - async def download_item_collection( - self, - item_collection: ItemCollection, - directory: PathLikeObject, - config: Optional[Config] = None, - ) -> ItemCollection: - """Downloads an item collection and all of its assets to the given directory. - - Args: - item_collection: The item collection to download - directory: The root location of the downloaded files - config: Configuration for downloading the item - - Returns: - ItemCollection: The :py:class:`~pystac.ItemCollection`, with the - updated asset hrefs - - Raises: - CantIncludeAndExclude: Raised if both include and exclude are not None. - """ - if config is None: - config = Config() - # Config validation happens at the download_item level - - directory_as_path = Path(directory) - if not directory_as_path.exists(): - if config.make_directory: - directory_as_path.mkdir() - else: - raise FileNotFoundError(f"output directory does not exist: {directory}") - directory_as_path.mkdir(exist_ok=True) - tasks: List[Task[Any]] = list() - for item in item_collection.items: - # TODO what happens if items share ids? - item_directory = directory_as_path / item.id - item_config = config.copy() - item_config.make_directory = True - item_config.file_name = None - tasks.append( - asyncio.create_task( - self.download_item( - item=item, - directory=item_directory, - config=item_config, - ) - ) - ) - results = await asyncio.gather(*tasks, return_exceptions=True) - exceptions = list() - for result in results: - if isinstance(result, Exception): - exceptions.append(result) - if exceptions: - raise DownloadError(exceptions) - item_collection.items = results - if config.file_name: - item_collection.save_object( - dest_href=str(directory_as_path / config.file_name) - ) - return item_collection + async def close(self) -> None: + """Close this client.""" + pass async def __aenter__(self) -> Client: return self diff --git a/src/stac_asset/config.py b/src/stac_asset/config.py index 6f2f143..bc90d0d 100644 --- a/src/stac_asset/config.py +++ b/src/stac_asset/config.py @@ -7,11 +7,16 @@ from .errors import CannotIncludeAndExclude from .strategy import FileNameStrategy +DEFAULT_S3_REGION_NAME = "us-west-2" + @dataclass class Config: """Configuration for downloading items and their assets.""" + alternate_assets: List[str] = field(default_factory=list) + """Alternate asset keys to prefer, if available.""" + asset_file_name_strategy: FileNameStrategy = FileNameStrategy.FILE_NAME """The file name strategy to use when downloading assets.""" @@ -42,6 +47,12 @@ class Config: warn: bool = False """When downloading, warn instead of erroring.""" + earthdata_token: Optional[str] = None + """A token for logging in to Earthdata.""" + + s3_region_name: str = DEFAULT_S3_REGION_NAME + """Default s3 region.""" + s3_requester_pays: bool = False """If using the s3 client, enable requester pays.""" diff --git a/src/stac_asset/earthdata_client.py b/src/stac_asset/earthdata_client.py index 857e858..f31b580 100644 --- a/src/stac_asset/earthdata_client.py +++ b/src/stac_asset/earthdata_client.py @@ -6,6 +6,7 @@ from aiohttp import ClientSession +from .config import Config from .http_client import HttpClient @@ -13,12 +14,19 @@ class EarthdataClient(HttpClient): """Access data from https://www.earthdata.nasa.gov/.""" @classmethod - async def default(cls) -> EarthdataClient: + async def from_config(cls, config: Config) -> EarthdataClient: """Logs in to Earthdata and returns the default earthdata client. - Uses a token stored in the ``EARTHDATA_PAT`` environment variable. + Uses a token stored in the ``EARTHDATA_PAT`` environment variable, if + the token is not provided in the config. + + Args: + config: A configuration object. + + Returns: + EarthdataClient: A logged-in EarthData client. """ - return await cls.login() + return await cls.login(config.earthdata_token) @classmethod async def login(cls, token: Optional[str] = None) -> EarthdataClient: @@ -53,5 +61,5 @@ async def __aexit__( exc_val: Optional[BaseException], exc_tb: Optional[TracebackType], ) -> Optional[bool]: - await self.session.close() + await self.close() return await super().__aexit__(exc_type, exc_val, exc_tb) diff --git a/src/stac_asset/errors.py b/src/stac_asset/errors.py index 564440b..5e3ed73 100644 --- a/src/stac_asset/errors.py +++ b/src/stac_asset/errors.py @@ -1,7 +1,5 @@ from typing import Any, List -from pystac import Asset - class AssetOverwriteError(Exception): """Raised when an asset would be overwritten during download.""" @@ -12,19 +10,6 @@ def __init__(self, hrefs: List[str]) -> None: ) -class AssetDownloadError(Exception): - """Raised when an asset was unable to be downloaded.""" - - def __init__(self, key: str, asset: Asset, err: Exception) -> None: - self.key = key - self.asset = asset - self.err = err - super().__init__( - f"error when downloading asset '{key}' with href '{asset.href}': {err}" - ) - self.__cause__ = err - - class DownloadWarning(Warning): """A warning for when something couldn't be downloaded. @@ -47,14 +32,25 @@ def __init__( ) -class SchemeError(Exception): - """Raised if the scheme is inappropriate for the client.""" +class ContentTypeError(Exception): + """The expected content type does not match the actual content type.""" + + def __init__(self, actual: str, expected: str, *args: Any, **kwargs: Any) -> None: + super().__init__( + f"the actual content type does not match the expected: actual={actual}, " + f"expected={expected}", + *args, + **kwargs, + ) class DownloadError(Exception): """A collection of exceptions encountered while downloading.""" + exceptions: List[Exception] + def __init__(self, exceptions: List[Exception], *args: Any, **kwargs: Any) -> None: + self.exceptions = exceptions messages = list() for exception in exceptions: messages.append(str(exception)) diff --git a/src/stac_asset/filesystem_client.py b/src/stac_asset/filesystem_client.py index 8206ed6..d4f0d5a 100644 --- a/src/stac_asset/filesystem_client.py +++ b/src/stac_asset/filesystem_client.py @@ -15,11 +15,15 @@ class FilesystemClient(Client): Mostly used for testing, but could be useful in some real-world cases. """ - async def open_url(self, url: URL) -> AsyncIterator[bytes]: + async def open_url( + self, url: URL, content_type: Optional[str] = None + ) -> AsyncIterator[bytes]: """Iterates over data from a local url. Args: url: The url to read bytes from + content_type: The expected content type. Ignored by this client, + because filesystems don't have content types. Yields: AsyncIterator[bytes]: An iterator over the file's bytes. diff --git a/src/stac_asset/functions.py b/src/stac_asset/functions.py index a456814..6b2e58e 100644 --- a/src/stac_asset/functions.py +++ b/src/stac_asset/functions.py @@ -1,14 +1,25 @@ -from typing import Optional - -from pystac import Item, ItemCollection +import asyncio +import os.path +import warnings +from pathlib import Path +from typing import Dict, Optional, Set, Tuple, Type + +import pystac.utils +from pystac import Asset, Item, ItemCollection from yarl import URL from .client import Client from .config import Config +from .errors import ( + AssetOverwriteError, + DownloadError, + DownloadWarning, +) from .filesystem_client import FilesystemClient from .http_client import HttpClient from .planetary_computer_client import PlanetaryComputerClient from .s3_client import S3Client +from .strategy import FileNameStrategy from .types import PathLikeObject @@ -25,24 +36,102 @@ async def download_item( config: The download configuration Returns: - Item: The `~pystac.Item`, with the updated asset hrefs. + Item: The `~pystac.Item`, with the updated asset hrefs and self href. Raises: - CantIncludeAndExclude: Raised if both include and exclude are not None. + ValueError: Raised if the item doesn't have any assets. """ if not item.assets: - raise ValueError("cannot guess a client if an item does not have any assets") + raise ValueError(f"no assets to download for item with id '{item.id}") + else: + # Will fail if the item doesn't have a self href and there's relative + # asset hrefs + item.make_asset_hrefs_absolute() + if config is None: config = Config() - async with await guess_client( - next(iter(item.assets.values())).href, - s3_requester_pays=config.s3_requester_pays, - ) as client: - return await client.download_item( - item=item, - directory=directory, - config=config, + else: + config.validate() + + directory_as_path = Path(directory) + if not directory_as_path.exists(): + if config.make_directory: + directory_as_path.mkdir(parents=True) + else: + raise FileNotFoundError(f"output directory does not exist: {directory}") + + if config.file_name: + item_path = directory_as_path / config.file_name + else: + self_href = item.get_self_href() + if self_href: + item_path = directory_as_path / os.path.basename(self_href) + else: + item_path = None + item.set_self_href(str(item_path)) + + file_names: Set[str] = set() + assets: Dict[str, Tuple[Asset, Path]] = dict() + for key, asset in ( + (k, a) + for k, a in item.assets.items() + if (not config.include or k in config.include) + and (not config.exclude or k not in config.exclude) + ): + if config.asset_file_name_strategy == FileNameStrategy.FILE_NAME: + file_name = os.path.basename(URL(asset.href).path) + elif config.asset_file_name_strategy == FileNameStrategy.KEY: + file_name = key + Path(asset.href).suffix + + if file_name in file_names: + raise AssetOverwriteError(list(file_names)) + else: + file_names.add(file_name) + + path = directory_as_path / file_name + assets[key] = (asset, path) + + tasks = dict() + clients: Dict[Type[Client], Client] = dict() + for key, (asset, path) in assets.items(): + client_class = guess_client_class(asset, config) + if client_class in clients: + client = clients[client_class] + else: + client = await client_class.from_config(config) + clients[client_class] = client + tasks[key] = asyncio.create_task( + client.download_asset(key, asset.clone(), path) ) + if item.get_self_href(): + item.assets[key].href = pystac.utils.make_relative_href( + str(path), str(item_path) + ) + else: + item.assets[key].href = str(path.absolute()) + + # TODO support fast failing + exceptions = list() + for key, task in tasks.items(): + try: + _ = await task + except Exception as exception: + if config.warn: + warnings.warn(str(exception), DownloadWarning) + del item.assets[key] + else: + exceptions.append(exception) + + for client in clients.values(): + await client.close() + + if exceptions: + raise DownloadError(exceptions) + + if item.get_self_href(): + item.save_object(include_self_link=True) + + return item async def download_item_collection( @@ -65,44 +154,94 @@ async def download_item_collection( """ if config is None: config = Config() - if not item_collection.items: - return item_collection - elif not item_collection.items[0].assets: - raise ValueError( - "cannot guess a client if an item collection's first item does not have " - "any assets" - ) - async with await guess_client( - next(iter(item_collection.items[0].assets.values())).href, - s3_requester_pays=config.s3_requester_pays, - ) as client: - return await client.download_item_collection( - item_collection, - directory, - config=config, + + directory_as_path = Path(directory) + if not directory_as_path.exists(): + if config.make_directory: + directory_as_path.mkdir(parents=True) + else: + raise FileNotFoundError(f"output directory does not exist: {directory}") + + tasks = list() + for item in item_collection.items: + # TODO what happens if items share ids? + item_directory = directory_as_path / item.id + item_config = config.copy() + item_config.make_directory = True + item_config.file_name = None + + # TODO we should share clients among items + tasks.append( + asyncio.create_task( + download_item( + item=item, + directory=item_directory, + config=item_config, + ) + ) ) + results = await asyncio.gather(*tasks, return_exceptions=True) + exceptions = list() + for result in results: + if isinstance(result, Exception): + exceptions.append(result) + if exceptions: + raise DownloadError(exceptions) + item_collection.items = results + if config.file_name: + item_collection.save_object(dest_href=str(directory_as_path / config.file_name)) + return item_collection -async def guess_client(href: str, s3_requester_pays: bool = False) -> Client: - """Guess which client should be used to open the given href. + +def guess_client_class(asset: Asset, config: Config) -> Type[Client]: + """Guess which client should be used to download the given asset. + + If the configuration has ``alternate_assets``, these will be used instead of + the asset's href, if present. The asset's href will be updated with the href picked. Args: - href: The input href. - s3_requester_pays: If there's a URL host, use the s3 client and enable - requester pays + asset: The asset + config: A download configuration - Yields: - Client: The most appropriate client for the href, maybe. + Returns: + Client: The most appropriate client class for the href, maybe. + """ + alternate = asset.extra_fields.get("alternate") + if not isinstance(alternate, dict): + alternate = None + if alternate and config.alternate_assets: + for alternate_asset in config.alternate_assets: + if alternate_asset in alternate: + try: + href = alternate[alternate_asset]["href"] + asset.href = href + return guess_client_class_from_href(href) + except KeyError: + raise ValueError( + "invalid alternate asset definition (missing href): " + f"{alternate}" + ) + return guess_client_class_from_href(asset.href) + + +def guess_client_class_from_href(href: str) -> Type[Client]: + """Guess the client class from an href. + + Args: + href: An href + + Returns: + A client class type. """ url = URL(href) - # TODO enable matching on domain and protocol if not url.host: - return await FilesystemClient.default() - elif url.scheme == "s3" or s3_requester_pays: - return S3Client(requester_pays=s3_requester_pays) + return FilesystemClient + elif url.scheme == "s3": + return S3Client elif url.host.endswith("blob.core.windows.net"): - return await PlanetaryComputerClient.default() + return PlanetaryComputerClient elif url.scheme == "http" or url.scheme == "https": - return await HttpClient.default() + return HttpClient else: - return await FilesystemClient.default() + raise ValueError(f"could not guess client class for href: {href}") diff --git a/src/stac_asset/http_client.py b/src/stac_asset/http_client.py index 4dc9960..d0ad385 100644 --- a/src/stac_asset/http_client.py +++ b/src/stac_asset/http_client.py @@ -7,6 +7,8 @@ from yarl import URL from .client import Client +from .config import Config +from .errors import ContentTypeError T = TypeVar("T", bound="HttpClient") @@ -22,20 +24,25 @@ class HttpClient(Client): """A atiohttp session that will be used for all requests.""" @classmethod - async def default(cls: Type[T]) -> T: + async def from_config(cls: Type[T], config: Config) -> T: """Creates the default http client with a vanilla session object.""" + # TODO add basic auth session = ClientSession() return cls(session) - def __init__(self, session: ClientSession) -> None: + def __init__(self, session: ClientSession, check_content_type: bool = True) -> None: super().__init__() self.session = session + self.check_content_type = check_content_type - async def open_url(self, url: URL) -> AsyncIterator[bytes]: + async def open_url( + self, url: URL, content_type: Optional[str] = None + ) -> AsyncIterator[bytes]: """Opens a url with this client's session and iterates over its bytes. Args: url: The url to open + content_type: The expected content type Yields: AsyncIterator[bytes]: An iterator over the file's bytes @@ -45,9 +52,20 @@ async def open_url(self, url: URL) -> AsyncIterator[bytes]: """ async with self.session.get(url, allow_redirects=True) as response: response.raise_for_status() + if content_type and response.content_type != content_type: + raise ContentTypeError( + actual=response.content_type, expected=content_type + ) async for chunk, _ in response.content.iter_chunks(): yield chunk + async def close(self) -> None: + """Close this http client. + + Closes the underlying session. + """ + await self.session.close() + async def __aenter__(self) -> HttpClient: return self @@ -57,5 +75,5 @@ async def __aexit__( exc_val: Optional[BaseException], exc_tb: Optional[TracebackType], ) -> Optional[bool]: - await self.session.close() + await self.close() return await super().__aexit__(exc_type, exc_val, exc_tb) diff --git a/src/stac_asset/planetary_computer_client.py b/src/stac_asset/planetary_computer_client.py index ea5f1dc..44527d1 100644 --- a/src/stac_asset/planetary_computer_client.py +++ b/src/stac_asset/planetary_computer_client.py @@ -65,7 +65,9 @@ def __init__( self._cache_lock = Lock() self.sas_token_endpoint = URL(sas_token_endpoint) - async def open_url(self, url: URL) -> AsyncIterator[bytes]: + async def open_url( + self, url: URL, content_type: Optional[str] = None + ) -> AsyncIterator[bytes]: """Opens a url and iterates over its bytes. Includes functionality to sign the url with a SAS token fetched from @@ -81,6 +83,7 @@ async def open_url(self, url: URL) -> AsyncIterator[bytes]: Args: url: The url to open + content_type: The expected content type Yields: AsyncIterator[bytes]: An iterator over the file's bytes @@ -92,7 +95,7 @@ async def open_url(self, url: URL) -> AsyncIterator[bytes]: and not set(url.query) & {"st", "se", "sp"} ): url = await self._sign(url) - async for chunk in super().open_url(url): + async for chunk in super().open_url(url, content_type=content_type): yield chunk async def _sign(self, url: URL) -> URL: @@ -122,5 +125,5 @@ async def __aexit__( exc_val: Optional[BaseException], exc_tb: Optional[TracebackType], ) -> Optional[bool]: - await self.session.close() + await self.close() return await super().__aexit__(exc_type, exc_val, exc_tb) diff --git a/src/stac_asset/s3_client.py b/src/stac_asset/s3_client.py index 3139aec..fd5067a 100644 --- a/src/stac_asset/s3_client.py +++ b/src/stac_asset/s3_client.py @@ -1,20 +1,17 @@ from __future__ import annotations -from pathlib import Path from types import TracebackType from typing import AsyncIterator, Optional, Type import aiobotocore.session +import botocore.config from aiobotocore.session import AioSession from botocore import UNSIGNED -from botocore.config import Config -from pystac import Asset from yarl import URL from .client import Client -from .errors import AssetDownloadError, SchemeError - -DEFAULT_REGION_NAME = "us-west-2" +from .config import DEFAULT_S3_REGION_NAME, Config +from .errors import ContentTypeError class S3Client(Client): @@ -29,19 +26,38 @@ class S3Client(Client): requester_pays: bool """If True, add `--request-payer requester` to all requests.""" + @classmethod + async def from_config(cls, config: Config) -> S3Client: + """Creates an s3 client from a config. + + Args: + config: The config object + + Returns: + S3Client: A new s3 client + """ + return cls( + requester_pays=config.s3_requester_pays, region_name=config.s3_region_name + ) + def __init__( - self, region_name: str = DEFAULT_REGION_NAME, requester_pays: bool = False + self, + requester_pays: bool = False, + region_name: str = DEFAULT_S3_REGION_NAME, ) -> None: super().__init__() self.session = aiobotocore.session.get_session() self.region_name = region_name self.requester_pays = requester_pays - async def open_url(self, url: URL) -> AsyncIterator[bytes]: + async def open_url( + self, url: URL, content_type: Optional[str] = None + ) -> AsyncIterator[bytes]: """Opens an s3 url and iterates over its bytes. Args: url: The url to open + content_type: The expected content type Yields: AsyncIterator[bytes]: An iterator over the file's bytes @@ -49,12 +65,10 @@ async def open_url(self, url: URL) -> AsyncIterator[bytes]: Raises: SchemeError: Raised if the url's scheme is not ``s3`` """ - if url.scheme != "s3": - raise SchemeError(f"only s3 urls are allowed: {url}") if self.requester_pays: - config = Config() + config = botocore.config.Config() else: - config = Config(signature_version=UNSIGNED) + config = botocore.config.Config(signature_version=UNSIGNED) async with self.session.create_client( "s3", region_name=self.region_name, @@ -69,29 +83,14 @@ async def open_url(self, url: URL) -> AsyncIterator[bytes]: if self.requester_pays: params["RequestPayer"] = "requester" response = await client.get_object(**params) + print(response) + if content_type and response["ContentType"] != content_type: + raise ContentTypeError( + actual=response["ContentType"], expected=content_type + ) async for chunk in response["Body"]: yield chunk - async def download_asset(self, key: str, asset: Asset, path: Path) -> None: - """Downloads an asset to a location. - - If the initial download fails with a scheme error, the client looks for - an alternate href and tries again with that. - - Args: - key: The asset key - asset: The asset - path: The destination path - """ - try: - return await super().download_asset(key, asset, path) - except AssetDownloadError as err: - if isinstance(err.err, SchemeError): - maybe_asset = self._asset_with_alternate_href(asset) - if maybe_asset: - return await super().download_asset(key, maybe_asset, path) - raise err - async def has_credentials(self) -> bool: """Returns true if the sessions has credentials.""" return await self.session.get_credentials() is not None @@ -106,23 +105,3 @@ async def __aexit__( exc_tb: Optional[TracebackType], ) -> Optional[bool]: return None - - def _asset_with_alternate_href(self, asset: Asset) -> Optional[Asset]: - # TODO some of this logic could be refactored out to be more common, but - # not all (e.g. requester pays) - alternate = asset.extra_fields.get("alternate") - if alternate and isinstance(alternate, dict): - s3 = alternate.get("s3") - if s3 and isinstance(s3, dict): - requester_pays = s3.get("storage:requester_pays", False) - href = s3.get("href") - platform = s3.get("storage:platform", "AWS") - if platform == "AWS" and href: - if requester_pays and not self.requester_pays: - raise ValueError( - f"alternate href {href} requires requester pays, but " - "requester pays is not enabled on this client" - ) - asset.href = href - return asset - return None diff --git a/tests/data/LC09_L2SP_092068_20230607_20230609_02_T1_SR.json b/tests/data/LC09_L2SP_092068_20230607_20230609_02_T1_SR.json index dc20335..42c240b 100644 --- a/tests/data/LC09_L2SP_092068_20230607_20230609_02_T1_SR.json +++ b/tests/data/LC09_L2SP_092068_20230607_20230609_02_T1_SR.json @@ -1 +1,1080 @@ -{"type":"Feature","stac_version":"1.0.0","stac_extensions":["https://landsat.usgs.gov/stac/landsat-extension/v1.1.1/schema.json","https://stac-extensions.github.io/view/v1.0.0/schema.json","https://stac-extensions.github.io/projection/v1.0.0/schema.json","https://stac-extensions.github.io/eo/v1.0.0/schema.json","https://stac-extensions.github.io/alternate-assets/v1.1.0/schema.json","https://stac-extensions.github.io/storage/v1.0.0/schema.json","https://stac-extensions.github.io/accuracy/v1.0.0/schema.json","https://stac-extensions.github.io/card4l/v0.1.0/optical/schema.json","https://stac-extensions.github.io/classification/v1.0.0/schema.json"],"id":"LC09_L2SP_092068_20230607_20230609_02_T1_SR","description":"Landsat Collection 2 Level-2 Surface Reflectance Product","bbox":[151.28163779194495,-12.614223157106734,153.3373165666754,-10.52667135213206],"geometry":{"type":"Polygon","coordinates":[[[151.66505188358275,-10.52667135213206],[151.28163779194495,-12.258332394290615],[152.9645969355396,-12.614223157106734],[153.3373165666754,-10.878222755599452],[151.66505188358275,-10.52667135213206]]]},"properties":{"datetime":"2023-06-07T23:55:41.862538Z","eo:cloud_cover":24.51,"view:sun_azimuth":39.05510881,"view:sun_elevation":45.72253417,"platform":"LANDSAT_9","instruments":["OLI","TIRS"],"view:off_nadir":0,"landsat:cloud_cover_land":41.12,"landsat:wrs_type":"2","landsat:wrs_path":"092","landsat:wrs_row":"068","landsat:scene_id":"LC90920682023158LGN00","landsat:collection_category":"T1","landsat:collection_number":"02","landsat:correction":"L2SP","accuracy:geometric_x_bias":0,"accuracy:geometric_y_bias":0,"accuracy:geometric_x_stddev":5.586,"accuracy:geometric_y_stddev":5.069,"accuracy:geometric_rmse":7.543,"proj:epsg":32656,"proj:shape":[7721,7601],"proj:transform":[30,0,310785,0,-30,-1163385],"card4l:specification":"SR","card4l:specification_version":"5.0","created":"2023-06-09T05:51:37.401Z","updated":"2023-06-09T05:51:37.401Z"},"assets":{"thumbnail":{"title":"Thumbnail image","type":"image/jpeg","roles":["thumbnail"],"href":"https://landsatlook.usgs.gov/data/collection02/level-2/standard/oli-tirs/2023/092/068/LC09_L2SP_092068_20230607_20230609_02_T1/LC09_L2SP_092068_20230607_20230609_02_T1_thumb_small.jpeg","alternate":{"s3":{"storage:platform":"AWS","storage:requester_pays":true,"href":"s3://usgs-landsat/collection02/level-2/standard/oli-tirs/2023/092/068/LC09_L2SP_092068_20230607_20230609_02_T1/LC09_L2SP_092068_20230607_20230609_02_T1_thumb_small.jpeg"}}},"reduced_resolution_browse":{"title":"Reduced resolution browse image","type":"image/jpeg","roles":["overview"],"href":"https://landsatlook.usgs.gov/data/collection02/level-2/standard/oli-tirs/2023/092/068/LC09_L2SP_092068_20230607_20230609_02_T1/LC09_L2SP_092068_20230607_20230609_02_T1_thumb_large.jpeg","alternate":{"s3":{"storage:platform":"AWS","storage:requester_pays":true,"href":"s3://usgs-landsat/collection02/level-2/standard/oli-tirs/2023/092/068/LC09_L2SP_092068_20230607_20230609_02_T1/LC09_L2SP_092068_20230607_20230609_02_T1_thumb_large.jpeg"}}},"index":{"title":"HTML index page","type":"text/html","roles":["metadata"],"href":"https://landsatlook.usgs.gov/stac-browser/collection02/level-2/standard/oli-tirs/2023/092/068/LC09_L2SP_092068_20230607_20230609_02_T1/LC09_L2SP_092068_20230607_20230609_02_T1"},"MTL.json":{"title":"Product Metadata File (json)","description":"Collection 2 Level-2 Product Metadata File (json)","type":"application/json","roles":["metadata"],"href":"https://landsatlook.usgs.gov/data/collection02/level-2/standard/oli-tirs/2023/092/068/LC09_L2SP_092068_20230607_20230609_02_T1/LC09_L2SP_092068_20230607_20230609_02_T1_MTL.json","alternate":{"s3":{"storage:platform":"AWS","storage:requester_pays":true,"href":"s3://usgs-landsat/collection02/level-2/standard/oli-tirs/2023/092/068/LC09_L2SP_092068_20230607_20230609_02_T1/LC09_L2SP_092068_20230607_20230609_02_T1_MTL.json"}}},"coastal":{"title":"Coastal/Aerosol Band (B1)","description":"Collection 2 Level-2 Coastal/Aerosol Band (B1) Surface Reflectance","type":"image/vnd.stac.geotiff; cloud-optimized=true","roles":["data"],"eo:bands":[{"name":"B1","common_name":"coastal","gsd":30,"center_wavelength":0.44}],"href":"https://landsatlook.usgs.gov/data/collection02/level-2/standard/oli-tirs/2023/092/068/LC09_L2SP_092068_20230607_20230609_02_T1/LC09_L2SP_092068_20230607_20230609_02_T1_SR_B1.TIF","alternate":{"s3":{"storage:platform":"AWS","storage:requester_pays":true,"href":"s3://usgs-landsat/collection02/level-2/standard/oli-tirs/2023/092/068/LC09_L2SP_092068_20230607_20230609_02_T1/LC09_L2SP_092068_20230607_20230609_02_T1_SR_B1.TIF"}}},"blue":{"title":"Blue Band (B2)","description":"Collection 2 Level-2 Blue Band (B2) Surface Reflectance","type":"image/vnd.stac.geotiff; cloud-optimized=true","roles":["data"],"eo:bands":[{"name":"B2","common_name":"blue","gsd":30,"center_wavelength":0.48}],"href":"https://landsatlook.usgs.gov/data/collection02/level-2/standard/oli-tirs/2023/092/068/LC09_L2SP_092068_20230607_20230609_02_T1/LC09_L2SP_092068_20230607_20230609_02_T1_SR_B2.TIF","alternate":{"s3":{"storage:platform":"AWS","storage:requester_pays":true,"href":"s3://usgs-landsat/collection02/level-2/standard/oli-tirs/2023/092/068/LC09_L2SP_092068_20230607_20230609_02_T1/LC09_L2SP_092068_20230607_20230609_02_T1_SR_B2.TIF"}}},"green":{"title":"Green Band (B3)","description":"Collection 2 Level-2 Green Band (B3) Surface Reflectance","type":"image/vnd.stac.geotiff; cloud-optimized=true","roles":["data"],"eo:bands":[{"name":"B3","common_name":"green","gsd":30,"center_wavelength":0.56}],"href":"https://landsatlook.usgs.gov/data/collection02/level-2/standard/oli-tirs/2023/092/068/LC09_L2SP_092068_20230607_20230609_02_T1/LC09_L2SP_092068_20230607_20230609_02_T1_SR_B3.TIF","alternate":{"s3":{"storage:platform":"AWS","storage:requester_pays":true,"href":"s3://usgs-landsat/collection02/level-2/standard/oli-tirs/2023/092/068/LC09_L2SP_092068_20230607_20230609_02_T1/LC09_L2SP_092068_20230607_20230609_02_T1_SR_B3.TIF"}}},"red":{"title":"Red Band (B4)","description":"Collection 2 Level-2 Red Band (B4) Surface Reflectance","type":"image/vnd.stac.geotiff; cloud-optimized=true","roles":["data"],"eo:bands":[{"name":"B4","common_name":"red","gsd":30,"center_wavelength":0.65}],"href":"https://landsatlook.usgs.gov/data/collection02/level-2/standard/oli-tirs/2023/092/068/LC09_L2SP_092068_20230607_20230609_02_T1/LC09_L2SP_092068_20230607_20230609_02_T1_SR_B4.TIF","alternate":{"s3":{"storage:platform":"AWS","storage:requester_pays":true,"href":"s3://usgs-landsat/collection02/level-2/standard/oli-tirs/2023/092/068/LC09_L2SP_092068_20230607_20230609_02_T1/LC09_L2SP_092068_20230607_20230609_02_T1_SR_B4.TIF"}}},"nir08":{"title":"Near Infrared Band 0.8 (B5)","description":"Collection 2 Level-2 Near Infrared Band 0.8 (B5) Surface Reflectance","type":"image/vnd.stac.geotiff; cloud-optimized=true","roles":["data","reflectance"],"eo:bands":[{"name":"B5","common_name":"nir08","gsd":30,"center_wavelength":0.86}],"href":"https://landsatlook.usgs.gov/data/collection02/level-2/standard/oli-tirs/2023/092/068/LC09_L2SP_092068_20230607_20230609_02_T1/LC09_L2SP_092068_20230607_20230609_02_T1_SR_B5.TIF","alternate":{"s3":{"storage:platform":"AWS","storage:requester_pays":true,"href":"s3://usgs-landsat/collection02/level-2/standard/oli-tirs/2023/092/068/LC09_L2SP_092068_20230607_20230609_02_T1/LC09_L2SP_092068_20230607_20230609_02_T1_SR_B5.TIF"}}},"swir16":{"title":"Short-wave Infrared Band 1.6 (B6)","description":"Collection 2 Level-2 Short-wave Infrared Band 1.6 (B6) Surface Reflectance","type":"image/vnd.stac.geotiff; cloud-optimized=true","roles":["data","reflectance"],"eo:bands":[{"name":"B6","common_name":"swir16","gsd":30,"center_wavelength":1.6}],"href":"https://landsatlook.usgs.gov/data/collection02/level-2/standard/oli-tirs/2023/092/068/LC09_L2SP_092068_20230607_20230609_02_T1/LC09_L2SP_092068_20230607_20230609_02_T1_SR_B6.TIF","alternate":{"s3":{"storage:platform":"AWS","storage:requester_pays":true,"href":"s3://usgs-landsat/collection02/level-2/standard/oli-tirs/2023/092/068/LC09_L2SP_092068_20230607_20230609_02_T1/LC09_L2SP_092068_20230607_20230609_02_T1_SR_B6.TIF"}}},"swir22":{"title":"Short-wave Infrared Band 2.2 (B7)","description":"Collection 2 Level-2 Short-wave Infrared Band 2.2 (B7) Surface Reflectance","type":"image/vnd.stac.geotiff; cloud-optimized=true","roles":["data","reflectance"],"eo:bands":[{"name":"B7","common_name":"swir22","gsd":30,"center_wavelength":2.2}],"href":"https://landsatlook.usgs.gov/data/collection02/level-2/standard/oli-tirs/2023/092/068/LC09_L2SP_092068_20230607_20230609_02_T1/LC09_L2SP_092068_20230607_20230609_02_T1_SR_B7.TIF","alternate":{"s3":{"storage:platform":"AWS","storage:requester_pays":true,"href":"s3://usgs-landsat/collection02/level-2/standard/oli-tirs/2023/092/068/LC09_L2SP_092068_20230607_20230609_02_T1/LC09_L2SP_092068_20230607_20230609_02_T1_SR_B7.TIF"}}},"qa_aerosol":{"title":"Aerosol Quality Analysis Band","description":"Collection 2 Level-2 Aerosol Quality Analysis Band Surface Reflectance","type":"image/vnd.stac.geotiff; cloud-optimized=true","roles":["metadata","data-mask","water-mask"],"classification:bitfields":[{"name":"fill","description":"Corresponding pixels in L1 image bands are fill","offset":0,"length":1,"classes":[{"name":"not_fill","description":"L1 image band pixels are not fill","value":0},{"name":"fill","description":"L1 image band pixels are fill","value":1}]},{"name":"retrieval","description":"Valid aerosol retrieval","offset":1,"length":1,"classes":[{"name":"not_valid","description":"Aerosol retrieval is not valid","value":0},{"name":"valid","description":"Aerosol retrieval is valid","value":1}]},{"name":"water","description":"Water mask","offset":2,"length":1,"classes":[{"name":"not_water","description":"Not water","value":0},{"name":"water","description":"Water","value":1}]},{"name":"unused","description":"Unused bit","offset":3,"length":1,"classes":[{"name":"unused","description":"Unused bit","value":0}]},{"name":"unused","description":"Unused bit","offset":4,"length":1,"classes":[{"name":"unused","description":"Unused bit","value":0}]},{"name":"interpolated","description":"Aerosol is interpolated","offset":5,"length":1,"classes":[{"name":"not_interpolated","description":"Aerosol is not interpolated","value":0},{"name":"interpolated","description":"Aerosol is interpolated","value":1}]},{"name":"level","description":"Aerosol level","offset":6,"length":2,"classes":[{"name":"climatology","description":"No aerosol correction applied","value":0},{"name":"low","description":"Low aerosol level","value":1},{"name":"medium","description":"Medium aerosol level","value":2},{"name":"high","description":"High aerosol level","value":3}]}],"href":"https://landsatlook.usgs.gov/data/collection02/level-2/standard/oli-tirs/2023/092/068/LC09_L2SP_092068_20230607_20230609_02_T1/LC09_L2SP_092068_20230607_20230609_02_T1_SR_QA_AEROSOL.TIF","alternate":{"s3":{"storage:platform":"AWS","storage:requester_pays":true,"href":"s3://usgs-landsat/collection02/level-2/standard/oli-tirs/2023/092/068/LC09_L2SP_092068_20230607_20230609_02_T1/LC09_L2SP_092068_20230607_20230609_02_T1_SR_QA_AEROSOL.TIF"}}},"qa_pixel":{"title":"Pixel Quality Assessment Band","description":"Collection 2 Level-2 Pixel Quality Assessment Band Surface Reflectance","type":"image/vnd.stac.geotiff; cloud-optimized=true","roles":["cloud","cloud-shadow","snow-ice","water-mask"],"classification:bitfields":[{"name":"fill","description":"Corresponding pixels in L1 image bands are fill","offset":0,"length":1,"classes":[{"name":"not_fill","description":"L1 image band pixels are not fill","value":0},{"name":"fill","description":"L1 image band pixels are fill","value":1}]},{"name":"dilated","description":"Dilated cloud","offset":1,"length":1,"classes":[{"name":"not_dilated","description":"Cloud is not dilated or no cloud","value":0},{"name":"dilated","description":"Cloud dilation","value":1}]},{"name":"cirrus","description":"Cirrus mask","offset":2,"length":1,"classes":[{"name":"not_cirrus","description":"No confidence level set or low confidence cirrus","value":0},{"name":"cirrus","description":"High confidence cirrus","value":1}]},{"name":"cloud","description":"Cloud mask","offset":3,"length":1,"classes":[{"name":"not_cloud","description":"Cloud confidence is not high","value":0},{"name":"cloud","description":"High confidence cloud","value":1}]},{"name":"shadow","description":"Cloud shadow mask","offset":4,"length":1,"classes":[{"name":"not_shadow","description":"Cloud shadow confidence is not high","value":0},{"name":"shadow","description":"High confidence cloud shadow","value":1}]},{"name":"snow","description":"Snow/Ice mask","offset":5,"length":1,"classes":[{"name":"not_snow","description":"Snow/Ice confidence is not high","value":0},{"name":"snow","description":"High confidence snow cover","value":1}]},{"name":"clear","description":"Cloud or dilated cloud bits set","offset":6,"length":1,"classes":[{"name":"not_clear","description":"Cloud or dilated cloud bits are set","value":0},{"name":"clear","description":"Cloud and dilated cloud bits are not set","value":1}]},{"name":"water","description":"Water mask","offset":7,"length":1,"classes":[{"name":"not_water","description":"Land or cloud","value":0},{"name":"water","description":"Water","value":1}]},{"name":"cloud_confidence","description":"Cloud confidence levels","offset":8,"length":2,"classes":[{"name":"not_set","description":"No confidence level set","value":0},{"name":"low","description":"Low confidence cloud","value":1},{"name":"medium","description":"Medium confidence cloud","value":2},{"name":"high","description":"High confidence cloud","value":3}]},{"name":"shadow_confidence","description":"Cloud shadow confidence levels","offset":10,"length":2,"classes":[{"name":"not_set","description":"No confidence level set","value":0},{"name":"low","description":"Low confidence cloud shadow","value":1},{"name":"reserved","description":"Reserved - value not used","value":2},{"name":"high","description":"High confidence cloud shadow","value":3}]},{"name":"snow_confidence","description":"Snow/Ice confidence levels","offset":12,"length":2,"classes":[{"name":"not_set","description":"No confidence level set","value":0},{"name":"low","description":"Low confidence snow/ice","value":1},{"name":"reserved","description":"Reserved - value not used","value":2},{"name":"high","description":"High confidence snow/ice","value":3}]},{"name":"cirrus_confidence","description":"Cirrus confidence levels","offset":14,"length":2,"classes":[{"name":"not_set","description":"No confidence level set","value":0},{"name":"low","description":"Low confidence cirrus","value":1},{"name":"reserved","description":"Reserved - value not used","value":2},{"name":"high","description":"High confidence cirrus","value":3}]}],"href":"https://landsatlook.usgs.gov/data/collection02/level-2/standard/oli-tirs/2023/092/068/LC09_L2SP_092068_20230607_20230609_02_T1/LC09_L2SP_092068_20230607_20230609_02_T1_QA_PIXEL.TIF","alternate":{"s3":{"storage:platform":"AWS","storage:requester_pays":true,"href":"s3://usgs-landsat/collection02/level-2/standard/oli-tirs/2023/092/068/LC09_L2SP_092068_20230607_20230609_02_T1/LC09_L2SP_092068_20230607_20230609_02_T1_QA_PIXEL.TIF"}}},"qa_radsat":{"title":"Radiometric Saturation Quality Assessment Band","description":"Collection 2 Level-2 Radiometric Saturation Quality Assessment Band Surface Reflectance","type":"image/vnd.stac.geotiff; cloud-optimized=true","roles":["saturation"],"classification:bitfields":[{"name":"band1","description":"Band 1 radiometric saturation","offset":0,"length":1,"classes":[{"name":"not_saturated","description":"Band 1 is not saturated","value":0},{"name":"saturated","description":"Band 1 is saturated","value":1}]},{"name":"band2","description":"Band 2 radiometric saturation","offset":1,"length":1,"classes":[{"name":"not_saturated","description":"Band 2 is not saturated","value":0},{"name":"saturated","description":"Band 2 is saturated","value":1}]},{"name":"band3","description":"Band 3 radiometric saturation","offset":2,"length":1,"classes":[{"name":"not_saturated","description":"Band 3 is not saturated","value":0},{"name":"saturated","description":"Band 3 is saturated","value":1}]},{"name":"band4","description":"Band 4 radiometric saturation","offset":3,"length":1,"classes":[{"name":"not_saturated","description":"Band 4 is not saturated","value":0},{"name":"saturated","description":"Band 4 is saturated","value":1}]},{"name":"band5","description":"Band 5 radiometric saturation","offset":4,"length":1,"classes":[{"name":"not_saturated","description":"Band 5 is not saturated","value":0},{"name":"saturated","description":"Band 5 is saturated","value":1}]},{"name":"band6","description":"Band 6 radiometric saturation","offset":5,"length":1,"classes":[{"name":"not_saturated","description":"Band 6 is not saturated","value":0},{"name":"saturated","description":"Band 6 is saturated","value":1}]},{"name":"band7","description":"Band 7 radiometric saturation","offset":6,"length":1,"classes":[{"name":"not_saturated","description":"Band 7 is not saturated","value":0},{"name":"saturated","description":"Band 7 is saturated","value":1}]},{"name":"unused","description":"Unused bit","offset":7,"length":1,"classes":[{"name":"unused","description":"Unused bit","value":0}]},{"name":"band9","description":"Band 9 radiometric saturation","offset":8,"length":1,"classes":[{"name":"not_saturated","description":"Band 9 is not saturated","value":0},{"name":"saturated","description":"Band 9 is saturated","value":1}]},{"name":"unused","description":"Unused bit","offset":9,"length":1,"classes":[{"name":"unused","description":"Unused bit","value":0}]},{"name":"unused","description":"Unused bit","offset":10,"length":1,"classes":[{"name":"unused","description":"Unused bit","value":0}]},{"name":"occlusion","description":"Terrain not visible from sensor due to intervening terrain","offset":11,"length":1,"classes":[{"name":"not_occluded","description":"Terrain is not occluded","value":0},{"name":"occluded","description":"Terrain is occluded","value":1}]},{"name":"unused","description":"Unused bit","offset":12,"length":1,"classes":[{"name":"unused","description":"Unused bit","value":0}]},{"name":"unused","description":"Unused bit","offset":13,"length":1,"classes":[{"name":"unused","description":"Unused bit","value":0}]},{"name":"unused","description":"Unused bit","offset":14,"length":1,"classes":[{"name":"unused","description":"Unused bit","value":0}]},{"name":"unused","description":"Unused bit","offset":15,"length":1,"classes":[{"name":"unused","description":"Unused bit","value":0}]}],"href":"https://landsatlook.usgs.gov/data/collection02/level-2/standard/oli-tirs/2023/092/068/LC09_L2SP_092068_20230607_20230609_02_T1/LC09_L2SP_092068_20230607_20230609_02_T1_QA_RADSAT.TIF","alternate":{"s3":{"storage:platform":"AWS","storage:requester_pays":true,"href":"s3://usgs-landsat/collection02/level-2/standard/oli-tirs/2023/092/068/LC09_L2SP_092068_20230607_20230609_02_T1/LC09_L2SP_092068_20230607_20230609_02_T1_QA_RADSAT.TIF"}}},"ANG.txt":{"title":"Angle Coefficients File","description":"Collection 2 Level-2 Angle Coefficients File (ANG)","type":"text/plain","roles":["metadata"],"href":"https://landsatlook.usgs.gov/data/collection02/level-2/standard/oli-tirs/2023/092/068/LC09_L2SP_092068_20230607_20230609_02_T1/LC09_L2SP_092068_20230607_20230609_02_T1_ANG.txt","alternate":{"s3":{"storage:platform":"AWS","storage:requester_pays":true,"href":"s3://usgs-landsat/collection02/level-2/standard/oli-tirs/2023/092/068/LC09_L2SP_092068_20230607_20230609_02_T1/LC09_L2SP_092068_20230607_20230609_02_T1_ANG.txt"}}},"MTL.txt":{"title":"Product Metadata File","description":"Collection 2 Level-2 Product Metadata File (MTL)","type":"text/plain","roles":["metadata"],"href":"https://landsatlook.usgs.gov/data/collection02/level-2/standard/oli-tirs/2023/092/068/LC09_L2SP_092068_20230607_20230609_02_T1/LC09_L2SP_092068_20230607_20230609_02_T1_MTL.txt","alternate":{"s3":{"storage:platform":"AWS","storage:requester_pays":true,"href":"s3://usgs-landsat/collection02/level-2/standard/oli-tirs/2023/092/068/LC09_L2SP_092068_20230607_20230609_02_T1/LC09_L2SP_092068_20230607_20230609_02_T1_MTL.txt"}}},"MTL.xml":{"title":"Product Metadata File (xml)","description":"Collection 2 Level-2 Product Metadata File (xml)","type":"application/xml","roles":["metadata"],"href":"https://landsatlook.usgs.gov/data/collection02/level-2/standard/oli-tirs/2023/092/068/LC09_L2SP_092068_20230607_20230609_02_T1/LC09_L2SP_092068_20230607_20230609_02_T1_MTL.xml","alternate":{"s3":{"storage:platform":"AWS","storage:requester_pays":true,"href":"s3://usgs-landsat/collection02/level-2/standard/oli-tirs/2023/092/068/LC09_L2SP_092068_20230607_20230609_02_T1/LC09_L2SP_092068_20230607_20230609_02_T1_MTL.xml"}}}},"links":[{"rel":"self","href":"https://landsatlook.usgs.gov/stac-server/collections/landsat-c2l2-sr/items/LC09_L2SP_092068_20230607_20230609_02_T1_SR"},{"rel":"parent","href":"https://landsatlook.usgs.gov/stac-server/collections/landsat-c2l2-sr"},{"rel":"collection","href":"https://landsatlook.usgs.gov/stac-server/collections/landsat-c2l2-sr"},{"rel":"root","href":"https://landsatlook.usgs.gov/stac-server/"}],"collection":"landsat-c2l2-sr"} \ No newline at end of file +{ + "type": "Feature", + "stac_version": "1.0.0", + "stac_extensions": [ + "https://landsat.usgs.gov/stac/landsat-extension/v1.1.1/schema.json", + "https://stac-extensions.github.io/view/v1.0.0/schema.json", + "https://stac-extensions.github.io/projection/v1.0.0/schema.json", + "https://stac-extensions.github.io/eo/v1.0.0/schema.json", + "https://stac-extensions.github.io/alternate-assets/v1.1.0/schema.json", + "https://stac-extensions.github.io/storage/v1.0.0/schema.json", + "https://stac-extensions.github.io/accuracy/v1.0.0/schema.json", + "https://stac-extensions.github.io/card4l/v0.1.0/optical/schema.json", + "https://stac-extensions.github.io/classification/v1.0.0/schema.json" + ], + "id": "LC09_L2SP_092068_20230607_20230609_02_T1_SR", + "description": "Landsat Collection 2 Level-2 Surface Reflectance Product", + "bbox": [ + 151.28163779194495, + -12.614223157106734, + 153.3373165666754, + -10.52667135213206 + ], + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 151.66505188358275, + -10.52667135213206 + ], + [ + 151.28163779194495, + -12.258332394290615 + ], + [ + 152.9645969355396, + -12.614223157106734 + ], + [ + 153.3373165666754, + -10.878222755599452 + ], + [ + 151.66505188358275, + -10.52667135213206 + ] + ] + ] + }, + "properties": { + "datetime": "2023-06-07T23:55:41.862538Z", + "eo:cloud_cover": 24.51, + "view:sun_azimuth": 39.05510881, + "view:sun_elevation": 45.72253417, + "platform": "LANDSAT_9", + "instruments": [ + "OLI", + "TIRS" + ], + "view:off_nadir": 0, + "landsat:cloud_cover_land": 41.12, + "landsat:wrs_type": "2", + "landsat:wrs_path": "092", + "landsat:wrs_row": "068", + "landsat:scene_id": "LC90920682023158LGN00", + "landsat:collection_category": "T1", + "landsat:collection_number": "02", + "landsat:correction": "L2SP", + "accuracy:geometric_x_bias": 0, + "accuracy:geometric_y_bias": 0, + "accuracy:geometric_x_stddev": 5.586, + "accuracy:geometric_y_stddev": 5.069, + "accuracy:geometric_rmse": 7.543, + "proj:epsg": 32656, + "proj:shape": [ + 7721, + 7601 + ], + "proj:transform": [ + 30, + 0, + 310785, + 0, + -30, + -1163385 + ], + "card4l:specification": "SR", + "card4l:specification_version": "5.0", + "created": "2023-06-09T05:51:37.401Z", + "updated": "2023-06-09T05:51:37.401Z" + }, + "assets": { + "thumbnail": { + "title": "Thumbnail image", + "type": "image/jpeg", + "roles": [ + "thumbnail" + ], + "href": "https://landsatlook.usgs.gov/data/collection02/level-2/standard/oli-tirs/2023/092/068/LC09_L2SP_092068_20230607_20230609_02_T1/LC09_L2SP_092068_20230607_20230609_02_T1_thumb_small.jpeg", + "alternate": { + "s3": { + "storage:platform": "AWS", + "storage:requester_pays": true, + "href": "s3://usgs-landsat/collection02/level-2/standard/oli-tirs/2023/092/068/LC09_L2SP_092068_20230607_20230609_02_T1/LC09_L2SP_092068_20230607_20230609_02_T1_thumb_small.jpeg" + } + } + }, + "reduced_resolution_browse": { + "title": "Reduced resolution browse image", + "type": "image/jpeg", + "roles": [ + "overview" + ], + "href": "https://landsatlook.usgs.gov/data/collection02/level-2/standard/oli-tirs/2023/092/068/LC09_L2SP_092068_20230607_20230609_02_T1/LC09_L2SP_092068_20230607_20230609_02_T1_thumb_large.jpeg", + "alternate": { + "s3": { + "storage:platform": "AWS", + "storage:requester_pays": true, + "href": "s3://usgs-landsat/collection02/level-2/standard/oli-tirs/2023/092/068/LC09_L2SP_092068_20230607_20230609_02_T1/LC09_L2SP_092068_20230607_20230609_02_T1_thumb_large.jpeg" + } + } + }, + "index": { + "title": "HTML index page", + "type": "text/html", + "roles": [ + "metadata" + ], + "href": "https://landsatlook.usgs.gov/stac-browser/collection02/level-2/standard/oli-tirs/2023/092/068/LC09_L2SP_092068_20230607_20230609_02_T1/LC09_L2SP_092068_20230607_20230609_02_T1" + }, + "MTL.json": { + "title": "Product Metadata File (json)", + "description": "Collection 2 Level-2 Product Metadata File (json)", + "type": "application/json", + "roles": [ + "metadata" + ], + "href": "https://landsatlook.usgs.gov/data/collection02/level-2/standard/oli-tirs/2023/092/068/LC09_L2SP_092068_20230607_20230609_02_T1/LC09_L2SP_092068_20230607_20230609_02_T1_MTL.json", + "alternate": { + "s3": { + "storage:platform": "AWS", + "storage:requester_pays": true, + "href": "s3://usgs-landsat/collection02/level-2/standard/oli-tirs/2023/092/068/LC09_L2SP_092068_20230607_20230609_02_T1/LC09_L2SP_092068_20230607_20230609_02_T1_MTL.json" + } + } + }, + "coastal": { + "title": "Coastal/Aerosol Band (B1)", + "description": "Collection 2 Level-2 Coastal/Aerosol Band (B1) Surface Reflectance", + "type": "image/vnd.stac.geotiff; cloud-optimized=true", + "roles": [ + "data" + ], + "eo:bands": [ + { + "name": "B1", + "common_name": "coastal", + "gsd": 30, + "center_wavelength": 0.44 + } + ], + "href": "https://landsatlook.usgs.gov/data/collection02/level-2/standard/oli-tirs/2023/092/068/LC09_L2SP_092068_20230607_20230609_02_T1/LC09_L2SP_092068_20230607_20230609_02_T1_SR_B1.TIF", + "alternate": { + "s3": { + "storage:platform": "AWS", + "storage:requester_pays": true, + "href": "s3://usgs-landsat/collection02/level-2/standard/oli-tirs/2023/092/068/LC09_L2SP_092068_20230607_20230609_02_T1/LC09_L2SP_092068_20230607_20230609_02_T1_SR_B1.TIF" + } + } + }, + "blue": { + "title": "Blue Band (B2)", + "description": "Collection 2 Level-2 Blue Band (B2) Surface Reflectance", + "type": "image/vnd.stac.geotiff; cloud-optimized=true", + "roles": [ + "data" + ], + "eo:bands": [ + { + "name": "B2", + "common_name": "blue", + "gsd": 30, + "center_wavelength": 0.48 + } + ], + "href": "https://landsatlook.usgs.gov/data/collection02/level-2/standard/oli-tirs/2023/092/068/LC09_L2SP_092068_20230607_20230609_02_T1/LC09_L2SP_092068_20230607_20230609_02_T1_SR_B2.TIF", + "alternate": { + "s3": { + "storage:platform": "AWS", + "storage:requester_pays": true, + "href": "s3://usgs-landsat/collection02/level-2/standard/oli-tirs/2023/092/068/LC09_L2SP_092068_20230607_20230609_02_T1/LC09_L2SP_092068_20230607_20230609_02_T1_SR_B2.TIF" + } + } + }, + "green": { + "title": "Green Band (B3)", + "description": "Collection 2 Level-2 Green Band (B3) Surface Reflectance", + "type": "image/vnd.stac.geotiff; cloud-optimized=true", + "roles": [ + "data" + ], + "eo:bands": [ + { + "name": "B3", + "common_name": "green", + "gsd": 30, + "center_wavelength": 0.56 + } + ], + "href": "https://landsatlook.usgs.gov/data/collection02/level-2/standard/oli-tirs/2023/092/068/LC09_L2SP_092068_20230607_20230609_02_T1/LC09_L2SP_092068_20230607_20230609_02_T1_SR_B3.TIF", + "alternate": { + "s3": { + "storage:platform": "AWS", + "storage:requester_pays": true, + "href": "s3://usgs-landsat/collection02/level-2/standard/oli-tirs/2023/092/068/LC09_L2SP_092068_20230607_20230609_02_T1/LC09_L2SP_092068_20230607_20230609_02_T1_SR_B3.TIF" + } + } + }, + "red": { + "title": "Red Band (B4)", + "description": "Collection 2 Level-2 Red Band (B4) Surface Reflectance", + "type": "image/vnd.stac.geotiff; cloud-optimized=true", + "roles": [ + "data" + ], + "eo:bands": [ + { + "name": "B4", + "common_name": "red", + "gsd": 30, + "center_wavelength": 0.65 + } + ], + "href": "https://landsatlook.usgs.gov/data/collection02/level-2/standard/oli-tirs/2023/092/068/LC09_L2SP_092068_20230607_20230609_02_T1/LC09_L2SP_092068_20230607_20230609_02_T1_SR_B4.TIF", + "alternate": { + "s3": { + "storage:platform": "AWS", + "storage:requester_pays": true, + "href": "s3://usgs-landsat/collection02/level-2/standard/oli-tirs/2023/092/068/LC09_L2SP_092068_20230607_20230609_02_T1/LC09_L2SP_092068_20230607_20230609_02_T1_SR_B4.TIF" + } + } + }, + "nir08": { + "title": "Near Infrared Band 0.8 (B5)", + "description": "Collection 2 Level-2 Near Infrared Band 0.8 (B5) Surface Reflectance", + "type": "image/vnd.stac.geotiff; cloud-optimized=true", + "roles": [ + "data", + "reflectance" + ], + "eo:bands": [ + { + "name": "B5", + "common_name": "nir08", + "gsd": 30, + "center_wavelength": 0.86 + } + ], + "href": "https://landsatlook.usgs.gov/data/collection02/level-2/standard/oli-tirs/2023/092/068/LC09_L2SP_092068_20230607_20230609_02_T1/LC09_L2SP_092068_20230607_20230609_02_T1_SR_B5.TIF", + "alternate": { + "s3": { + "storage:platform": "AWS", + "storage:requester_pays": true, + "href": "s3://usgs-landsat/collection02/level-2/standard/oli-tirs/2023/092/068/LC09_L2SP_092068_20230607_20230609_02_T1/LC09_L2SP_092068_20230607_20230609_02_T1_SR_B5.TIF" + } + } + }, + "swir16": { + "title": "Short-wave Infrared Band 1.6 (B6)", + "description": "Collection 2 Level-2 Short-wave Infrared Band 1.6 (B6) Surface Reflectance", + "type": "image/vnd.stac.geotiff; cloud-optimized=true", + "roles": [ + "data", + "reflectance" + ], + "eo:bands": [ + { + "name": "B6", + "common_name": "swir16", + "gsd": 30, + "center_wavelength": 1.6 + } + ], + "href": "https://landsatlook.usgs.gov/data/collection02/level-2/standard/oli-tirs/2023/092/068/LC09_L2SP_092068_20230607_20230609_02_T1/LC09_L2SP_092068_20230607_20230609_02_T1_SR_B6.TIF", + "alternate": { + "s3": { + "storage:platform": "AWS", + "storage:requester_pays": true, + "href": "s3://usgs-landsat/collection02/level-2/standard/oli-tirs/2023/092/068/LC09_L2SP_092068_20230607_20230609_02_T1/LC09_L2SP_092068_20230607_20230609_02_T1_SR_B6.TIF" + } + } + }, + "swir22": { + "title": "Short-wave Infrared Band 2.2 (B7)", + "description": "Collection 2 Level-2 Short-wave Infrared Band 2.2 (B7) Surface Reflectance", + "type": "image/vnd.stac.geotiff; cloud-optimized=true", + "roles": [ + "data", + "reflectance" + ], + "eo:bands": [ + { + "name": "B7", + "common_name": "swir22", + "gsd": 30, + "center_wavelength": 2.2 + } + ], + "href": "https://landsatlook.usgs.gov/data/collection02/level-2/standard/oli-tirs/2023/092/068/LC09_L2SP_092068_20230607_20230609_02_T1/LC09_L2SP_092068_20230607_20230609_02_T1_SR_B7.TIF", + "alternate": { + "s3": { + "storage:platform": "AWS", + "storage:requester_pays": true, + "href": "s3://usgs-landsat/collection02/level-2/standard/oli-tirs/2023/092/068/LC09_L2SP_092068_20230607_20230609_02_T1/LC09_L2SP_092068_20230607_20230609_02_T1_SR_B7.TIF" + } + } + }, + "qa_aerosol": { + "title": "Aerosol Quality Analysis Band", + "description": "Collection 2 Level-2 Aerosol Quality Analysis Band Surface Reflectance", + "type": "image/vnd.stac.geotiff; cloud-optimized=true", + "roles": [ + "metadata", + "data-mask", + "water-mask" + ], + "classification:bitfields": [ + { + "name": "fill", + "description": "Corresponding pixels in L1 image bands are fill", + "offset": 0, + "length": 1, + "classes": [ + { + "name": "not_fill", + "description": "L1 image band pixels are not fill", + "value": 0 + }, + { + "name": "fill", + "description": "L1 image band pixels are fill", + "value": 1 + } + ] + }, + { + "name": "retrieval", + "description": "Valid aerosol retrieval", + "offset": 1, + "length": 1, + "classes": [ + { + "name": "not_valid", + "description": "Aerosol retrieval is not valid", + "value": 0 + }, + { + "name": "valid", + "description": "Aerosol retrieval is valid", + "value": 1 + } + ] + }, + { + "name": "water", + "description": "Water mask", + "offset": 2, + "length": 1, + "classes": [ + { + "name": "not_water", + "description": "Not water", + "value": 0 + }, + { + "name": "water", + "description": "Water", + "value": 1 + } + ] + }, + { + "name": "unused", + "description": "Unused bit", + "offset": 3, + "length": 1, + "classes": [ + { + "name": "unused", + "description": "Unused bit", + "value": 0 + } + ] + }, + { + "name": "unused", + "description": "Unused bit", + "offset": 4, + "length": 1, + "classes": [ + { + "name": "unused", + "description": "Unused bit", + "value": 0 + } + ] + }, + { + "name": "interpolated", + "description": "Aerosol is interpolated", + "offset": 5, + "length": 1, + "classes": [ + { + "name": "not_interpolated", + "description": "Aerosol is not interpolated", + "value": 0 + }, + { + "name": "interpolated", + "description": "Aerosol is interpolated", + "value": 1 + } + ] + }, + { + "name": "level", + "description": "Aerosol level", + "offset": 6, + "length": 2, + "classes": [ + { + "name": "climatology", + "description": "No aerosol correction applied", + "value": 0 + }, + { + "name": "low", + "description": "Low aerosol level", + "value": 1 + }, + { + "name": "medium", + "description": "Medium aerosol level", + "value": 2 + }, + { + "name": "high", + "description": "High aerosol level", + "value": 3 + } + ] + } + ], + "href": "https://landsatlook.usgs.gov/data/collection02/level-2/standard/oli-tirs/2023/092/068/LC09_L2SP_092068_20230607_20230609_02_T1/LC09_L2SP_092068_20230607_20230609_02_T1_SR_QA_AEROSOL.TIF", + "alternate": { + "s3": { + "storage:platform": "AWS", + "storage:requester_pays": true, + "href": "s3://usgs-landsat/collection02/level-2/standard/oli-tirs/2023/092/068/LC09_L2SP_092068_20230607_20230609_02_T1/LC09_L2SP_092068_20230607_20230609_02_T1_SR_QA_AEROSOL.TIF" + } + } + }, + "qa_pixel": { + "title": "Pixel Quality Assessment Band", + "description": "Collection 2 Level-2 Pixel Quality Assessment Band Surface Reflectance", + "type": "image/vnd.stac.geotiff; cloud-optimized=true", + "roles": [ + "cloud", + "cloud-shadow", + "snow-ice", + "water-mask" + ], + "classification:bitfields": [ + { + "name": "fill", + "description": "Corresponding pixels in L1 image bands are fill", + "offset": 0, + "length": 1, + "classes": [ + { + "name": "not_fill", + "description": "L1 image band pixels are not fill", + "value": 0 + }, + { + "name": "fill", + "description": "L1 image band pixels are fill", + "value": 1 + } + ] + }, + { + "name": "dilated", + "description": "Dilated cloud", + "offset": 1, + "length": 1, + "classes": [ + { + "name": "not_dilated", + "description": "Cloud is not dilated or no cloud", + "value": 0 + }, + { + "name": "dilated", + "description": "Cloud dilation", + "value": 1 + } + ] + }, + { + "name": "cirrus", + "description": "Cirrus mask", + "offset": 2, + "length": 1, + "classes": [ + { + "name": "not_cirrus", + "description": "No confidence level set or low confidence cirrus", + "value": 0 + }, + { + "name": "cirrus", + "description": "High confidence cirrus", + "value": 1 + } + ] + }, + { + "name": "cloud", + "description": "Cloud mask", + "offset": 3, + "length": 1, + "classes": [ + { + "name": "not_cloud", + "description": "Cloud confidence is not high", + "value": 0 + }, + { + "name": "cloud", + "description": "High confidence cloud", + "value": 1 + } + ] + }, + { + "name": "shadow", + "description": "Cloud shadow mask", + "offset": 4, + "length": 1, + "classes": [ + { + "name": "not_shadow", + "description": "Cloud shadow confidence is not high", + "value": 0 + }, + { + "name": "shadow", + "description": "High confidence cloud shadow", + "value": 1 + } + ] + }, + { + "name": "snow", + "description": "Snow/Ice mask", + "offset": 5, + "length": 1, + "classes": [ + { + "name": "not_snow", + "description": "Snow/Ice confidence is not high", + "value": 0 + }, + { + "name": "snow", + "description": "High confidence snow cover", + "value": 1 + } + ] + }, + { + "name": "clear", + "description": "Cloud or dilated cloud bits set", + "offset": 6, + "length": 1, + "classes": [ + { + "name": "not_clear", + "description": "Cloud or dilated cloud bits are set", + "value": 0 + }, + { + "name": "clear", + "description": "Cloud and dilated cloud bits are not set", + "value": 1 + } + ] + }, + { + "name": "water", + "description": "Water mask", + "offset": 7, + "length": 1, + "classes": [ + { + "name": "not_water", + "description": "Land or cloud", + "value": 0 + }, + { + "name": "water", + "description": "Water", + "value": 1 + } + ] + }, + { + "name": "cloud_confidence", + "description": "Cloud confidence levels", + "offset": 8, + "length": 2, + "classes": [ + { + "name": "not_set", + "description": "No confidence level set", + "value": 0 + }, + { + "name": "low", + "description": "Low confidence cloud", + "value": 1 + }, + { + "name": "medium", + "description": "Medium confidence cloud", + "value": 2 + }, + { + "name": "high", + "description": "High confidence cloud", + "value": 3 + } + ] + }, + { + "name": "shadow_confidence", + "description": "Cloud shadow confidence levels", + "offset": 10, + "length": 2, + "classes": [ + { + "name": "not_set", + "description": "No confidence level set", + "value": 0 + }, + { + "name": "low", + "description": "Low confidence cloud shadow", + "value": 1 + }, + { + "name": "reserved", + "description": "Reserved - value not used", + "value": 2 + }, + { + "name": "high", + "description": "High confidence cloud shadow", + "value": 3 + } + ] + }, + { + "name": "snow_confidence", + "description": "Snow/Ice confidence levels", + "offset": 12, + "length": 2, + "classes": [ + { + "name": "not_set", + "description": "No confidence level set", + "value": 0 + }, + { + "name": "low", + "description": "Low confidence snow/ice", + "value": 1 + }, + { + "name": "reserved", + "description": "Reserved - value not used", + "value": 2 + }, + { + "name": "high", + "description": "High confidence snow/ice", + "value": 3 + } + ] + }, + { + "name": "cirrus_confidence", + "description": "Cirrus confidence levels", + "offset": 14, + "length": 2, + "classes": [ + { + "name": "not_set", + "description": "No confidence level set", + "value": 0 + }, + { + "name": "low", + "description": "Low confidence cirrus", + "value": 1 + }, + { + "name": "reserved", + "description": "Reserved - value not used", + "value": 2 + }, + { + "name": "high", + "description": "High confidence cirrus", + "value": 3 + } + ] + } + ], + "href": "https://landsatlook.usgs.gov/data/collection02/level-2/standard/oli-tirs/2023/092/068/LC09_L2SP_092068_20230607_20230609_02_T1/LC09_L2SP_092068_20230607_20230609_02_T1_QA_PIXEL.TIF", + "alternate": { + "s3": { + "storage:platform": "AWS", + "storage:requester_pays": true, + "href": "s3://usgs-landsat/collection02/level-2/standard/oli-tirs/2023/092/068/LC09_L2SP_092068_20230607_20230609_02_T1/LC09_L2SP_092068_20230607_20230609_02_T1_QA_PIXEL.TIF" + } + } + }, + "qa_radsat": { + "title": "Radiometric Saturation Quality Assessment Band", + "description": "Collection 2 Level-2 Radiometric Saturation Quality Assessment Band Surface Reflectance", + "type": "image/vnd.stac.geotiff; cloud-optimized=true", + "roles": [ + "saturation" + ], + "classification:bitfields": [ + { + "name": "band1", + "description": "Band 1 radiometric saturation", + "offset": 0, + "length": 1, + "classes": [ + { + "name": "not_saturated", + "description": "Band 1 is not saturated", + "value": 0 + }, + { + "name": "saturated", + "description": "Band 1 is saturated", + "value": 1 + } + ] + }, + { + "name": "band2", + "description": "Band 2 radiometric saturation", + "offset": 1, + "length": 1, + "classes": [ + { + "name": "not_saturated", + "description": "Band 2 is not saturated", + "value": 0 + }, + { + "name": "saturated", + "description": "Band 2 is saturated", + "value": 1 + } + ] + }, + { + "name": "band3", + "description": "Band 3 radiometric saturation", + "offset": 2, + "length": 1, + "classes": [ + { + "name": "not_saturated", + "description": "Band 3 is not saturated", + "value": 0 + }, + { + "name": "saturated", + "description": "Band 3 is saturated", + "value": 1 + } + ] + }, + { + "name": "band4", + "description": "Band 4 radiometric saturation", + "offset": 3, + "length": 1, + "classes": [ + { + "name": "not_saturated", + "description": "Band 4 is not saturated", + "value": 0 + }, + { + "name": "saturated", + "description": "Band 4 is saturated", + "value": 1 + } + ] + }, + { + "name": "band5", + "description": "Band 5 radiometric saturation", + "offset": 4, + "length": 1, + "classes": [ + { + "name": "not_saturated", + "description": "Band 5 is not saturated", + "value": 0 + }, + { + "name": "saturated", + "description": "Band 5 is saturated", + "value": 1 + } + ] + }, + { + "name": "band6", + "description": "Band 6 radiometric saturation", + "offset": 5, + "length": 1, + "classes": [ + { + "name": "not_saturated", + "description": "Band 6 is not saturated", + "value": 0 + }, + { + "name": "saturated", + "description": "Band 6 is saturated", + "value": 1 + } + ] + }, + { + "name": "band7", + "description": "Band 7 radiometric saturation", + "offset": 6, + "length": 1, + "classes": [ + { + "name": "not_saturated", + "description": "Band 7 is not saturated", + "value": 0 + }, + { + "name": "saturated", + "description": "Band 7 is saturated", + "value": 1 + } + ] + }, + { + "name": "unused", + "description": "Unused bit", + "offset": 7, + "length": 1, + "classes": [ + { + "name": "unused", + "description": "Unused bit", + "value": 0 + } + ] + }, + { + "name": "band9", + "description": "Band 9 radiometric saturation", + "offset": 8, + "length": 1, + "classes": [ + { + "name": "not_saturated", + "description": "Band 9 is not saturated", + "value": 0 + }, + { + "name": "saturated", + "description": "Band 9 is saturated", + "value": 1 + } + ] + }, + { + "name": "unused", + "description": "Unused bit", + "offset": 9, + "length": 1, + "classes": [ + { + "name": "unused", + "description": "Unused bit", + "value": 0 + } + ] + }, + { + "name": "unused", + "description": "Unused bit", + "offset": 10, + "length": 1, + "classes": [ + { + "name": "unused", + "description": "Unused bit", + "value": 0 + } + ] + }, + { + "name": "occlusion", + "description": "Terrain not visible from sensor due to intervening terrain", + "offset": 11, + "length": 1, + "classes": [ + { + "name": "not_occluded", + "description": "Terrain is not occluded", + "value": 0 + }, + { + "name": "occluded", + "description": "Terrain is occluded", + "value": 1 + } + ] + }, + { + "name": "unused", + "description": "Unused bit", + "offset": 12, + "length": 1, + "classes": [ + { + "name": "unused", + "description": "Unused bit", + "value": 0 + } + ] + }, + { + "name": "unused", + "description": "Unused bit", + "offset": 13, + "length": 1, + "classes": [ + { + "name": "unused", + "description": "Unused bit", + "value": 0 + } + ] + }, + { + "name": "unused", + "description": "Unused bit", + "offset": 14, + "length": 1, + "classes": [ + { + "name": "unused", + "description": "Unused bit", + "value": 0 + } + ] + }, + { + "name": "unused", + "description": "Unused bit", + "offset": 15, + "length": 1, + "classes": [ + { + "name": "unused", + "description": "Unused bit", + "value": 0 + } + ] + } + ], + "href": "https://landsatlook.usgs.gov/data/collection02/level-2/standard/oli-tirs/2023/092/068/LC09_L2SP_092068_20230607_20230609_02_T1/LC09_L2SP_092068_20230607_20230609_02_T1_QA_RADSAT.TIF", + "alternate": { + "s3": { + "storage:platform": "AWS", + "storage:requester_pays": true, + "href": "s3://usgs-landsat/collection02/level-2/standard/oli-tirs/2023/092/068/LC09_L2SP_092068_20230607_20230609_02_T1/LC09_L2SP_092068_20230607_20230609_02_T1_QA_RADSAT.TIF" + } + } + }, + "ANG.txt": { + "title": "Angle Coefficients File", + "description": "Collection 2 Level-2 Angle Coefficients File (ANG)", + "type": "text/plain", + "roles": [ + "metadata" + ], + "href": "https://landsatlook.usgs.gov/data/collection02/level-2/standard/oli-tirs/2023/092/068/LC09_L2SP_092068_20230607_20230609_02_T1/LC09_L2SP_092068_20230607_20230609_02_T1_ANG.txt", + "alternate": { + "s3": { + "storage:platform": "AWS", + "storage:requester_pays": true, + "href": "s3://usgs-landsat/collection02/level-2/standard/oli-tirs/2023/092/068/LC09_L2SP_092068_20230607_20230609_02_T1/LC09_L2SP_092068_20230607_20230609_02_T1_ANG.txt" + } + } + }, + "MTL.txt": { + "title": "Product Metadata File", + "description": "Collection 2 Level-2 Product Metadata File (MTL)", + "type": "text/plain", + "roles": [ + "metadata" + ], + "href": "https://landsatlook.usgs.gov/data/collection02/level-2/standard/oli-tirs/2023/092/068/LC09_L2SP_092068_20230607_20230609_02_T1/LC09_L2SP_092068_20230607_20230609_02_T1_MTL.txt", + "alternate": { + "s3": { + "storage:platform": "AWS", + "storage:requester_pays": true, + "href": "s3://usgs-landsat/collection02/level-2/standard/oli-tirs/2023/092/068/LC09_L2SP_092068_20230607_20230609_02_T1/LC09_L2SP_092068_20230607_20230609_02_T1_MTL.txt" + } + } + }, + "MTL.xml": { + "title": "Product Metadata File (xml)", + "description": "Collection 2 Level-2 Product Metadata File (xml)", + "type": "application/xml", + "roles": [ + "metadata" + ], + "href": "https://landsatlook.usgs.gov/data/collection02/level-2/standard/oli-tirs/2023/092/068/LC09_L2SP_092068_20230607_20230609_02_T1/LC09_L2SP_092068_20230607_20230609_02_T1_MTL.xml", + "alternate": { + "s3": { + "storage:platform": "AWS", + "storage:requester_pays": true, + "href": "s3://usgs-landsat/collection02/level-2/standard/oli-tirs/2023/092/068/LC09_L2SP_092068_20230607_20230609_02_T1/LC09_L2SP_092068_20230607_20230609_02_T1_MTL.xml" + } + } + } + }, + "links": [ + { + "rel": "self", + "href": "https://landsatlook.usgs.gov/stac-server/collections/landsat-c2l2-sr/items/LC09_L2SP_092068_20230607_20230609_02_T1_SR" + }, + { + "rel": "parent", + "href": "https://landsatlook.usgs.gov/stac-server/collections/landsat-c2l2-sr" + }, + { + "rel": "collection", + "href": "https://landsatlook.usgs.gov/stac-server/collections/landsat-c2l2-sr" + }, + { + "rel": "root", + "href": "https://landsatlook.usgs.gov/stac-server/" + } + ], + "collection": "landsat-c2l2-sr" +} \ No newline at end of file diff --git a/tests/test_cli.py b/tests/test_cli.py index 3a9076e..7be7a29 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -63,6 +63,8 @@ def test_download_item_s3_requester_pays(tmp_path: Path) -> None: "--s3-requester-pays", "-i", "thumbnail", + "--alternate-assets", + "s3", ], ) assert result.exit_code == 0 diff --git a/tests/test_earthdata_client.py b/tests/test_earthdata_client.py index d018e0e..f2782bc 100644 --- a/tests/test_earthdata_client.py +++ b/tests/test_earthdata_client.py @@ -16,6 +16,6 @@ async def test_download_href(tmp_path: Path) -> None: href = "https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/MYD11A1.061/MYD11A1.A2023145.h14v17.061.2023146183035/MYD11A1.A2023145.h14v17.061.2023146183035.hdf" - async with await EarthdataClient.default() as client: + async with await EarthdataClient.login() as client: await client.download_href(href, tmp_path / "out.hdf") assert os.path.getsize(tmp_path / "out.hdf") == 197419 diff --git a/tests/test_filesystem_client.py b/tests/test_filesystem_client.py index 2c2171d..7a11a92 100644 --- a/tests/test_filesystem_client.py +++ b/tests/test_filesystem_client.py @@ -2,6 +2,7 @@ from pathlib import Path import pytest +import stac_asset from pystac import Asset, Item, ItemCollection from stac_asset import ( AssetOverwriteError, @@ -29,9 +30,7 @@ async def test_download(tmp_path: Path, asset_href: str) -> None: async def test_download_item(tmp_path: Path, item: Item) -> None: - async with FilesystemClient() as client: - item = await client.download_item(item, tmp_path) - + item = await stac_asset.download_item(item, tmp_path) assert Path(tmp_path / "item.json").exists(), item.get_self_href() asset = item.assets["data"] assert asset.href == "./20201211_223832_CS2.jpg" @@ -40,74 +39,61 @@ async def test_download_item(tmp_path: Path, item: Item) -> None: async def test_download_item_collection( tmp_path: Path, item_collection: ItemCollection ) -> None: - async with FilesystemClient() as client: - await client.download_item_collection( - item_collection, tmp_path, Config(file_name="item-collection.json") - ) - + await stac_asset.download_item_collection( + item_collection, tmp_path, Config(file_name="item-collection.json") + ) assert os.path.exists(tmp_path / "item-collection.json") assert os.path.exists(tmp_path / "test-item" / "20201211_223832_CS2.jpg") async def test_item_download_404(tmp_path: Path, item: Item) -> None: item.assets["missing-asset"] = Asset(href=str(Path(__file__).parent / "not-a-file")) - async with FilesystemClient() as client: - with pytest.raises(DownloadError): - await client.download_item(item, tmp_path) - + with pytest.raises(DownloadError): + await stac_asset.download_item(item, tmp_path) assert not (tmp_path / "not-a-file").exists() async def test_item_download_404_warn(tmp_path: Path, item: Item) -> None: item.assets["missing-asset"] = Asset(href=str(Path(__file__).parent / "not-a-file")) - async with FilesystemClient() as client: - with pytest.warns(DownloadWarning): - item = await client.download_item(item, tmp_path, Config(warn=True)) - + with pytest.warns(DownloadWarning): + item = await stac_asset.download_item(item, tmp_path, Config(warn=True)) assert not (tmp_path / "not-a-file").exists() assert "missing-asset" not in item.assets async def test_item_download_no_directory(tmp_path: Path, item: Item) -> None: - async with FilesystemClient() as client: - with pytest.raises(FileNotFoundError): - await client.download_item( - item, tmp_path / "doesnt-exist", Config(make_directory=False) - ) + with pytest.raises(FileNotFoundError): + await stac_asset.download_item( + item, tmp_path / "doesnt-exist", Config(make_directory=False) + ) async def test_item_download_key(tmp_path: Path, item: Item) -> None: - async with FilesystemClient() as client: - await client.download_item( - item, tmp_path, Config(asset_file_name_strategy=FileNameStrategy.KEY) - ) - + await stac_asset.download_item( + item, tmp_path, Config(asset_file_name_strategy=FileNameStrategy.KEY) + ) assert Path(tmp_path / "data.jpg").exists() async def test_item_download_same_file_name(tmp_path: Path, item: Item) -> None: item.assets["other-data"] = item.assets["data"].clone() - async with FilesystemClient() as client: - with pytest.raises(AssetOverwriteError): - await client.download_item(item, tmp_path) + with pytest.raises(AssetOverwriteError): + await stac_asset.download_item(item, tmp_path) async def test_include(tmp_path: Path, item: Item) -> None: item.assets["other-data"] = item.assets["data"].clone() - async with FilesystemClient() as client: - await client.download_item(item, tmp_path, Config(include=["data"])) + await stac_asset.download_item(item, tmp_path, Config(include=["data"])) async def test_exclude(tmp_path: Path, item: Item) -> None: item.assets["other-data"] = item.assets["data"].clone() - async with FilesystemClient() as client: - await client.download_item(item, tmp_path, Config(exclude=["other-data"])) + await stac_asset.download_item(item, tmp_path, Config(exclude=["other-data"])) async def test_cant_include_and_exclude(tmp_path: Path, item: Item) -> None: item.assets["other-data"] = item.assets["data"].clone() - async with FilesystemClient() as client: - with pytest.raises(CannotIncludeAndExclude): - await client.download_item( - item, tmp_path, Config(include=["data"], exclude=["other-data"]) - ) + with pytest.raises(CannotIncludeAndExclude): + await stac_asset.download_item( + item, tmp_path, Config(include=["data"], exclude=["other-data"]) + ) diff --git a/tests/test_functions.py b/tests/test_functions.py new file mode 100644 index 0000000..fef28e3 --- /dev/null +++ b/tests/test_functions.py @@ -0,0 +1,20 @@ +from pathlib import Path + +import pytest +import stac_asset +from pystac import Asset, Item +from stac_asset import Config, FileNameStrategy + +pytestmark = [ + pytest.mark.asyncio, + pytest.mark.network_access, +] + + +async def test_multiple_clients(tmp_path: Path, item: Item) -> None: + item.assets["remote"] = Asset( + href="https://storage.googleapis.com/open-cogs/stac-examples/20201211_223832_CS2.jpg", + ) + item = await stac_asset.download_item( + item, tmp_path, Config(asset_file_name_strategy=FileNameStrategy.KEY) + ) diff --git a/tests/test_planetary_computer_client.py b/tests/test_planetary_computer_client.py index 7ab88ea..f7b1781 100644 --- a/tests/test_planetary_computer_client.py +++ b/tests/test_planetary_computer_client.py @@ -2,7 +2,7 @@ from pathlib import Path import pytest -from stac_asset import PlanetaryComputerClient +from stac_asset import Config, PlanetaryComputerClient pytestmark = [ pytest.mark.network_access, @@ -16,7 +16,7 @@ def asset_href() -> str: async def test_download(tmp_path: Path, asset_href: str) -> None: - async with await PlanetaryComputerClient.default() as client: + async with await PlanetaryComputerClient.from_config(Config()) as client: await client.download_href(asset_href, tmp_path / "out.tif") assert os.path.getsize(tmp_path / "out.tif") == 4096 diff --git a/tests/test_s3_client.py b/tests/test_s3_client.py index 3d32012..201f996 100644 --- a/tests/test_s3_client.py +++ b/tests/test_s3_client.py @@ -4,6 +4,7 @@ import pystac import pytest +import stac_asset from pystac import Item from stac_asset import Config, S3Client @@ -34,7 +35,7 @@ def requester_pays_item(data_path: Path) -> Item: async def test_download(tmp_path: Path, asset_href: str) -> None: - async with await S3Client.default() as client: + async with S3Client() as client: await client.download_href(asset_href, tmp_path / "out.jpg") assert os.path.getsize(tmp_path / "out.jpg") == 6060 @@ -53,15 +54,14 @@ async def test_download_requester_pays_asset( async def test_download_requester_pays_item( tmp_path: Path, requester_pays_item: Item ) -> None: - async with S3Client(requester_pays=True) as client: - if not await client.has_credentials(): - pytest.skip("aws credentials are invalid or not present") - await client.download_item( - requester_pays_item, tmp_path, Config(include=["thumbnail"]) - ) - assert ( - os.path.getsize( - tmp_path / "LC09_L2SP_092068_20230607_20230609_02_T1_thumb_small.jpeg" - ) - == 19554 + await stac_asset.download_item( + requester_pays_item, + tmp_path, + Config(include=["thumbnail"], s3_requester_pays=True, alternate_assets=["s3"]), + ) + assert ( + os.path.getsize( + tmp_path / "LC09_L2SP_092068_20230607_20230609_02_T1_thumb_small.jpeg" ) + == 19554 + )