Add s3 configuration to testing's external data

src/stactools/testing/test_data.py

@@ -46,9 +46,25 @@ def get_external_data(self, rel_path: str) -> str:
             print('Downloading external test data {}...'.format(rel_path))
             os.makedirs(os.path.dirname(path), exist_ok=True)
 
-            with fsspec.open(entry["url"]) as f:
-                data = f.read()
-            if entry['compress'] == 'zip':
+            s3_config = entry.get("s3")
+            if s3_config:
+                try:
+                    import s3fs
+                except ImportError as e:
+                    print(
+                        "Trying to download external test data via s3, "
+                        "but s3fs is not installed and the download requires "
+                        "configuring the s3fs filesystem. Install stactools "
+                        "with s3fs via `pip install stactools[s3]` and try again."
+                    )
+                    raise (e)
+                s3 = s3fs.S3FileSystem(**s3_config)
+                with s3.open(entry["url"]) as f:
+                    data = f.read()
+            else:
+                with fsspec.open(entry["url"]) as f:
+                    data = f.read()
+            if entry.get("compress") == 'zip':
                 with TemporaryDirectory() as tmp_dir:
                     tmp_path = os.path.join(tmp_dir, 'file.zip')
                     with open(tmp_path, 'wb') as f:

tests/testing/__init__.py

@@ -9,6 +9,14 @@
         },
         "goes-16/index.html": {
             "url": "s3://noaa-goes16/index.html",
-            "compress": False
+        },
+        "AW3D30_global.vrt": {
+            "url": "s3://raster/AW3D30/AW3D30_global.vrt",
+            "s3": {
+                "anon": True,
+                "client_kwargs": {
+                    "endpoint_url": "https://opentopography.s3.sdsc.edu"
+                }
+            }
         }
     })

tests/testing/test_test_data.py

@@ -13,16 +13,29 @@ def test_external_data_https(self):
         self.assertEqual(item.id, "20201211_223832_CS2")
 
     def test_external_data_s3(self):
-        try:
-            import s3fs  # noqa
-        except ImportError:
-            raise SkipTest(
-                "Skipping testing external data with s3 because s3fs is not installed"
-            )
+        skip_without_s3fs()
         path = test_data.get_external_data("goes-16/index.html")
         with open(path) as f:
             html = f.read()
         self.assertNotEqual(
             html.find("Amazon"), -1,
             "Could not find 'Amazon' in the index page for the AWS public dataset"
         )
+
+    def test_external_data_s3_with_config(self):
+        skip_without_s3fs()
+        path = test_data.get_external_data("AW3D30_global.vrt")
+        with open(path) as f:
+            xml = f.read()
+        self.assertNotEqual(
+            xml.find("ALPSMLC30_N041W106_DSM"), -1,
+            "Could not find 'ALPSMLC30_N041W106_DSM' in the ALOS VRT")
+
+
+def skip_without_s3fs():
+    try:
+        import s3fs  # noqa
+    except ImportError:
+        raise SkipTest(
+            "Skipping testing external data with s3 because s3fs is not installed"
+        )