diff --git a/docs/api/datamodules.rst b/docs/api/datamodules.rst index a4afe8c366..abb0c6eaef 100644 --- a/docs/api/datamodules.rst +++ b/docs/api/datamodules.rst @@ -193,7 +193,9 @@ So2Sat SpaceNet ^^^^^^^^ +.. autoclass:: SpaceNetBaseDataModule .. autoclass:: SpaceNet1DataModule +.. autoclass:: SpaceNet6DataModule SSL4EO ^^^^^^ diff --git a/tests/conf/spacenet1.yaml b/tests/conf/spacenet1.yaml index 275e54eee0..6f47d88d35 100644 --- a/tests/conf/spacenet1.yaml +++ b/tests/conf/spacenet1.yaml @@ -15,4 +15,4 @@ data: val_split_pct: 0.34 test_split_pct: 0.34 dict_kwargs: - root: 'tests/data/spacenet' + root: 'tests/data/spacenet/spacenet1' diff --git a/tests/conf/spacenet6.yaml b/tests/conf/spacenet6.yaml new file mode 100644 index 0000000000..c017102d3d --- /dev/null +++ b/tests/conf/spacenet6.yaml @@ -0,0 +1,19 @@ +model: + class_path: SemanticSegmentationTask + init_args: + loss: 'ce' + model: 'unet' + backbone: 'resnet18' + in_channels: 4 + num_classes: 3 + num_filters: 1 + ignore_index: null +data: + class_path: SpaceNet6DataModule + init_args: + batch_size: 1 + val_split_pct: 0.34 + test_split_pct: 0.34 + dict_kwargs: + root: 'tests/data/spacenet/spacenet6' + image: 'SAR-Intensity' diff --git a/tests/data/spacenet/SN1_buildings/train/SN1_buildings_train_AOI_1_Rio_3band.tar.gz b/tests/data/spacenet/SN1_buildings/train/SN1_buildings_train_AOI_1_Rio_3band.tar.gz deleted file mode 100644 index 78352e2c5f..0000000000 Binary files a/tests/data/spacenet/SN1_buildings/train/SN1_buildings_train_AOI_1_Rio_3band.tar.gz and /dev/null differ diff --git a/tests/data/spacenet/SN1_buildings/train/SN1_buildings_train_AOI_1_Rio_8band.tar.gz b/tests/data/spacenet/SN1_buildings/train/SN1_buildings_train_AOI_1_Rio_8band.tar.gz deleted file mode 100644 index 29f8d9cbc2..0000000000 Binary files a/tests/data/spacenet/SN1_buildings/train/SN1_buildings_train_AOI_1_Rio_8band.tar.gz and /dev/null differ diff --git a/tests/data/spacenet/SN1_buildings/train/SN1_buildings_train_AOI_1_Rio_geojson_buildings.tar.gz b/tests/data/spacenet/SN1_buildings/train/SN1_buildings_train_AOI_1_Rio_geojson_buildings.tar.gz deleted file mode 100644 index 1889962783..0000000000 Binary files a/tests/data/spacenet/SN1_buildings/train/SN1_buildings_train_AOI_1_Rio_geojson_buildings.tar.gz and /dev/null differ diff --git a/tests/data/spacenet/data.py b/tests/data/spacenet/data.py deleted file mode 100755 index 4d99e415e6..0000000000 --- a/tests/data/spacenet/data.py +++ /dev/null @@ -1,92 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. - -import json -import os -import shutil - -import numpy as np -import rasterio -from rasterio.crs import CRS -from rasterio.transform import Affine - -SIZE = 2 - -dataset_id = 'SN1_buildings' - -profile = { - 'driver': 'GTiff', - 'dtype': 'uint8', - 'width': SIZE, - 'height': SIZE, - 'crs': CRS.from_epsg(4326), - 'transform': Affine( - 4.489235388119662e-06, - 0.0, - -43.7732462563, - 0.0, - -4.486127586210932e-06, - -22.9214851954, - ), -} - -np.random.seed(0) -Z = np.random.randint(np.iinfo('uint8').max, size=(SIZE, SIZE), dtype='uint8') - -for count in [3, 8]: - os.makedirs(os.path.join(dataset_id, 'train', f'{count}band'), exist_ok=True) - for i in range(1, 5): - path = os.path.join( - dataset_id, 'train', f'{count}band', f'3band_AOI_1_RIO_img{i}.tif' - ) - profile['count'] = count - with rasterio.open(path, 'w', **profile) as src: - for j in range(1, count + 1): - src.write(Z, j) - - shutil.make_archive( - os.path.join(dataset_id, 'train', f'SN1_buildings_train_AOI_1_Rio_{count}band'), - 'gztar', - os.path.join(dataset_id, 'train'), - f'{count}band', - ) - -geojson = { - 'type': 'FeatureCollection', - 'crs': {'type': 'name', 'properties': {'name': 'urn:ogc:def:crs:OGC:1.3:CRS84'}}, - 'features': [ - { - 'type': 'Feature', - 'geometry': { - 'type': 'Polygon', - 'coordinates': [ - [ - [-43.7720361, -22.922229499999958, 0.0], - [-43.772064, -22.9222724, 0.0], - [-43.772102399999937, -22.922247399999947, 0.0], - [-43.772074499999974, -22.9222046, 0.0], - [-43.7720361, -22.922229499999958, 0.0], - ] - ], - }, - } - ], -} - -os.makedirs(os.path.join(dataset_id, 'train', 'geojson'), exist_ok=True) -for i in range(1, 4): - path = os.path.join(dataset_id, 'train', 'geojson', f'Geo_AOI_1_RIO_img{i}.geojson') - with open(path, 'w') as src: - if i % 2 == 0: - json.dump(geojson, src) - -shutil.make_archive( - os.path.join( - dataset_id, 'train', 'SN1_buildings_train_AOI_1_Rio_geojson_buildings' - ), - 'gztar', - os.path.join(dataset_id, 'train'), - 'geojson', -) diff --git a/tests/data/spacenet/SN1_buildings/train/3band/3band_AOI_1_RIO_img1.tif b/tests/data/spacenet/spacenet1/SN1_buildings/test/3band/3band_AOI_1_RIO_img1.tif similarity index 100% rename from tests/data/spacenet/SN1_buildings/train/3band/3band_AOI_1_RIO_img1.tif rename to tests/data/spacenet/spacenet1/SN1_buildings/test/3band/3band_AOI_1_RIO_img1.tif diff --git a/tests/data/spacenet/SN1_buildings/train/3band/3band_AOI_1_RIO_img2.tif b/tests/data/spacenet/spacenet1/SN1_buildings/test/3band/3band_AOI_1_RIO_img2.tif similarity index 100% rename from tests/data/spacenet/SN1_buildings/train/3band/3band_AOI_1_RIO_img2.tif rename to tests/data/spacenet/spacenet1/SN1_buildings/test/3band/3band_AOI_1_RIO_img2.tif diff --git a/tests/data/spacenet/SN1_buildings/train/3band/3band_AOI_1_RIO_img3.tif b/tests/data/spacenet/spacenet1/SN1_buildings/test/3band/3band_AOI_1_RIO_img3.tif similarity index 100% rename from tests/data/spacenet/SN1_buildings/train/3band/3band_AOI_1_RIO_img3.tif rename to tests/data/spacenet/spacenet1/SN1_buildings/test/3band/3band_AOI_1_RIO_img3.tif diff --git a/tests/data/spacenet/SN1_buildings/train/3band/3band_AOI_1_RIO_img4.tif b/tests/data/spacenet/spacenet1/SN1_buildings/test/3band/3band_AOI_1_RIO_img4.tif similarity index 100% rename from tests/data/spacenet/SN1_buildings/train/3band/3band_AOI_1_RIO_img4.tif rename to tests/data/spacenet/spacenet1/SN1_buildings/test/3band/3band_AOI_1_RIO_img4.tif diff --git a/tests/data/spacenet/SN1_buildings/train/8band/3band_AOI_1_RIO_img1.tif b/tests/data/spacenet/spacenet1/SN1_buildings/test/8band/8band_AOI_1_RIO_img1.tif similarity index 100% rename from tests/data/spacenet/SN1_buildings/train/8band/3band_AOI_1_RIO_img1.tif rename to tests/data/spacenet/spacenet1/SN1_buildings/test/8band/8band_AOI_1_RIO_img1.tif diff --git a/tests/data/spacenet/SN1_buildings/train/8band/3band_AOI_1_RIO_img2.tif b/tests/data/spacenet/spacenet1/SN1_buildings/test/8band/8band_AOI_1_RIO_img2.tif similarity index 100% rename from tests/data/spacenet/SN1_buildings/train/8band/3band_AOI_1_RIO_img2.tif rename to tests/data/spacenet/spacenet1/SN1_buildings/test/8band/8band_AOI_1_RIO_img2.tif diff --git a/tests/data/spacenet/SN1_buildings/train/8band/3band_AOI_1_RIO_img3.tif b/tests/data/spacenet/spacenet1/SN1_buildings/test/8band/8band_AOI_1_RIO_img3.tif similarity index 100% rename from tests/data/spacenet/SN1_buildings/train/8band/3band_AOI_1_RIO_img3.tif rename to tests/data/spacenet/spacenet1/SN1_buildings/test/8band/8band_AOI_1_RIO_img3.tif diff --git a/tests/data/spacenet/SN1_buildings/train/8band/3band_AOI_1_RIO_img4.tif b/tests/data/spacenet/spacenet1/SN1_buildings/test/8band/8band_AOI_1_RIO_img4.tif similarity index 100% rename from tests/data/spacenet/SN1_buildings/train/8band/3band_AOI_1_RIO_img4.tif rename to tests/data/spacenet/spacenet1/SN1_buildings/test/8band/8band_AOI_1_RIO_img4.tif diff --git a/tests/data/spacenet/spacenet1/SN1_buildings/test/SN1_buildings_test_AOI_1_Rio_3band.tar.gz b/tests/data/spacenet/spacenet1/SN1_buildings/test/SN1_buildings_test_AOI_1_Rio_3band.tar.gz new file mode 100644 index 0000000000..34d22b20d0 Binary files /dev/null and b/tests/data/spacenet/spacenet1/SN1_buildings/test/SN1_buildings_test_AOI_1_Rio_3band.tar.gz differ diff --git a/tests/data/spacenet/spacenet1/SN1_buildings/test/SN1_buildings_test_AOI_1_Rio_8band.tar.gz b/tests/data/spacenet/spacenet1/SN1_buildings/test/SN1_buildings_test_AOI_1_Rio_8band.tar.gz new file mode 100644 index 0000000000..a7da7248e5 Binary files /dev/null and b/tests/data/spacenet/spacenet1/SN1_buildings/test/SN1_buildings_test_AOI_1_Rio_8band.tar.gz differ diff --git a/tests/data/spacenet/spacenet1/SN1_buildings/train/3band/3band_AOI_1_RIO_img1.tif b/tests/data/spacenet/spacenet1/SN1_buildings/train/3band/3band_AOI_1_RIO_img1.tif new file mode 100644 index 0000000000..beab6f5fd8 Binary files /dev/null and b/tests/data/spacenet/spacenet1/SN1_buildings/train/3band/3band_AOI_1_RIO_img1.tif differ diff --git a/tests/data/spacenet/spacenet1/SN1_buildings/train/3band/3band_AOI_1_RIO_img2.tif b/tests/data/spacenet/spacenet1/SN1_buildings/train/3band/3band_AOI_1_RIO_img2.tif new file mode 100644 index 0000000000..beab6f5fd8 Binary files /dev/null and b/tests/data/spacenet/spacenet1/SN1_buildings/train/3band/3band_AOI_1_RIO_img2.tif differ diff --git a/tests/data/spacenet/spacenet1/SN1_buildings/train/3band/3band_AOI_1_RIO_img3.tif b/tests/data/spacenet/spacenet1/SN1_buildings/train/3band/3band_AOI_1_RIO_img3.tif new file mode 100644 index 0000000000..beab6f5fd8 Binary files /dev/null and b/tests/data/spacenet/spacenet1/SN1_buildings/train/3band/3band_AOI_1_RIO_img3.tif differ diff --git a/tests/data/spacenet/spacenet1/SN1_buildings/train/3band/3band_AOI_1_RIO_img4.tif b/tests/data/spacenet/spacenet1/SN1_buildings/train/3band/3band_AOI_1_RIO_img4.tif new file mode 100644 index 0000000000..beab6f5fd8 Binary files /dev/null and b/tests/data/spacenet/spacenet1/SN1_buildings/train/3band/3band_AOI_1_RIO_img4.tif differ diff --git a/tests/data/spacenet/spacenet1/SN1_buildings/train/8band/8band_AOI_1_RIO_img1.tif b/tests/data/spacenet/spacenet1/SN1_buildings/train/8band/8band_AOI_1_RIO_img1.tif new file mode 100644 index 0000000000..cfb2a40c34 Binary files /dev/null and b/tests/data/spacenet/spacenet1/SN1_buildings/train/8band/8band_AOI_1_RIO_img1.tif differ diff --git a/tests/data/spacenet/spacenet1/SN1_buildings/train/8band/8band_AOI_1_RIO_img2.tif b/tests/data/spacenet/spacenet1/SN1_buildings/train/8band/8band_AOI_1_RIO_img2.tif new file mode 100644 index 0000000000..cfb2a40c34 Binary files /dev/null and b/tests/data/spacenet/spacenet1/SN1_buildings/train/8band/8band_AOI_1_RIO_img2.tif differ diff --git a/tests/data/spacenet/spacenet1/SN1_buildings/train/8band/8band_AOI_1_RIO_img3.tif b/tests/data/spacenet/spacenet1/SN1_buildings/train/8band/8band_AOI_1_RIO_img3.tif new file mode 100644 index 0000000000..cfb2a40c34 Binary files /dev/null and b/tests/data/spacenet/spacenet1/SN1_buildings/train/8band/8band_AOI_1_RIO_img3.tif differ diff --git a/tests/data/spacenet/spacenet1/SN1_buildings/train/8band/8band_AOI_1_RIO_img4.tif b/tests/data/spacenet/spacenet1/SN1_buildings/train/8band/8band_AOI_1_RIO_img4.tif new file mode 100644 index 0000000000..cfb2a40c34 Binary files /dev/null and b/tests/data/spacenet/spacenet1/SN1_buildings/train/8band/8band_AOI_1_RIO_img4.tif differ diff --git a/tests/data/spacenet/spacenet1/SN1_buildings/train/SN1_buildings_train_AOI_1_Rio_3band.tar.gz b/tests/data/spacenet/spacenet1/SN1_buildings/train/SN1_buildings_train_AOI_1_Rio_3band.tar.gz new file mode 100644 index 0000000000..40ef873152 Binary files /dev/null and b/tests/data/spacenet/spacenet1/SN1_buildings/train/SN1_buildings_train_AOI_1_Rio_3band.tar.gz differ diff --git a/tests/data/spacenet/spacenet1/SN1_buildings/train/SN1_buildings_train_AOI_1_Rio_8band.tar.gz b/tests/data/spacenet/spacenet1/SN1_buildings/train/SN1_buildings_train_AOI_1_Rio_8band.tar.gz new file mode 100644 index 0000000000..6eaebc3a9a Binary files /dev/null and b/tests/data/spacenet/spacenet1/SN1_buildings/train/SN1_buildings_train_AOI_1_Rio_8band.tar.gz differ diff --git a/tests/data/spacenet/spacenet1/SN1_buildings/train/SN1_buildings_train_AOI_1_Rio_geojson_buildings.tar.gz b/tests/data/spacenet/spacenet1/SN1_buildings/train/SN1_buildings_train_AOI_1_Rio_geojson_buildings.tar.gz new file mode 100644 index 0000000000..ed4f60826e Binary files /dev/null and b/tests/data/spacenet/spacenet1/SN1_buildings/train/SN1_buildings_train_AOI_1_Rio_geojson_buildings.tar.gz differ diff --git a/tests/data/spacenet/SN1_buildings/train/geojson/Geo_AOI_1_RIO_img1.geojson b/tests/data/spacenet/spacenet1/SN1_buildings/train/geojson/Geo_AOI_1_RIO_img1.geojson similarity index 100% rename from tests/data/spacenet/SN1_buildings/train/geojson/Geo_AOI_1_RIO_img1.geojson rename to tests/data/spacenet/spacenet1/SN1_buildings/train/geojson/Geo_AOI_1_RIO_img1.geojson diff --git a/tests/data/spacenet/SN1_buildings/train/geojson/Geo_AOI_1_RIO_img2.geojson b/tests/data/spacenet/spacenet1/SN1_buildings/train/geojson/Geo_AOI_1_RIO_img2.geojson similarity index 100% rename from tests/data/spacenet/SN1_buildings/train/geojson/Geo_AOI_1_RIO_img2.geojson rename to tests/data/spacenet/spacenet1/SN1_buildings/train/geojson/Geo_AOI_1_RIO_img2.geojson diff --git a/tests/data/spacenet/SN1_buildings/train/geojson/Geo_AOI_1_RIO_img3.geojson b/tests/data/spacenet/spacenet1/SN1_buildings/train/geojson/Geo_AOI_1_RIO_img3.geojson similarity index 100% rename from tests/data/spacenet/SN1_buildings/train/geojson/Geo_AOI_1_RIO_img3.geojson rename to tests/data/spacenet/spacenet1/SN1_buildings/train/geojson/Geo_AOI_1_RIO_img3.geojson diff --git a/tests/data/spacenet/spacenet1/SN1_buildings/train/geojson/Geo_AOI_1_RIO_img4.geojson b/tests/data/spacenet/spacenet1/SN1_buildings/train/geojson/Geo_AOI_1_RIO_img4.geojson new file mode 100644 index 0000000000..dabcc4ce30 --- /dev/null +++ b/tests/data/spacenet/spacenet1/SN1_buildings/train/geojson/Geo_AOI_1_RIO_img4.geojson @@ -0,0 +1 @@ +{"type": "FeatureCollection", "crs": {"type": "name", "properties": {"name": "urn:ogc:def:crs:OGC:1.3:CRS84"}}, "features": [{"type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[[-43.7720361, -22.922229499999958, 0.0], [-43.772064, -22.9222724, 0.0], [-43.77210239999994, -22.922247399999947, 0.0], [-43.772074499999974, -22.9222046, 0.0], [-43.7720361, -22.922229499999958, 0.0]]]}}]} \ No newline at end of file diff --git a/tests/data/spacenet/spacenet1/data.py b/tests/data/spacenet/spacenet1/data.py new file mode 100755 index 0000000000..f8187583cc --- /dev/null +++ b/tests/data/spacenet/spacenet1/data.py @@ -0,0 +1,161 @@ +#!/usr/bin/env python3 + +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import hashlib +import json +import os +import shutil +from typing import Any + +import numpy as np +import rasterio +from rasterio.crs import CRS +from rasterio.transform import Affine + +SIZE = 2 + +NUM_SAMPLES = 4 + +dataset_id = 'SN1_buildings' + +profile = { + 'driver': 'GTiff', + 'dtype': 'uint8', + 'width': SIZE, + 'height': SIZE, + 'crs': CRS.from_epsg(4326), + 'transform': Affine( + 4.489235388119662e-06, + 0.0, + -43.7732462563, + 0.0, + -4.486127586210932e-06, + -22.9214851954, + ), +} + +np.random.seed(0) +Z = np.random.randint(np.iinfo('uint8').max, size=(SIZE, SIZE), dtype='uint8') + + +def create_directories(base_path: str, band_counts: list[int]) -> None: + for count in band_counts: + os.makedirs(os.path.join(base_path, f'{count}band'), exist_ok=True) + + +def generate_geotiff_files( + base_path: str, band_counts: list[int], profile: dict[str, Any], Z: np.ndarray +) -> None: + for count in band_counts: + for i in range(1, NUM_SAMPLES + 1): + path = os.path.join( + base_path, f'{count}band', f'{count}band_AOI_1_RIO_img{i}.tif' + ) + profile['count'] = count + with rasterio.open(path, 'w', **profile) as src: + for j in range(1, count + 1): + src.write(Z, j) + + +def generate_geojson_files(base_path: str, geojson: dict[str, Any]) -> None: + os.makedirs(os.path.join(base_path, 'geojson'), exist_ok=True) + for i in range(1, NUM_SAMPLES + 1): + path = os.path.join(base_path, 'geojson', f'Geo_AOI_1_RIO_img{i}.geojson') + with open(path, 'w') as src: + if i % 2 == 0: + json.dump(geojson, src) + + +def compute_md5(file_path: str) -> str: + hash_md5 = hashlib.md5() + with open(file_path, 'rb') as f: + for chunk in iter(lambda: f.read(4096), b''): + hash_md5.update(chunk) + return hash_md5.hexdigest() + + +# Generate dummy GeoJSON files for building footprints +geojson = { + 'type': 'FeatureCollection', + 'crs': {'type': 'name', 'properties': {'name': 'urn:ogc:def:crs:OGC:1.3:CRS84'}}, + 'features': [ + { + 'type': 'Feature', + 'geometry': { + 'type': 'Polygon', + 'coordinates': [ + [ + [-43.7720361, -22.922229499999958, 0.0], + [-43.772064, -22.9222724, 0.0], + [-43.772102399999937, -22.922247399999947, 0.0], + [-43.772074499999974, -22.9222046, 0.0], + [-43.7720361, -22.922229499999958, 0.0], + ] + ], + }, + } + ], +} + +# Remove existing data if it exists +if os.path.exists(dataset_id): + shutil.rmtree(dataset_id) + +train_base_path = os.path.join(dataset_id, 'train') +test_base_path = os.path.join(dataset_id, 'test') + +# Create directories and generate dummy GeoTIFF files for train dataset +create_directories(train_base_path, [3, 8]) +generate_geotiff_files(train_base_path, [3, 8], profile, Z) +generate_geojson_files(train_base_path, geojson) + +# Create directories and generate dummy GeoTIFF files for test dataset (only 3band and 8band) +create_directories(test_base_path, [3, 8]) +generate_geotiff_files(test_base_path, [3, 8], profile, Z) + +# Create tarballs for train and test datasets +tarball_specs = { + 'train': { + '3band': 'SN1_buildings_train_AOI_1_Rio_3band', + '8band': 'SN1_buildings_train_AOI_1_Rio_8band', + 'geojson': 'SN1_buildings_train_AOI_1_Rio_geojson_buildings', + }, + 'test': { + '3band': 'SN1_buildings_test_AOI_1_Rio_3band', + '8band': 'SN1_buildings_test_AOI_1_Rio_8band', + }, +} + +for split, specs in tarball_specs.items(): + for subdir, tarball_name in specs.items(): + tarball_path = os.path.join(dataset_id, split, tarball_name) + shutil.make_archive( + tarball_path, + 'gztar', + root_dir=os.path.join(dataset_id, split), + base_dir=subdir, + ) + +# Compute and print MD5 checksums for the generated tarballs +print('MD5 Checksums for Train Dataset:') +train_tarballs = [ + 'SN1_buildings_train_AOI_1_Rio_3band.tar.gz', + 'SN1_buildings_train_AOI_1_Rio_8band.tar.gz', + 'SN1_buildings_train_AOI_1_Rio_geojson_buildings.tar.gz', +] +for tarball in train_tarballs: + tarball_path = os.path.join(dataset_id, 'train', tarball) + if os.path.exists(tarball_path): + print(f'{tarball}: {compute_md5(tarball_path)}') + +print('\nMD5 Checksums for Test Dataset:') +test_tarballs = [ + 'SN1_buildings_test_AOI_1_Rio_3band.tar.gz', + 'SN1_buildings_test_AOI_1_Rio_8band.tar.gz', +] +for tarball in test_tarballs: + tarball_path = os.path.join(dataset_id, 'test', tarball) + if os.path.exists(tarball_path): + print(f'{tarball}: {compute_md5(tarball_path)}') diff --git a/tests/data/spacenet/spacenet6/SN6_buildings/test/SN6_buildings_AOI_11_Rotterdam_test.tar.gz b/tests/data/spacenet/spacenet6/SN6_buildings/test/SN6_buildings_AOI_11_Rotterdam_test.tar.gz new file mode 100644 index 0000000000..d787b796d9 Binary files /dev/null and b/tests/data/spacenet/spacenet6/SN6_buildings/test/SN6_buildings_AOI_11_Rotterdam_test.tar.gz differ diff --git a/tests/data/spacenet/spacenet6/SN6_buildings/test/test_public/AOI_11_Rotterdam/SAR-Intensity/SN6_Test_Public_AOI_11_Rotterdam_SAR-Intensity_20190804111224_20190804111453_tile_1.tif b/tests/data/spacenet/spacenet6/SN6_buildings/test/test_public/AOI_11_Rotterdam/SAR-Intensity/SN6_Test_Public_AOI_11_Rotterdam_SAR-Intensity_20190804111224_20190804111453_tile_1.tif new file mode 100644 index 0000000000..cce10b4555 Binary files /dev/null and b/tests/data/spacenet/spacenet6/SN6_buildings/test/test_public/AOI_11_Rotterdam/SAR-Intensity/SN6_Test_Public_AOI_11_Rotterdam_SAR-Intensity_20190804111224_20190804111453_tile_1.tif differ diff --git a/tests/data/spacenet/spacenet6/SN6_buildings/test/test_public/AOI_11_Rotterdam/SAR-Intensity/SN6_Test_Public_AOI_11_Rotterdam_SAR-Intensity_20190804111224_20190804111453_tile_2.tif b/tests/data/spacenet/spacenet6/SN6_buildings/test/test_public/AOI_11_Rotterdam/SAR-Intensity/SN6_Test_Public_AOI_11_Rotterdam_SAR-Intensity_20190804111224_20190804111453_tile_2.tif new file mode 100644 index 0000000000..cce10b4555 Binary files /dev/null and b/tests/data/spacenet/spacenet6/SN6_buildings/test/test_public/AOI_11_Rotterdam/SAR-Intensity/SN6_Test_Public_AOI_11_Rotterdam_SAR-Intensity_20190804111224_20190804111453_tile_2.tif differ diff --git a/tests/data/spacenet/spacenet6/SN6_buildings/test/test_public/AOI_11_Rotterdam/SAR-Intensity/SN6_Test_Public_AOI_11_Rotterdam_SAR-Intensity_20190804111224_20190804111453_tile_3.tif b/tests/data/spacenet/spacenet6/SN6_buildings/test/test_public/AOI_11_Rotterdam/SAR-Intensity/SN6_Test_Public_AOI_11_Rotterdam_SAR-Intensity_20190804111224_20190804111453_tile_3.tif new file mode 100644 index 0000000000..cce10b4555 Binary files /dev/null and b/tests/data/spacenet/spacenet6/SN6_buildings/test/test_public/AOI_11_Rotterdam/SAR-Intensity/SN6_Test_Public_AOI_11_Rotterdam_SAR-Intensity_20190804111224_20190804111453_tile_3.tif differ diff --git a/tests/data/spacenet/spacenet6/SN6_buildings/test/test_public/AOI_11_Rotterdam/SAR-Intensity/SN6_Test_Public_AOI_11_Rotterdam_SAR-Intensity_20190804111224_20190804111453_tile_4.tif b/tests/data/spacenet/spacenet6/SN6_buildings/test/test_public/AOI_11_Rotterdam/SAR-Intensity/SN6_Test_Public_AOI_11_Rotterdam_SAR-Intensity_20190804111224_20190804111453_tile_4.tif new file mode 100644 index 0000000000..cce10b4555 Binary files /dev/null and b/tests/data/spacenet/spacenet6/SN6_buildings/test/test_public/AOI_11_Rotterdam/SAR-Intensity/SN6_Test_Public_AOI_11_Rotterdam_SAR-Intensity_20190804111224_20190804111453_tile_4.tif differ diff --git a/tests/data/spacenet/spacenet6/SN6_buildings/train/SN6_buildings_AOI_11_Rotterdam_train.tar.gz b/tests/data/spacenet/spacenet6/SN6_buildings/train/SN6_buildings_AOI_11_Rotterdam_train.tar.gz new file mode 100644 index 0000000000..a863cfbe02 Binary files /dev/null and b/tests/data/spacenet/spacenet6/SN6_buildings/train/SN6_buildings_AOI_11_Rotterdam_train.tar.gz differ diff --git a/tests/data/spacenet/spacenet6/SN6_buildings/train/train/AOI_11_Rotterdam/PAN/SN6_Train_AOI_11_Rotterdam_PAN_20190804111224_20190804111453_tile_1.tif b/tests/data/spacenet/spacenet6/SN6_buildings/train/train/AOI_11_Rotterdam/PAN/SN6_Train_AOI_11_Rotterdam_PAN_20190804111224_20190804111453_tile_1.tif new file mode 100644 index 0000000000..650664c080 Binary files /dev/null and b/tests/data/spacenet/spacenet6/SN6_buildings/train/train/AOI_11_Rotterdam/PAN/SN6_Train_AOI_11_Rotterdam_PAN_20190804111224_20190804111453_tile_1.tif differ diff --git a/tests/data/spacenet/spacenet6/SN6_buildings/train/train/AOI_11_Rotterdam/PAN/SN6_Train_AOI_11_Rotterdam_PAN_20190804111224_20190804111453_tile_2.tif b/tests/data/spacenet/spacenet6/SN6_buildings/train/train/AOI_11_Rotterdam/PAN/SN6_Train_AOI_11_Rotterdam_PAN_20190804111224_20190804111453_tile_2.tif new file mode 100644 index 0000000000..650664c080 Binary files /dev/null and b/tests/data/spacenet/spacenet6/SN6_buildings/train/train/AOI_11_Rotterdam/PAN/SN6_Train_AOI_11_Rotterdam_PAN_20190804111224_20190804111453_tile_2.tif differ diff --git a/tests/data/spacenet/spacenet6/SN6_buildings/train/train/AOI_11_Rotterdam/PAN/SN6_Train_AOI_11_Rotterdam_PAN_20190804111224_20190804111453_tile_3.tif b/tests/data/spacenet/spacenet6/SN6_buildings/train/train/AOI_11_Rotterdam/PAN/SN6_Train_AOI_11_Rotterdam_PAN_20190804111224_20190804111453_tile_3.tif new file mode 100644 index 0000000000..650664c080 Binary files /dev/null and b/tests/data/spacenet/spacenet6/SN6_buildings/train/train/AOI_11_Rotterdam/PAN/SN6_Train_AOI_11_Rotterdam_PAN_20190804111224_20190804111453_tile_3.tif differ diff --git a/tests/data/spacenet/spacenet6/SN6_buildings/train/train/AOI_11_Rotterdam/PAN/SN6_Train_AOI_11_Rotterdam_PAN_20190804111224_20190804111453_tile_4.tif b/tests/data/spacenet/spacenet6/SN6_buildings/train/train/AOI_11_Rotterdam/PAN/SN6_Train_AOI_11_Rotterdam_PAN_20190804111224_20190804111453_tile_4.tif new file mode 100644 index 0000000000..650664c080 Binary files /dev/null and b/tests/data/spacenet/spacenet6/SN6_buildings/train/train/AOI_11_Rotterdam/PAN/SN6_Train_AOI_11_Rotterdam_PAN_20190804111224_20190804111453_tile_4.tif differ diff --git a/tests/data/spacenet/spacenet6/SN6_buildings/train/train/AOI_11_Rotterdam/PS-RGB/SN6_Train_AOI_11_Rotterdam_PS-RGB_20190804111224_20190804111453_tile_1.tif b/tests/data/spacenet/spacenet6/SN6_buildings/train/train/AOI_11_Rotterdam/PS-RGB/SN6_Train_AOI_11_Rotterdam_PS-RGB_20190804111224_20190804111453_tile_1.tif new file mode 100644 index 0000000000..96950caa28 Binary files /dev/null and b/tests/data/spacenet/spacenet6/SN6_buildings/train/train/AOI_11_Rotterdam/PS-RGB/SN6_Train_AOI_11_Rotterdam_PS-RGB_20190804111224_20190804111453_tile_1.tif differ diff --git a/tests/data/spacenet/spacenet6/SN6_buildings/train/train/AOI_11_Rotterdam/PS-RGB/SN6_Train_AOI_11_Rotterdam_PS-RGB_20190804111224_20190804111453_tile_2.tif b/tests/data/spacenet/spacenet6/SN6_buildings/train/train/AOI_11_Rotterdam/PS-RGB/SN6_Train_AOI_11_Rotterdam_PS-RGB_20190804111224_20190804111453_tile_2.tif new file mode 100644 index 0000000000..96950caa28 Binary files /dev/null and b/tests/data/spacenet/spacenet6/SN6_buildings/train/train/AOI_11_Rotterdam/PS-RGB/SN6_Train_AOI_11_Rotterdam_PS-RGB_20190804111224_20190804111453_tile_2.tif differ diff --git a/tests/data/spacenet/spacenet6/SN6_buildings/train/train/AOI_11_Rotterdam/PS-RGB/SN6_Train_AOI_11_Rotterdam_PS-RGB_20190804111224_20190804111453_tile_3.tif b/tests/data/spacenet/spacenet6/SN6_buildings/train/train/AOI_11_Rotterdam/PS-RGB/SN6_Train_AOI_11_Rotterdam_PS-RGB_20190804111224_20190804111453_tile_3.tif new file mode 100644 index 0000000000..96950caa28 Binary files /dev/null and b/tests/data/spacenet/spacenet6/SN6_buildings/train/train/AOI_11_Rotterdam/PS-RGB/SN6_Train_AOI_11_Rotterdam_PS-RGB_20190804111224_20190804111453_tile_3.tif differ diff --git a/tests/data/spacenet/spacenet6/SN6_buildings/train/train/AOI_11_Rotterdam/PS-RGB/SN6_Train_AOI_11_Rotterdam_PS-RGB_20190804111224_20190804111453_tile_4.tif b/tests/data/spacenet/spacenet6/SN6_buildings/train/train/AOI_11_Rotterdam/PS-RGB/SN6_Train_AOI_11_Rotterdam_PS-RGB_20190804111224_20190804111453_tile_4.tif new file mode 100644 index 0000000000..96950caa28 Binary files /dev/null and b/tests/data/spacenet/spacenet6/SN6_buildings/train/train/AOI_11_Rotterdam/PS-RGB/SN6_Train_AOI_11_Rotterdam_PS-RGB_20190804111224_20190804111453_tile_4.tif differ diff --git a/tests/data/spacenet/spacenet6/SN6_buildings/train/train/AOI_11_Rotterdam/PS-RGBNIR/SN6_Train_AOI_11_Rotterdam_PS-RGBNIR_20190804111224_20190804111453_tile_1.tif b/tests/data/spacenet/spacenet6/SN6_buildings/train/train/AOI_11_Rotterdam/PS-RGBNIR/SN6_Train_AOI_11_Rotterdam_PS-RGBNIR_20190804111224_20190804111453_tile_1.tif new file mode 100644 index 0000000000..cce10b4555 Binary files /dev/null and b/tests/data/spacenet/spacenet6/SN6_buildings/train/train/AOI_11_Rotterdam/PS-RGBNIR/SN6_Train_AOI_11_Rotterdam_PS-RGBNIR_20190804111224_20190804111453_tile_1.tif differ diff --git a/tests/data/spacenet/spacenet6/SN6_buildings/train/train/AOI_11_Rotterdam/PS-RGBNIR/SN6_Train_AOI_11_Rotterdam_PS-RGBNIR_20190804111224_20190804111453_tile_2.tif b/tests/data/spacenet/spacenet6/SN6_buildings/train/train/AOI_11_Rotterdam/PS-RGBNIR/SN6_Train_AOI_11_Rotterdam_PS-RGBNIR_20190804111224_20190804111453_tile_2.tif new file mode 100644 index 0000000000..cce10b4555 Binary files /dev/null and b/tests/data/spacenet/spacenet6/SN6_buildings/train/train/AOI_11_Rotterdam/PS-RGBNIR/SN6_Train_AOI_11_Rotterdam_PS-RGBNIR_20190804111224_20190804111453_tile_2.tif differ diff --git a/tests/data/spacenet/spacenet6/SN6_buildings/train/train/AOI_11_Rotterdam/PS-RGBNIR/SN6_Train_AOI_11_Rotterdam_PS-RGBNIR_20190804111224_20190804111453_tile_3.tif b/tests/data/spacenet/spacenet6/SN6_buildings/train/train/AOI_11_Rotterdam/PS-RGBNIR/SN6_Train_AOI_11_Rotterdam_PS-RGBNIR_20190804111224_20190804111453_tile_3.tif new file mode 100644 index 0000000000..cce10b4555 Binary files /dev/null and b/tests/data/spacenet/spacenet6/SN6_buildings/train/train/AOI_11_Rotterdam/PS-RGBNIR/SN6_Train_AOI_11_Rotterdam_PS-RGBNIR_20190804111224_20190804111453_tile_3.tif differ diff --git a/tests/data/spacenet/spacenet6/SN6_buildings/train/train/AOI_11_Rotterdam/PS-RGBNIR/SN6_Train_AOI_11_Rotterdam_PS-RGBNIR_20190804111224_20190804111453_tile_4.tif b/tests/data/spacenet/spacenet6/SN6_buildings/train/train/AOI_11_Rotterdam/PS-RGBNIR/SN6_Train_AOI_11_Rotterdam_PS-RGBNIR_20190804111224_20190804111453_tile_4.tif new file mode 100644 index 0000000000..cce10b4555 Binary files /dev/null and b/tests/data/spacenet/spacenet6/SN6_buildings/train/train/AOI_11_Rotterdam/PS-RGBNIR/SN6_Train_AOI_11_Rotterdam_PS-RGBNIR_20190804111224_20190804111453_tile_4.tif differ diff --git a/tests/data/spacenet/spacenet6/SN6_buildings/train/train/AOI_11_Rotterdam/RGBNIR/SN6_Train_AOI_11_Rotterdam_RGBNIR_20190804111224_20190804111453_tile_1.tif b/tests/data/spacenet/spacenet6/SN6_buildings/train/train/AOI_11_Rotterdam/RGBNIR/SN6_Train_AOI_11_Rotterdam_RGBNIR_20190804111224_20190804111453_tile_1.tif new file mode 100644 index 0000000000..cce10b4555 Binary files /dev/null and b/tests/data/spacenet/spacenet6/SN6_buildings/train/train/AOI_11_Rotterdam/RGBNIR/SN6_Train_AOI_11_Rotterdam_RGBNIR_20190804111224_20190804111453_tile_1.tif differ diff --git a/tests/data/spacenet/spacenet6/SN6_buildings/train/train/AOI_11_Rotterdam/RGBNIR/SN6_Train_AOI_11_Rotterdam_RGBNIR_20190804111224_20190804111453_tile_2.tif b/tests/data/spacenet/spacenet6/SN6_buildings/train/train/AOI_11_Rotterdam/RGBNIR/SN6_Train_AOI_11_Rotterdam_RGBNIR_20190804111224_20190804111453_tile_2.tif new file mode 100644 index 0000000000..cce10b4555 Binary files /dev/null and b/tests/data/spacenet/spacenet6/SN6_buildings/train/train/AOI_11_Rotterdam/RGBNIR/SN6_Train_AOI_11_Rotterdam_RGBNIR_20190804111224_20190804111453_tile_2.tif differ diff --git a/tests/data/spacenet/spacenet6/SN6_buildings/train/train/AOI_11_Rotterdam/RGBNIR/SN6_Train_AOI_11_Rotterdam_RGBNIR_20190804111224_20190804111453_tile_3.tif b/tests/data/spacenet/spacenet6/SN6_buildings/train/train/AOI_11_Rotterdam/RGBNIR/SN6_Train_AOI_11_Rotterdam_RGBNIR_20190804111224_20190804111453_tile_3.tif new file mode 100644 index 0000000000..cce10b4555 Binary files /dev/null and b/tests/data/spacenet/spacenet6/SN6_buildings/train/train/AOI_11_Rotterdam/RGBNIR/SN6_Train_AOI_11_Rotterdam_RGBNIR_20190804111224_20190804111453_tile_3.tif differ diff --git a/tests/data/spacenet/spacenet6/SN6_buildings/train/train/AOI_11_Rotterdam/RGBNIR/SN6_Train_AOI_11_Rotterdam_RGBNIR_20190804111224_20190804111453_tile_4.tif b/tests/data/spacenet/spacenet6/SN6_buildings/train/train/AOI_11_Rotterdam/RGBNIR/SN6_Train_AOI_11_Rotterdam_RGBNIR_20190804111224_20190804111453_tile_4.tif new file mode 100644 index 0000000000..cce10b4555 Binary files /dev/null and b/tests/data/spacenet/spacenet6/SN6_buildings/train/train/AOI_11_Rotterdam/RGBNIR/SN6_Train_AOI_11_Rotterdam_RGBNIR_20190804111224_20190804111453_tile_4.tif differ diff --git a/tests/data/spacenet/spacenet6/SN6_buildings/train/train/AOI_11_Rotterdam/SAR-Intensity/SN6_Train_AOI_11_Rotterdam_SAR-Intensity_20190804111224_20190804111453_tile_1.tif b/tests/data/spacenet/spacenet6/SN6_buildings/train/train/AOI_11_Rotterdam/SAR-Intensity/SN6_Train_AOI_11_Rotterdam_SAR-Intensity_20190804111224_20190804111453_tile_1.tif new file mode 100644 index 0000000000..cce10b4555 Binary files /dev/null and b/tests/data/spacenet/spacenet6/SN6_buildings/train/train/AOI_11_Rotterdam/SAR-Intensity/SN6_Train_AOI_11_Rotterdam_SAR-Intensity_20190804111224_20190804111453_tile_1.tif differ diff --git a/tests/data/spacenet/spacenet6/SN6_buildings/train/train/AOI_11_Rotterdam/SAR-Intensity/SN6_Train_AOI_11_Rotterdam_SAR-Intensity_20190804111224_20190804111453_tile_2.tif b/tests/data/spacenet/spacenet6/SN6_buildings/train/train/AOI_11_Rotterdam/SAR-Intensity/SN6_Train_AOI_11_Rotterdam_SAR-Intensity_20190804111224_20190804111453_tile_2.tif new file mode 100644 index 0000000000..cce10b4555 Binary files /dev/null and b/tests/data/spacenet/spacenet6/SN6_buildings/train/train/AOI_11_Rotterdam/SAR-Intensity/SN6_Train_AOI_11_Rotterdam_SAR-Intensity_20190804111224_20190804111453_tile_2.tif differ diff --git a/tests/data/spacenet/spacenet6/SN6_buildings/train/train/AOI_11_Rotterdam/SAR-Intensity/SN6_Train_AOI_11_Rotterdam_SAR-Intensity_20190804111224_20190804111453_tile_3.tif b/tests/data/spacenet/spacenet6/SN6_buildings/train/train/AOI_11_Rotterdam/SAR-Intensity/SN6_Train_AOI_11_Rotterdam_SAR-Intensity_20190804111224_20190804111453_tile_3.tif new file mode 100644 index 0000000000..cce10b4555 Binary files /dev/null and b/tests/data/spacenet/spacenet6/SN6_buildings/train/train/AOI_11_Rotterdam/SAR-Intensity/SN6_Train_AOI_11_Rotterdam_SAR-Intensity_20190804111224_20190804111453_tile_3.tif differ diff --git a/tests/data/spacenet/spacenet6/SN6_buildings/train/train/AOI_11_Rotterdam/SAR-Intensity/SN6_Train_AOI_11_Rotterdam_SAR-Intensity_20190804111224_20190804111453_tile_4.tif b/tests/data/spacenet/spacenet6/SN6_buildings/train/train/AOI_11_Rotterdam/SAR-Intensity/SN6_Train_AOI_11_Rotterdam_SAR-Intensity_20190804111224_20190804111453_tile_4.tif new file mode 100644 index 0000000000..cce10b4555 Binary files /dev/null and b/tests/data/spacenet/spacenet6/SN6_buildings/train/train/AOI_11_Rotterdam/SAR-Intensity/SN6_Train_AOI_11_Rotterdam_SAR-Intensity_20190804111224_20190804111453_tile_4.tif differ diff --git a/tests/data/spacenet/spacenet6/SN6_buildings/train/train/AOI_11_Rotterdam/geojson_buildings/SN6_Train_AOI_11_Rotterdam_Buildings_20190804111224_20190804111453_tile_1.geojson b/tests/data/spacenet/spacenet6/SN6_buildings/train/train/AOI_11_Rotterdam/geojson_buildings/SN6_Train_AOI_11_Rotterdam_Buildings_20190804111224_20190804111453_tile_1.geojson new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/data/spacenet/spacenet6/SN6_buildings/train/train/AOI_11_Rotterdam/geojson_buildings/SN6_Train_AOI_11_Rotterdam_Buildings_20190804111224_20190804111453_tile_2.geojson b/tests/data/spacenet/spacenet6/SN6_buildings/train/train/AOI_11_Rotterdam/geojson_buildings/SN6_Train_AOI_11_Rotterdam_Buildings_20190804111224_20190804111453_tile_2.geojson new file mode 100644 index 0000000000..7ae5755f9f --- /dev/null +++ b/tests/data/spacenet/spacenet6/SN6_buildings/train/train/AOI_11_Rotterdam/geojson_buildings/SN6_Train_AOI_11_Rotterdam_Buildings_20190804111224_20190804111453_tile_2.geojson @@ -0,0 +1 @@ +{"type": "FeatureCollection", "crs": {"type": "name", "properties": {"name": "urn:ogc:def:crs:OGC:1.3:CRS84"}}, "features": [{"type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[[4.47917, 51.9225, 0.0], [4.4792, 51.92255, 0.0], [4.47925, 51.92252, 0.0], [4.47922, 51.92247, 0.0], [4.47917, 51.9225, 0.0]]]}}]} \ No newline at end of file diff --git a/tests/data/spacenet/spacenet6/SN6_buildings/train/train/AOI_11_Rotterdam/geojson_buildings/SN6_Train_AOI_11_Rotterdam_Buildings_20190804111224_20190804111453_tile_3.geojson b/tests/data/spacenet/spacenet6/SN6_buildings/train/train/AOI_11_Rotterdam/geojson_buildings/SN6_Train_AOI_11_Rotterdam_Buildings_20190804111224_20190804111453_tile_3.geojson new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/data/spacenet/spacenet6/SN6_buildings/train/train/AOI_11_Rotterdam/geojson_buildings/SN6_Train_AOI_11_Rotterdam_Buildings_20190804111224_20190804111453_tile_4.geojson b/tests/data/spacenet/spacenet6/SN6_buildings/train/train/AOI_11_Rotterdam/geojson_buildings/SN6_Train_AOI_11_Rotterdam_Buildings_20190804111224_20190804111453_tile_4.geojson new file mode 100644 index 0000000000..7ae5755f9f --- /dev/null +++ b/tests/data/spacenet/spacenet6/SN6_buildings/train/train/AOI_11_Rotterdam/geojson_buildings/SN6_Train_AOI_11_Rotterdam_Buildings_20190804111224_20190804111453_tile_4.geojson @@ -0,0 +1 @@ +{"type": "FeatureCollection", "crs": {"type": "name", "properties": {"name": "urn:ogc:def:crs:OGC:1.3:CRS84"}}, "features": [{"type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[[4.47917, 51.9225, 0.0], [4.4792, 51.92255, 0.0], [4.47925, 51.92252, 0.0], [4.47922, 51.92247, 0.0], [4.47917, 51.9225, 0.0]]]}}]} \ No newline at end of file diff --git a/tests/data/spacenet/spacenet6/data.py b/tests/data/spacenet/spacenet6/data.py new file mode 100644 index 0000000000..47f2b59510 --- /dev/null +++ b/tests/data/spacenet/spacenet6/data.py @@ -0,0 +1,171 @@ +#!/usr/bin/env python3 + +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import hashlib +import json +import os +import shutil +from typing import Any + +import numpy as np +import rasterio +from rasterio.crs import CRS +from rasterio.transform import Affine + +SIZE = 2 + +NUM_SAMPLES = 4 + +dataset_id = 'SN6_buildings' + +profile = { + 'driver': 'GTiff', + 'dtype': 'uint8', + 'width': SIZE, + 'height': SIZE, + 'crs': CRS.from_epsg(4326), + 'transform': Affine( + 4.489235388119662e-06, 0.0, 4.47917, 0.0, -4.486127586210932e-06, 51.9225 + ), +} + +np.random.seed(0) +Z = np.random.randint(np.iinfo('uint8').max, size=(SIZE, SIZE), dtype='uint8') + +# Define the types of imagery for SpaceNet6 +imagery_types = ['PAN', 'PS-RGB', 'PS-RGBNIR', 'RGBNIR', 'SAR-Intensity'] +imagery_channels = { + 'PAN': 1, + 'PS-RGB': 3, + 'PS-RGBNIR': 4, + 'RGBNIR': 4, + 'SAR-Intensity': 4, +} + + +def create_directories(base_path: str, imagery_types: list[str]) -> None: + for imagery_type in imagery_types: + os.makedirs(os.path.join(base_path, imagery_type), exist_ok=True) + + +def generate_geotiff_files( + base_path: str, + imagery_types: str, + imagery_channels: int, + profile: dict[str, Any], + Z: np.ndarray, + test: bool = False, +) -> None: + for imagery_type in imagery_types: + for i in range(1, NUM_SAMPLES + 1): + if test and imagery_type == 'SAR-Intensity': + path = os.path.join( + base_path, + f'SN6_Test_Public_AOI_11_Rotterdam_SAR-Intensity_20190804111224_20190804111453_tile_{i}.tif', + ) + else: + path = os.path.join( + base_path, + imagery_type, + f'SN6_Train_AOI_11_Rotterdam_{imagery_type}_20190804111224_20190804111453_tile_{i}.tif', + ) + profile['count'] = imagery_channels[imagery_type] + with rasterio.open(path, 'w', **profile) as src: + for j in range(1, profile['count'] + 1): + src.write(Z, j) + + +def generate_geojson_files(base_path: str, geojson: dict[str, Any]) -> None: + os.makedirs(os.path.join(base_path, 'geojson_buildings'), exist_ok=True) + for i in range(1, NUM_SAMPLES + 1): + path = os.path.join( + base_path, + 'geojson_buildings', + f'SN6_Train_AOI_11_Rotterdam_Buildings_20190804111224_20190804111453_tile_{i}.geojson', + ) + with open(path, 'w') as src: + if i % 2 == 0: + json.dump(geojson, src) + + +def compute_md5(file_path: str) -> str: + hash_md5 = hashlib.md5() + with open(file_path, 'rb') as f: + for chunk in iter(lambda: f.read(4096), b''): + hash_md5.update(chunk) + return hash_md5.hexdigest() + + +# Generate dummy GeoJSON files for building footprints +geojson = { + 'type': 'FeatureCollection', + 'crs': {'type': 'name', 'properties': {'name': 'urn:ogc:def:crs:OGC:1.3:CRS84'}}, + 'features': [ + { + 'type': 'Feature', + 'geometry': { + 'type': 'Polygon', + 'coordinates': [ + [ + [4.47917, 51.9225, 0.0], + [4.47920, 51.92255, 0.0], + [4.47925, 51.92252, 0.0], + [4.47922, 51.92247, 0.0], + [4.47917, 51.9225, 0.0], + ] + ], + }, + } + ], +} + +# Remove existing data if it exists +if os.path.exists(dataset_id): + shutil.rmtree(dataset_id) + +train_base_path = os.path.join(dataset_id, 'train/train/AOI_11_Rotterdam') +test_base_path = os.path.join( + dataset_id, 'test/test_public/AOI_11_Rotterdam/SAR-Intensity' +) + +# Create directories and generate dummy GeoTIFF files for train dataset +create_directories(train_base_path, imagery_types) +generate_geotiff_files(train_base_path, imagery_types, imagery_channels, profile, Z) +generate_geojson_files(train_base_path, geojson) + +# Create directories and generate dummy GeoTIFF files for test dataset (only SAR-Intensity) +os.makedirs(test_base_path, exist_ok=True) +generate_geotiff_files( + test_base_path, ['SAR-Intensity'], imagery_channels, profile, Z, test=True +) + +# Create tarballs for train and test datasets +shutil.make_archive( + os.path.join(dataset_id, 'train', 'SN6_buildings_AOI_11_Rotterdam_train'), + 'gztar', + root_dir=os.path.join(dataset_id, 'train'), + base_dir='train', +) +shutil.make_archive( + os.path.join(dataset_id, 'test', 'SN6_buildings_AOI_11_Rotterdam_test'), + 'gztar', + root_dir=os.path.join(dataset_id, 'test'), + base_dir='test_public', +) + +# Compute and print MD5 checksums for the generated tarballs +print('MD5 Checksums for Train Dataset:') +train_tarball_path = os.path.join( + dataset_id, 'train', 'SN6_buildings_AOI_11_Rotterdam_train.tar.gz' +) +if os.path.exists(train_tarball_path): + print(f'Train: {compute_md5(train_tarball_path)}') + +print('\nMD5 Checksums for Test Dataset:') +test_tarball_path = os.path.join( + dataset_id, 'test', 'SN6_buildings_AOI_11_Rotterdam_test.tar.gz' +) +if os.path.exists(test_tarball_path): + print(f'Test: {compute_md5(test_tarball_path)}') diff --git a/tests/datasets/test_spacenet.py b/tests/datasets/test_spacenet.py index 4385e7eee0..36d0d57c24 100644 --- a/tests/datasets/test_spacenet.py +++ b/tests/datasets/test_spacenet.py @@ -9,48 +9,70 @@ import pytest import torch import torch.nn as nn +from _pytest.fixtures import SubRequest from pytest import MonkeyPatch -from torchgeo.datasets import DatasetNotFoundError, SpaceNet1 +from torchgeo.datasets import DatasetNotFoundError, SpaceNet, SpaceNet1, SpaceNet6 from torchgeo.datasets.utils import Executable class TestSpaceNet: - @pytest.fixture + @pytest.fixture(params=[SpaceNet1, SpaceNet6]) def dataset( - self, aws: Executable, monkeypatch: MonkeyPatch, tmp_path: Path - ) -> SpaceNet1: + self, + request: SubRequest, + aws: Executable, + monkeypatch: MonkeyPatch, + tmp_path: Path, + ) -> SpaceNet: + dataset_class: type[SpaceNet] = request.param url = os.path.join( - 'tests', 'data', 'spacenet', '{dataset_id}', 'train', '{tarball}' + 'tests', + 'data', + 'spacenet', + dataset_class.__name__.lower(), + '{dataset_id}', + 'train', + '{tarball}', ) - monkeypatch.setattr(SpaceNet1, 'url', url) + monkeypatch.setattr(dataset_class, 'url', url) transforms = nn.Identity() - return SpaceNet1(tmp_path, transforms=transforms, download=True) + return dataset_class(tmp_path, transforms=transforms, download=True) @pytest.mark.parametrize('index', [0, 1]) - def test_getitem(self, dataset: SpaceNet1, index: int) -> None: + def test_getitem(self, dataset: SpaceNet, index: int) -> None: x = dataset[index] assert isinstance(x, dict) assert isinstance(x['image'], torch.Tensor) assert isinstance(x['mask'], torch.Tensor) - def test_len(self, dataset: SpaceNet1) -> None: - assert len(dataset) == 3 + def test_len(self, dataset: SpaceNet) -> None: + assert len(dataset) == 4 - def test_already_extracted(self, dataset: SpaceNet1) -> None: - SpaceNet1(root=dataset.root) + def test_already_extracted(self, dataset: SpaceNet) -> None: + dataset.__class__(root=dataset.root) - def test_already_downloaded(self, dataset: SpaceNet1) -> None: - for product in ['3band', '8band', 'geojson']: - dir = os.path.join(dataset.root, dataset.dataset_id, dataset.split, product) + def test_already_downloaded(self, dataset: SpaceNet) -> None: + if dataset.dataset_id == 'SN1_buildings': + base_dir = os.path.join(dataset.root, dataset.dataset_id, dataset.split) + elif dataset.dataset_id == 'SN6_buildings': + base_dir = os.path.join( + dataset.root, + dataset.dataset_id, + dataset.split, + dataset.split, + 'AOI_11_Rotterdam', + ) + for product in dataset.valid_images['train'] + list(dataset.valid_masks): + dir = os.path.join(base_dir, product) shutil.rmtree(dir) - SpaceNet1(root=dataset.root) + dataset.__class__(root=dataset.root) - def test_not_downloaded(self, tmp_path: Path) -> None: + def test_not_downloaded(self, tmp_path: Path, dataset: SpaceNet) -> None: with pytest.raises(DatasetNotFoundError, match='Dataset not found'): - SpaceNet1(tmp_path) + dataset.__class__(root=os.path.join(tmp_path, 'dummy')) - def test_plot(self, dataset: SpaceNet1) -> None: + def test_plot(self, dataset: SpaceNet) -> None: x = dataset[0] dataset.plot(x, show_titles=False) plt.close() @@ -58,12 +80,12 @@ def test_plot(self, dataset: SpaceNet1) -> None: dataset.plot(x, suptitle='Test') plt.close() - def test_image_id(self, monkeypatch: MonkeyPatch, dataset: SpaceNet1) -> None: + def test_image_id(self, monkeypatch: MonkeyPatch, dataset: SpaceNet) -> None: file_regex = r'global_monthly_(\d+.*\d+)' monkeypatch.setattr(dataset, 'file_regex', file_regex) dataset._image_id('global_monthly_2018_01_mosaic_L15-0331E-1257N_1327_3160.tif') - def test_list_files(self, monkeypatch: MonkeyPatch, dataset: SpaceNet1) -> None: + def test_list_files(self, monkeypatch: MonkeyPatch, dataset: SpaceNet) -> None: directory_glob = os.path.join('**', 'AOI_{aoi}_*', '{product}') monkeypatch.setattr(dataset, 'directory_glob', directory_glob) dataset._list_files(aoi=1) diff --git a/tests/trainers/test_segmentation.py b/tests/trainers/test_segmentation.py index 8c33fa702f..4bdd966a1b 100644 --- a/tests/trainers/test_segmentation.py +++ b/tests/trainers/test_segmentation.py @@ -77,6 +77,7 @@ class TestSemanticSegmentationTask: 'sentinel2_south_america_soybean', 'southafricacroptype', 'spacenet1', + 'spacenet6', 'ssl4eo_l_benchmark_cdl', 'ssl4eo_l_benchmark_nlcd', 'vaihingen2d', diff --git a/torchgeo/datamodules/__init__.py b/torchgeo/datamodules/__init__.py index 3bbb5153b4..ae4f029c93 100644 --- a/torchgeo/datamodules/__init__.py +++ b/torchgeo/datamodules/__init__.py @@ -43,7 +43,7 @@ from .skippd import SKIPPDDataModule from .so2sat import So2SatDataModule from .southafricacroptype import SouthAfricaCropTypeDataModule -from .spacenet import SpaceNet1DataModule +from .spacenet import SpaceNet1DataModule, SpaceNet6DataModule, SpaceNetBaseDataModule from .ssl4eo import SSL4EOLDataModule, SSL4EOS12DataModule from .ssl4eo_benchmark import SSL4EOLBenchmarkDataModule from .sustainbench_crop_yield import SustainBenchCropYieldDataModule @@ -99,7 +99,9 @@ 'SEN12MSDataModule', 'SKIPPDDataModule', 'So2SatDataModule', + 'SpaceNetBaseDataModule', 'SpaceNet1DataModule', + 'SpaceNet6DataModule', 'SSL4EOLBenchmarkDataModule', 'SSL4EOLDataModule', 'SSL4EOS12DataModule', diff --git a/torchgeo/datamodules/spacenet.py b/torchgeo/datamodules/spacenet.py index dad903bd71..e6b0b32582 100644 --- a/torchgeo/datamodules/spacenet.py +++ b/torchgeo/datamodules/spacenet.py @@ -10,57 +10,44 @@ from torch import Tensor from torch.utils.data import random_split -from ..datasets import SpaceNet1 +from ..datasets import SpaceNet, SpaceNet1, SpaceNet6 from ..transforms import AugmentationSequential from .geo import NonGeoDataModule -class SpaceNet1DataModule(NonGeoDataModule): - """LightningDataModule implementation for the SpaceNet1 dataset. +class SpaceNetBaseDataModule(NonGeoDataModule): + """LightningDataModule implementation for the SpaceNet datasets. - Randomly splits into train/val/test. + Randomly splits the train split into train/val/test. The test split does not have labels, + and is only used for prediction. - .. versionadded:: 0.4 + .. versionadded:: 0.7 """ def __init__( self, + spacenet_ds_class: type[SpaceNet], batch_size: int = 64, num_workers: int = 0, val_split_pct: float = 0.1, test_split_pct: float = 0.2, **kwargs: Any, ) -> None: - """Initialize a new SpaceNet1DataModule instance. + """Initialize a new SpaceNetBaseDataModule instance. Args: + spacenet_ds_class: The SpaceNet dataset class to use. batch_size: Size of each mini-batch. - num_workers: Number of workers for parallel data loading. val_split_pct: Percentage of the dataset to use as a validation set. test_split_pct: Percentage of the dataset to use as a test set. - **kwargs: Additional keyword arguments passed to - :class:`~torchgeo.datasets.SpaceNet1`. + num_workers: Number of workers for parallel data loading. + **kwargs: Additional keyword arguments passed to the SpaceNet dataset. """ - super().__init__(SpaceNet1, batch_size, num_workers, **kwargs) + super().__init__(spacenet_ds_class, batch_size, num_workers, **kwargs) self.val_split_pct = val_split_pct self.test_split_pct = test_split_pct - - self.train_aug = AugmentationSequential( - K.Normalize(mean=self.mean, std=self.std), - K.PadTo((448, 448)), - K.RandomRotation(p=0.5, degrees=90), - K.RandomHorizontalFlip(p=0.5), - K.RandomVerticalFlip(p=0.5), - K.RandomSharpness(p=0.5), - K.ColorJitter(p=0.5, brightness=0.1, contrast=0.1, saturation=0.1, hue=0.1), - data_keys=['image', 'mask'], - ) - self.aug = AugmentationSequential( - K.Normalize(mean=self.mean, std=self.std), - K.PadTo((448, 448)), - data_keys=['image', 'mask'], - ) + self.spacenet_ds_class = spacenet_ds_class def setup(self, stage: str) -> None: """Set up datasets. @@ -68,17 +55,22 @@ def setup(self, stage: str) -> None: Args: stage: Either 'fit', 'validate', 'test', or 'predict'. """ - self.dataset = SpaceNet1(**self.kwargs) - generator = torch.Generator().manual_seed(0) - self.train_dataset, self.val_dataset, self.test_dataset = random_split( - self.dataset, - [ - 1 - self.val_split_pct - self.test_split_pct, - self.val_split_pct, - self.test_split_pct, - ], - generator, - ) + if stage in ['fit', 'validate', 'test']: + self.dataset = self.spacenet_ds_class(split='train', **self.kwargs) + generator = torch.Generator().manual_seed(0) + self.train_dataset, self.val_dataset, self.test_dataset = random_split( + self.dataset, + [ + 1 - self.val_split_pct - self.test_split_pct, + self.val_split_pct, + self.test_split_pct, + ], + generator, + ) + + # test split in SpaceNet does not have labels + if stage in ['predict']: + self.predict_dataset = self.spacenet_ds_class(split='test', **self.kwargs) def on_after_batch_transfer( self, batch: dict[str, Tensor], dataloader_idx: int @@ -95,6 +87,93 @@ def on_after_batch_transfer( # We add 1 to the mask to map the current {background, building} labels to # the values {1, 2}. This is necessary because we add 0 padding to the # mask that we want to ignore in the loss function. - batch['mask'] += 1 + if 'mask' in batch: + batch['mask'] += 1 return super().on_after_batch_transfer(batch, dataloader_idx) + + +class SpaceNet1DataModule(SpaceNetBaseDataModule): + """LightningDataModule implementation for the SpaceNet1 dataset. + + Randomly splits the train split into train/val/test. The test split does not have labels, + and is only used for prediction. + + .. versionadded:: 0.4 + """ + + def __init__( + self, + batch_size: int = 64, + num_workers: int = 0, + val_split_pct: float = 0.1, + test_split_pct: float = 0.2, + **kwargs: Any, + ) -> None: + """Initialize a new SpaceNet1DataModule instance. + + Args: + batch_size: Size of each mini-batch. + num_workers: Number of workers for parallel data loading. + val_split_pct: Percentage of the dataset to use as a validation set. + test_split_pct: Percentage of the dataset to use as a test set. + **kwargs: Additional keyword arguments passed to + :class:`~torchgeo.datasets.SpaceNet1`. + """ + super().__init__( + SpaceNet1, batch_size, num_workers, val_split_pct, test_split_pct, **kwargs + ) + + self.train_aug = AugmentationSequential( + K.Normalize(mean=self.mean, std=self.std), + K.PadTo((448, 448)), + K.RandomRotation(p=0.5, degrees=90), + K.RandomHorizontalFlip(p=0.5), + K.RandomVerticalFlip(p=0.5), + K.RandomSharpness(p=0.5), + K.ColorJitter(p=0.5, brightness=0.1, contrast=0.1, saturation=0.1, hue=0.1), + data_keys=['image', 'mask'], + ) + self.aug = AugmentationSequential( + K.Normalize(mean=self.mean, std=self.std), + K.PadTo((448, 448)), + data_keys=['image', 'mask'], + ) + + self.predict_aug = AugmentationSequential( + K.Normalize(mean=self.mean, std=self.std), + K.PadTo((448, 448)), + data_keys=['image'], + ) + + +class SpaceNet6DataModule(SpaceNetBaseDataModule): + """LightningDataModule implementation for the SpaceNet6 dataset. + + Randomly splits the train split into train/val/test. The test split does not have labels, + and is only used for prediction. + + .. versionadded:: 0.7 + """ + + def __init__( + self, + batch_size: int = 64, + num_workers: int = 0, + val_split_pct: float = 0.1, + test_split_pct: float = 0.2, + **kwargs: Any, + ) -> None: + """Initialize a new SpaceNet6DataModule instance. + + Args: + batch_size: Size of each mini-batch. + num_workers: Number of workers for parallel data loading. + val_split_pct: Percentage of the dataset to use as a validation set. + test_split_pct: Percentage of the dataset to use as a test set. + **kwargs: Additional keyword arguments passed to + :class:`~torchgeo.datasets.SpaceNet6`. + """ + super().__init__( + SpaceNet6, batch_size, num_workers, val_split_pct, test_split_pct, **kwargs + )