Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix(config): do not use "rgb" or projection codes for imagery names #2908

Merged
merged 3 commits into from
Aug 23, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 36 additions & 1 deletion packages/config/src/json/__tests__/config.loader.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import { FsMemory, SourceMemory } from '@chunkd/source-memory';
import { fileURLToPath } from 'node:url';
import o from 'ospec';
import { ConfigProviderMemory } from '../../memory/memory.config.js';
import { initConfigFromUrls } from '../tiff.config.js';
import { getImageryName, initConfigFromUrls } from '../tiff.config.js';

const simpleTiff = new URL('../../../../__tests__/static/rgba8_tiled.tiff', import.meta.url);

Expand Down Expand Up @@ -57,4 +57,39 @@ o.spec('config import', () => {
o(ret.tileSet.layers[1][3857]).equals(ret.imagery[1].id);
o(ret.tileSet.layers[1].name).equals(ret.imagery[1].name);
});

o('should load tiff from filesystem with projection and imagery type in name', async () => {
const buf = await fsa.read(fileURLToPath(simpleTiff));
await fsa.write('memory://tiffs/tile-tiff-name/2193/rgb/tiff-a.tiff', buf);

const cfg = new ConfigProviderMemory();
const ret = await initConfigFromUrls(cfg, [new URL('memory://tiffs/tile-tiff-name/2193/rgb/')]);

o(ret.imagery.length).equals(1);
const imagery = ret.imagery[0];
o(imagery.name).equals('tile-tiff-name');
o(imagery.files).deepEquals([{ name: 'tiff-a.tiff', x: 0, y: -64, width: 64, height: 64 }]);
});

o('should ignore 2193 and rgb from imagery names', () => {
o(getImageryName(new URL('s3://linz-imagery/auckland/auckland_sn5600_1979_0.375m/2193/rgb/'))).equals(
'auckland_sn5600_1979_0.375m',
);
o(getImageryName(new URL('s3://linz-imagery/auckland/auckland_sn5600_1979_0.375m/2193/rgbi/'))).equals(
'auckland_sn5600_1979_0.375m',
);
o(getImageryName(new URL('s3://linz-imagery/auckland/auckland_sn5600_1979_0.375m/2193/dem_1m/'))).equals(
'auckland_sn5600_1979_0.375m',
);
o(getImageryName(new URL('s3://linz-imagery/auckland/auckland_sn5600_1979_0.375m/2193/dsm_1m/'))).equals(
'auckland_sn5600_1979_0.375m',
);
o(getImageryName(new URL('s3://linz-imagery/auckland/auckland_sn5600_1979_0.375m/3857/dsm_1m/'))).equals(
'auckland_sn5600_1979_0.375m',
);

o(getImageryName(new URL('s3://linz-imagery/auckland/auckland_sn5600_1979_0.375m/3857/DSM_1m/'))).equals(
'auckland_sn5600_1979_0.375m',
);
});
});
68 changes: 58 additions & 10 deletions packages/config/src/json/tiff.config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,54 @@ async function loadStacFromURL(target: URL): Promise<StacCollection | null> {
}
}

/** When attempting to guess a folder name, try and ignore common imagery types and projection */
const IgnoredTitles = new Set([
// Imagery
'rgb',
'rgbi',
// Elevation
'dem_1m',
'dsm_1m',
//Projections
'2193',
'3857',
]);

/**
* Guess a better imagery name from a target URL
*
* A lot of our source paths include the type of imagery eg "rgb", "rgbi" or "dem_1m",
* these names are not super helpful and often there are better names further up the pathname
*
* @example
* ```typescript
* getImageryName('s3://linz-imagery/auckland/auckland_sn5600_1979_0.375m/2193/rgb/')
* // auckland_sn5600_1979_0.375m
* ```
*
* The list of paths ignored are from
*
* @see {IgnoredTitles}
*
* For common imagery paths see:
*
* @see {@link https://github.com/linz/imagery}
* @see {@link https://github.com/linz/elevation}
*/
export function getImageryName(target: URL): string {
const parts = target.pathname.split('/'); // URL folders are always "/"

for (let i = parts.length - 1; i >= 0; i--) {
const part = parts[i];
if (part === '') continue;
if (IgnoredTitles.has(part.toLowerCase())) continue;
return part;
}

// Everything is ignored just use basename
return basename(target.pathname);
}

/**
* Attempt to load all imagery inside of a path and create a configuration from it
*
Expand All @@ -144,14 +192,14 @@ export async function initImageryFromTiffUrl(
if (stac == null) log?.warn({ target: targetPath }, 'Tiff:StacNotFound');
const params = computeTiffSummary(target, tiffs);

const folderName = basename(targetPath);
const title = stac?.title ?? folderName;
const imageryName = getImageryName(target);
const title = stac?.title ?? imageryName;
const tileMatrix =
params.projection === EpsgCode.Nztm2000 ? Nztm2000QuadTms : TileMatrixSets.tryGet(params.projection);

const imagery: ConfigImageryTiff = {
id: provider.Imagery.id(sha256base58(target.href)),
name: folderName,
name: imageryName,
title,
updatedAt: Date.now(),
projection: params.projection,
Expand All @@ -164,7 +212,7 @@ export async function initImageryFromTiffUrl(
collection: stac ?? undefined,
};
imagery.overviews = await ConfigJson.findImageryOverviews(imagery);
log?.info({ title, files: imagery.files.length }, 'Tiff:Loaded');
log?.info({ title, imageryName, files: imagery.files.length }, 'Tiff:Loaded');
provider.put(imagery);

return imagery;
Expand All @@ -178,23 +226,23 @@ export async function initImageryFromTiffUrl(
*
* Given the following folder structure
*
* ```
* /imagery/invercargill_2022_0.05m/*.tiff
* /imagery/wellington_2022_0.05/*.tiff
* ```typescript
* "/imagery/invercargill_2022_0.05m/*.tiff"
* "/imagery/wellington_2022_0.05/*.tiff"
* ```
*
* A. A single imagery datasets
*
* ```
* ```typescript
* target = ["/imagery/invercargill_2022_0.05m/"]
* ```
*
* will load all tiffs from the resulting folder into a single tile set `aerial`
*
* B: A tree of imagery datasets
*
* ```
* target = ["/imagery/invercargill_2022_0.05m/", "/imagery/wellington_2022_0.05/*.tiff"]
* ```typescript
* target = ["/imagery/invercargill_2022_0.05m/*.tiff", "/imagery/wellington_2022_0.05/*.tiff"]
* ```
*
* will load all tiffs from all folders targets into a single tile set "aerial",
Expand Down
Loading