Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat(cli): log when all tiffs in a job have been uploaded #1928

Merged
merged 2 commits into from
Oct 22, 2021
Merged

feat(cli): log when all tiffs in a job have been uploaded #1928

Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
b5ed328
feat(cli): log when all tiffs in a job have been uploaded
blacha Oct 22, 2021
7a7ba3a
Merge branch 'master' into feat/log-job-done
kodiakhq[bot] Oct 22, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 39 additions & 8 deletions packages/cli/src/cli/cogify/action.cog.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { Env, fsa, LogConfig, LoggerFatalError } from '@basemaps/shared';
import { Env, fsa, LogConfig, LoggerFatalError, LogType } from '@basemaps/shared';
import {
CommandLineAction,
CommandLineFlagParameter,
Expand All @@ -15,6 +15,7 @@ import { CogJob } from '../../cog/types.js';
import { Gdal } from '../../gdal/gdal.js';
import { CliId } from '../base.cli.js';
import { makeTempFolder } from '../folder.js';
import path from 'path';

export class ActionCogCreate extends CommandLineAction {
private job?: CommandLineStringParameter;
Expand Down Expand Up @@ -69,11 +70,17 @@ export class ActionCogCreate extends CommandLineAction {
const job = await CogStacJob.load(jobFn);
const isCommit = this.commit?.value ?? false;

const logger = LogConfig.get().child({ correlationId: job.id, imageryName: job.name });
const logger = LogConfig.get().child({
correlationId: job.id,
imageryName: job.name,
tileMatrix: job.tileMatrix.identifier,
});

LogConfig.set(logger);
logger.info('CogCreate:Start');

const gdalVersion = await Gdal.version(logger);
logger.info({ version: gdalVersion }, 'GdalVersion');
logger.info({ version: gdalVersion }, 'CogCreate:GdalVersion');

const name = this.getName(job);
if (name == null) return;
Expand All @@ -82,10 +89,10 @@ export class ActionCogCreate extends CommandLineAction {
fsa.configure(job.output.location);

const outputExists = await fsa.exists(targetPath);
logger.info({ targetPath, outputExists }, 'CheckExists');
logger.info({ targetPath, outputExists }, 'CogCreate:CheckExists');
// Output file exists don't try and overwrite it
if (outputExists) {
logger.warn({ targetPath }, 'OutputExists');
logger.warn({ targetPath }, 'CogCreate:OutputExists');
return;
}

Expand All @@ -95,7 +102,7 @@ export class ActionCogCreate extends CommandLineAction {
let cutlineJson: FeatureCollection | undefined;
if (job.output.cutline != null) {
const cutlinePath = job.getJobPath('cutline.geojson.gz');
logger.info({ path: cutlinePath }, 'UsingCutLine');
logger.info({ path: cutlinePath }, 'CogCreate:UsingCutLine');
cutlineJson = await Cutline.loadCutline(cutlinePath);
} else {
logger.warn('NoCutLine');
Expand All @@ -105,16 +112,17 @@ export class ActionCogCreate extends CommandLineAction {
const tmpVrtPath = await CogVrt.buildVrt(tmpFolder, job, cutline, name, logger);

if (tmpVrtPath == null) {
logger.warn({ name, tileMatrix: job.tileMatrix.identifier }, 'NoMatchingSourceImagery');
logger.warn({ name }, 'CogCreate:NoMatchingSourceImagery');
return;
}

const tmpTiff = fsa.join(tmpFolder, `${name}.tiff`);

await buildCogForName(job, name, tmpVrtPath, tmpTiff, logger, isCommit);
logger.info({ target: targetPath }, 'StoreTiff');
logger.info({ target: targetPath }, 'CogCreate:StoreTiff');
if (isCommit) {
await fsa.write(targetPath, createReadStream(tmpTiff));
await this.checkJobStatus(job, logger);
} else {
logger.warn('DryRun:Done');
}
Expand All @@ -124,6 +132,29 @@ export class ActionCogCreate extends CommandLineAction {
}
}

/** Check to see how many tiffs are remaining in the job */
async checkJobStatus(job: CogStacJob, logger: LogType): Promise<void> {
const basePath = job.getJobPath();
const expectedTiffs = new Set<string>();
for await (const fileName of fsa.list(basePath)) {
const basename = path.basename(fileName);
// Look for tile tiffs only
if (!basename.includes('-') || !basename.endsWith('.tiff')) continue;
expectedTiffs.add(basename);
}

const jobSize = job.output.files.length;
for (const file of job.output.files) {
if (expectedTiffs.has(`${file.name}.tiff`)) expectedTiffs.delete(`${file.name}.tiff`);
}

if (expectedTiffs.size === 0) {
logger.info({ tiffCount: jobSize }, 'CogCreate:JobComplete');
} else {
logger.info({ tiffCount: jobSize, remainingTiff: expectedTiffs.size }, 'CogCreate:JobProgress');
}
}

protected onDefineParameters(): void {
this.job = this.defineStringParameter({
argumentName: 'JOB',
Expand Down