feat: convert a tif using a docker based gdal

Uses the new 3.1 GDAL cog driver

jest.config.js

@@ -1,10 +1,4 @@
 module.exports = {
-    transform: {
-        '^.+\\.tsx?$': 'ts-jest',
-    },
-    testRegex: '(/__tests__/.*|(\\.|/)(test|spec))\\.tsx?$',
-    moduleFileExtensions: ['ts', 'tsx', 'js', 'jsx', 'json', 'node'],
-    globals: {
-        'ts-jest': {},
-    },
+    testRegex: '(/__tests__/.*|(\\.|/)(test|spec))\\.js$',
+    moduleFileExtensions: ['js'],
 };

packages/cog/package.json

@@ -0,0 +1,16 @@
+{
+  "name": "@basemaps/cog",
+  "version": "0.0.1",
+  "private": true,
+  "repository": "git@github.com:linz/basemaps.git",
+  "author": "",
+  "license": "BSD 3-Clause",
+  "main": "./build/index.js",
+  "types": "./build/index.d.ts",
+  "scripts": {
+    "build": "tsc",
+    "test": "jest"
+  },
+  "dependencies": {},
+  "devDependencies": {}
+}

packages/cog/src/__test__/gdal.progress.test.ts

@@ -0,0 +1,24 @@
+import { GdalProgressParser } from '../gdal.progress';
+
+describe('GdalProgressParser', () => {
+    it('should emit on progress', () => {
+        const prog = new GdalProgressParser();
+        expect(prog.progress).toEqual(0);
+
+        prog.data(Buffer.from('\n.'));
+        expect(prog.progress.toFixed(2)).toEqual('3.23');
+    });
+
+    it('should take 31 dots to finish', () => {
+        const prog = new GdalProgressParser();
+        let processCount = 0;
+        prog.data(Buffer.from('\n'));
+        prog.on('progress', () => processCount++);
+
+        for (let i = 0; i < 31; i++) {
+            prog.data(Buffer.from('.'));
+            expect(processCount).toEqual(i + 1);
+        }
+        expect(prog.progress.toFixed(2)).toEqual('100.00');
+    });
+});

packages/cog/src/__test__/gdal.test.ts

@@ -0,0 +1,35 @@
+import { GdalCogBuilder } from '../gdal';
+
+jest.mock('child_process');
+
+describe('GdalCogBuilder', () => {
+    it('should default all options', () => {
+        const builder = new GdalCogBuilder('/foo', 'bar.tiff');
+
+        expect(builder.config.bbox).toEqual(undefined);
+        expect(builder.config.compression).toEqual('webp');
+        expect(builder.config.resampling).toEqual('lanczos');
+        expect(builder.config.blockSize).toEqual(512);
+        expect(builder.config.alignmentLevels).toEqual(1);
+    });
+
+    it('should create a docker command', () => {
+        const builder = new GdalCogBuilder('/foo/foo.tiff', '/bar/bar.tiff');
+
+        const args = builder.args;
+
+        // Should mount the source and target folders
+        expect(args.includes('/foo:/foo')).toEqual(true);
+        expect(args.includes('/bar:/bar')).toEqual(true);
+
+        expect(args.includes('TILING_SCHEME=GoogleMapsCompatible')).toEqual(true);
+        expect(args.includes('COMPRESS=webp')).toEqual(true);
+        expect(builder.args.includes('BLOCKSIZE=512')).toEqual(true);
+
+        builder.config.compression = 'jpeg';
+        expect(builder.args.includes('COMPRESS=jpeg')).toEqual(true);
+
+        builder.config.blockSize = 256;
+        expect(builder.args.includes('BLOCKSIZE=256')).toEqual(true);
+    });
+});

packages/cog/src/gdal.config.ts

@@ -0,0 +1,29 @@
+export interface GdalCogBuilderOptions {
+    /**
+     * Number of aligned tile levels
+     * @default 1
+     */
+    alignmentLevels: number;
+
+    /** Limit the output to a bounding box
+     */
+    bbox?: number;
+
+    /**
+     * Compression to use for the cog
+     * @default 'webp'
+     */
+    compression: 'webp' | 'jpeg';
+
+    /**
+     * Resampling method to use
+     * @default 'lanczos'
+     */
+    resampling: 'lanczos';
+
+    /**
+     * Output tile size
+     * @default 512
+     */
+    blockSize: 256 | 512 | 1024 | 2048 | 4096;
+}

packages/cog/src/gdal.progress.ts

@@ -0,0 +1,41 @@
+import { EventEmitter } from 'events';
+
+/**
+ * Emit a "progress" event every time a "." is recorded in the output
+ */
+export class GdalProgressParser extends EventEmitter {
+    // Progress starts with "Input file size is .., ..\n"
+    waitNewLine = true;
+    dotCount = 0;
+    byteCount = 0;
+
+    get progress(): number {
+        return this.dotCount * (100 / 31);
+    }
+
+    data(data: Buffer): void {
+        const str = data.toString('utf8');
+        this.byteCount += str.length;
+        // In theory only a small amount of output bytes should be recorded
+        if (this.byteCount > 1024) {
+            throw new Error('Too much data: ' + str);
+        }
+
+        if (this.waitNewLine) {
+            const newLine = str.indexOf('\n');
+            if (newLine > -1) {
+                this.waitNewLine = false;
+                return this.data(Buffer.from(str.substr(newLine + 1)));
+            }
+            return;
+        }
+
+        const bytes = str.split('');
+        for (const byte of bytes) {
+            if (byte == '.') {
+                this.dotCount++;
+                this.emit('progress', this.progress);
+            }
+        }
+    }
+}

packages/cog/src/gdal.ts

@@ -0,0 +1,141 @@
+import { ChildProcessWithoutNullStreams, spawn } from 'child_process';
+import * as os from 'os';
+import * as path from 'path';
+import { GdalCogBuilderOptions } from './gdal.config';
+import { GdalProgressParser } from './gdal.progress';
+
+const DOCKER_CONTAINER = 'osgeo/gdal';
+const DOCKER_CONTAINER_TAG = 'ubuntu-small-latest';
+
+/**
+ * A docker based GDAL Cog Builder
+ *
+ * This uses the new 3.1 COG Driver https://gdal.org/drivers/raster/cog.html
+ *
+ * When GDAL 3.1 is released docker could be removed from this process.
+ */
+export class GdalCogBuilder {
+    config: GdalCogBuilderOptions;
+
+    /**
+     * Source file generally a .vrt
+     */
+    source: string;
+    /**
+     * Output file
+     */
+    target: string;
+
+    /**
+     * Current running child process
+     */
+    child: ChildProcessWithoutNullStreams | null;
+    /**
+     * Promise waiting for child process to finish
+     */
+    promise: Promise<void> | null;
+    /** When the process started */
+    startTime: number;
+    /** Parse the output looking for "." */
+    parser: GdalProgressParser;
+
+    constructor(source: string, target: string, config: Partial<GdalCogBuilderOptions> = {}) {
+        this.source = source;
+        this.target = target;
+        this.config = {
+            bbox: config.bbox,
+            alignmentLevels: config.alignmentLevels ?? 1,
+            compression: config.compression ?? 'webp',
+            resampling: config.resampling ?? 'lanczos',
+            blockSize: config.blockSize ?? 512,
+        };
+        this.parser = new GdalProgressParser();
+    }
+
+    getMount(source: string): string[] {
+        if (source == null) {
+            return [];
+        }
+        if (this.source.startsWith('/')) {
+            const sourcePath = path.dirname(source);
+            return ['-v', `${sourcePath}:${sourcePath}`];
+        }
+        return [];
+    }
+
+    get args(): string[] {
+        const userInfo = os.userInfo();
+        return [
+            'run',
+            // Config the container to be run as the current user
+            '--user',
+            `${userInfo.uid}:${userInfo.gid}`,
+
+            ...this.getMount(this.source),
+            ...this.getMount(this.target),
+
+            // Docker container
+            '-i',
+            `${DOCKER_CONTAINER}:${DOCKER_CONTAINER_TAG}`,
+
+            // GDAL Arguments
+            `gdal_translate`,
+            // Force output using COG Driver
+            '-of',
+            'COG',
+            // Force GoogleMaps tiling
+            '-co',
+            'TILING_SCHEME=GoogleMapsCompatible',
+            // Max CPU POWER
+            '-co',
+            'NUM_THREADS=ALL_CPUS',
+            // Force big tiff the extra few bytes savings of using little tiffs does not affect us
+            '-co',
+            'BIGTIFF=YES',
+            // Force a alpha layer
+            '-co',
+            'ADD_ALPHA=YES',
+            // User configured output block size
+            '-co',
+            `BLOCKSIZE=${this.config.blockSize}`,
+            // User configured resampling method
+            '-co',
+            `RESAMPLING=${this.config.resampling}`,
+            // User configured compression
+            '-co',
+            `COMPRESS=${this.config.compression}`,
+            this.source,
+            this.target,
+        ];
+    }
+
+    convert(): Promise<void> {
+        if (this.promise != null) {
+            return this.promise;
+        }
+        this.startTime = Date.now();
+
+        const child = spawn('docker', this.args);
+        this.child = child;
+
+        const errorBuff: Buffer[] = [];
+        child.stderr.on('data', (data: Buffer) => errorBuff.push(data));
+        child.stdout.on('data', (data: Buffer) => this.parser.data(data));
+
+        this.promise = new Promise((resolve, reject) => {
+            child.on('exit', (code: number) => {
+                if (code != 0) {
+                    // TODO log out errorBuff
+                    return reject(new Error('Failed to execute GDAL: ' + errorBuff.join('').trim()));
+                }
+                return resolve();
+            });
+            child.on('error', (err: Error) => {
+                // TODO log out errorBuff
+                reject(err);
+            });
+        });
+
+        return this.promise;
+    }
+}

packages/cog/src/index.ts

@@ -0,0 +1,2 @@
+export { GdalCogBuilder } from './gdal';
+export { GdalCogBuilderOptions } from './gdal.config';

packages/cog/tsconfig.json

@@ -0,0 +1,9 @@
+{
+    "extends": "../../tsconfig.base.json",
+    "compilerOptions": {
+        "rootDir": "./src",
+        "outDir": "./build"
+    },
+    "include": ["src/**/*"],
+    "references": []
+}

tsconfig.json

@@ -5,6 +5,8 @@
     }, {
         "path": "./packages/tiler"
     }, {
+        "path": "./packages/cog"
+    },{
         "path": "./packages/_infra"
     },{
         "path": "./packages/lambda-api-tracker"