Merge pull request #53 from linz/build/improve-test

Build/improve test

packages/shared/src/index.ts

@@ -11,3 +11,4 @@ export { Logger } from './log';
 export { getXyzFromPath, PathData } from './path';
 export { Projection } from './projection';
 export { LambdaSession } from './session';
+export { Bounds, BoundingBox, Size } from './bounds';

packages/tiler/package.json

@@ -17,6 +17,11 @@
     "sharp": "^0.23.1"
   },
   "devDependencies": {
-    "@types/sharp": "^0.22.3"
+    "@cogeotiff/source-file": "^0.4.0",
+    "@types/pixelmatch": "^5.0.0",
+    "@types/pngjs": "^3.3.2",
+    "@types/sharp": "^0.22.3",
+    "pixelmatch": "^5.1.0",
+    "pngjs": "^3.4.0"
   }
 }

packages/tiler/src/__test__/tile.creation.test.ts

@@ -0,0 +1,89 @@
+import { LambdaSession, Logger } from '@basemaps/shared';
+import { CogSourceFile } from '@cogeotiff/source-file';
+import { readFileSync, writeFileSync } from 'fs';
+import * as path from 'path';
+import { PNG } from 'pngjs';
+import { Tiler } from '../tiler';
+import PixelMatch = require('pixelmatch');
+import { CogTiff } from '@cogeotiff/core';
+
+// To regenerate all the expected images set this to true and run the tests
+const WRITE_IMAGES = false;
+
+function getExpectedTileName(tileSize: number, x: number, y: number, zoom: number): string {
+    return path.join(__dirname, `../../data/expected/tile_${tileSize}_${x}_${y}_z${zoom}.png`);
+}
+function getExpectedTile(tileSize: number, x: number, y: number, zoom: number): PNG {
+    const fileName = getExpectedTileName(tileSize, x, y, zoom);
+    const bytes = readFileSync(fileName);
+    return PNG.sync.read(bytes);
+}
+
+describe('TileCreation', () => {
+    const tiffPath = path.join(__dirname, '../../data/rgba8_tiled.tiff');
+    const tiff = new CogTiff(new CogSourceFile(tiffPath));
+
+    beforeEach(async () => {
+        await tiff.init();
+        LambdaSession.reset();
+    });
+
+    it('should generate a tile', async () => {
+        // Make a really large tile so this image will be visible at zoom zero
+        const tiler = new Tiler(2 ** 20);
+        const layers = await tiler.tile([tiff], 0, 0, 0, Logger);
+
+        expect(layers).not.toEqual(null);
+        if (layers == null) throw new Error('Tile is null');
+
+        expect(layers.length).toEqual(1);
+        const [layer] = layers;
+        expect(layer.id).toEqual(tiff.source.name);
+        expect(layer.extract).toEqual({ height: 4, width: 4 });
+        expect(layer.resize).toEqual({ height: 2, width: 2 });
+        expect(layer.x).toEqual(tiler.tileSize / 2);
+        expect(layer.y).toEqual(tiler.tileSize / 2);
+    });
+
+    [
+        { tileSize: 256, zoom: 18 },
+        { tileSize: 256, zoom: 19 },
+        { tileSize: 512, zoom: 19 },
+        { tileSize: 1024, zoom: 19 },
+        { tileSize: 2048, zoom: 19 },
+        { tileSize: 4096, zoom: 19 },
+    ].forEach(({ tileSize, zoom }) => {
+        it(`should render a tile zoom:${zoom} tile: ${tileSize}`, async () => {
+            const center = 2 ** zoom;
+            const centerTile = center / 2;
+            const tiler = new Tiler(tileSize);
+
+            // Make the background black to easily spot flaws
+            tiler.raster.background.alpha = 1;
+            const layers = await tiler.tile([tiff], centerTile, centerTile, zoom, Logger);
+            expect(layers).not.toEqual(null);
+            if (layers == null) throw new Error('Tile is null');
+
+            const png = await tiler.raster.compose(
+                layers,
+                Logger,
+            );
+            const newImage = PNG.sync.read(png);
+            if (WRITE_IMAGES) {
+                const fileName = getExpectedTileName(tileSize, centerTile, centerTile, zoom);
+                writeFileSync(fileName, png);
+            }
+
+            const expectedImage = await getExpectedTile(tileSize, centerTile, centerTile, zoom);
+
+            const missMatchedPixels = PixelMatch(expectedImage.data, newImage.data, null, tileSize, tileSize);
+            if (missMatchedPixels > 0) {
+                const fileName = getExpectedTileName(tileSize, centerTile, centerTile, zoom) + '.diff.png';
+                const output = new PNG({ width: tileSize, height: tileSize });
+                PixelMatch(expectedImage.data, newImage.data, output.data, tileSize, tileSize);
+                writeFileSync(fileName, PNG.sync.write(output));
+            }
+            expect(missMatchedPixels).toEqual(0);
+        });
+    });
+});

packages/tiler/src/raster.ts

@@ -1,5 +1,4 @@
-import { Logger, LambdaSession } from '@basemaps/shared';
-import { BoundingBox, Size } from '@basemaps/shared/build/bounds';
+import { Logger, LambdaSession, BoundingBox, Size } from '@basemaps/shared';
 import * as Sharp from 'sharp';
 
 export interface Composition {
@@ -24,6 +23,8 @@ const SharpScaleOptions = { fit: Sharp.fit.cover };
 
 export class Raster {
     private tileSize: number;
+    /** The background of all tiles that are created */
+    public background = { r: 0, g: 0, b: 0, alpha: 0 };
 
     public constructor(tileSize: number) {
         this.tileSize = tileSize;
@@ -37,14 +38,19 @@ export class Raster {
         // 1. Load all image bytes
         // 2. Create image overlays
         timer.start('compose:overlay');
-        const overlays = await Promise.all(composition.map(this.composeTile.bind(this)));
+        const todo: Promise<SharpOverlay>[] = [];
+        for (const comp of composition) {
+            todo.push(this.composeTile(comp));
+        }
+        const overlays = await Promise.all(todo);
         const overlayDuration = timer.end('compose:overlay');
 
         timer.start('compose:compress');
         const output = await sharp
             .composite(overlays)
             .png() // TODO should we configure output options (eg WebP/Png/Jpeg)
             .toBuffer();
+
         const compressDuration = timer.end('compose:compress');
         logger.info({ compressDuration, overlayDuration }, 'TileCompose');
 
@@ -86,7 +92,7 @@ export class Raster {
                 width: this.tileSize,
                 height: this.tileSize,
                 channels: 4,
-                background: { r: 0, g: 0, b: 0, alpha: 0 },
+                background: this.background,
             },
         });
     }

packages/tiler/src/tiler.ts

@@ -19,7 +19,11 @@ export class Tiler {
     public projection: Projection;
     public raster: Raster;
 
+    /** Tile size for the tiler and sub objects */
+    public readonly tileSize: number;
+
     public constructor(tileSize: number) {
+        this.tileSize = tileSize;
         this.projection = new Projection(tileSize);
         this.raster = new Raster(tileSize);
     }
@@ -44,7 +48,7 @@ export class Tiler {
         const timer = LambdaSession.get().timer;
         timer.start('tile:get');
         for (const tiff of tiffs) {
-            const tileOverlays = this.getTiles(tiff, x, y, zoom, logger.child({ tiff: tiff.source.name }));
+            const tileOverlays = this.getTiles(tiff, x, y, zoom, logger);
             if (tileOverlays == null) {
                 continue;
             }
@@ -154,7 +158,8 @@ export class Tiler {
         if (rasterBounds == null) {
             return null;
         }
-        logger.info({ inBounds: true }, 'TiffBoundsCheck');
+        const tiffName = tiff.source.name;
+        logger.info({ tiffName, inBounds: true }, 'TiffBoundsCheck');
         // Find the best internal overview tiff to use with the desired XYZ resolution
         const targetResolution = this.projection.getResolution(z);
         const img = tiff.getImageByResolution(targetResolution);
@@ -184,7 +189,7 @@ export class Tiler {
         }
 
         const tiffTileCount = (endX - startX) * (endY - startY);
-        logger.info({ tiffTileCount, tiffTileUsed: composites.length }, 'TiffInBounds');
+        logger.info({ tiffName, tiffTileCount, tiffTileUsed: composites.length }, 'TiffInBounds');
 
         if (composites.length === 0) {
             return null;

tsconfig.base.json

@@ -17,6 +17,7 @@
         "noFallthroughCasesInSwitch": false,
         "inlineSourceMap": true,
         "inlineSources": true,
+        "noEmitOnError": false,
         "experimentalDecorators": true,
         "strictPropertyInitialization": false,
         "resolveJsonModule": true,

yarn.lock

@@ -434,6 +434,13 @@
     "@cogeotiff/core" "^0.4.0"
     aws-sdk "^2.524.0"
 
+"@cogeotiff/source-file@^0.4.0":
+  version "0.4.0"
+  resolved "https://registry.yarnpkg.com/@cogeotiff/source-file/-/source-file-0.4.0.tgz#5b80bfb6b5d7b5e2f5e8f4b586576346e4a379b5"
+  integrity sha512-8yFHajAZwbirr6JZasUjChTe7JCnKso6DNcuXtV3V0CiZ3HLfK5jJDt2IUPuAbl1RneqxLvKoPhwVY1Vdw6Ftg==
+  dependencies:
+    "@cogeotiff/core" "^0.4.0"
+
 "@commitlint/cli@^8.1.0", "@commitlint/cli@^8.2.0":
   version "8.2.0"
   resolved "https://registry.yarnpkg.com/@commitlint/cli/-/cli-8.2.0.tgz#fbf9969e04e2162d985eaa644fdad6ce807aadb6"
@@ -1670,6 +1677,20 @@
     "@types/pino-std-serializers" "*"
     "@types/sonic-boom" "*"
 
+"@types/pixelmatch@^5.0.0":
+  version "5.0.0"
+  resolved "https://registry.yarnpkg.com/@types/pixelmatch/-/pixelmatch-5.0.0.tgz#3b9222a3116946bbb10ba28c6eab4df4acb93bef"
+  integrity sha512-hx+RYj4KeVrz3VoD/6NaAXqNolMHfKEbPFYdkZhE6dUQXMdNpAMF4tmtoKfs+3T217jnc5KDoWhqEI/u/yPprg==
+  dependencies:
+    "@types/node" "*"
+
+"@types/pngjs@^3.3.2":
+  version "3.3.2"
+  resolved "https://registry.yarnpkg.com/@types/pngjs/-/pngjs-3.3.2.tgz#8ed3bd655ab3a92ea32ada7a21f618e63b93b1d4"
+  integrity sha512-/SBsv93rVnjByzcau24rBwb+N7BHFp2LateaXz1e7m7M0Wzck/ymXTNdWVrCtkuMbwTHAnfdc3X/I/5szsTEAA==
+  dependencies:
+    "@types/node" "*"
+
 "@types/semver@^6.0.1":
   version "6.0.2"
   resolved "https://registry.yarnpkg.com/@types/semver/-/semver-6.0.2.tgz#5e8b09f0e4af53034b1d0fb9977a277847836205"
@@ -7026,6 +7047,13 @@ pirates@^4.0.1:
   dependencies:
     node-modules-regexp "^1.0.0"
 
+pixelmatch@^5.1.0:
+  version "5.1.0"
+  resolved "https://registry.yarnpkg.com/pixelmatch/-/pixelmatch-5.1.0.tgz#b640f0e5a03a09f235a4b818ef3b9b98d9d0b911"
+  integrity sha512-HqtgvuWN12tBzKJf7jYsc38Ha28Q2NYpmBL9WostEGgDHJqbTLkjydZXL1ZHM02ZnB+Dkwlxo87HBY38kMiD6A==
+  dependencies:
+    pngjs "^3.4.0"
+
 pkg-dir@^3.0.0:
   version "3.0.0"
   resolved "https://registry.yarnpkg.com/pkg-dir/-/pkg-dir-3.0.0.tgz#2749020f239ed990881b1f71210d51eb6523bea3"
@@ -7038,6 +7066,11 @@ pn@^1.1.0:
   resolved "https://registry.yarnpkg.com/pn/-/pn-1.1.0.tgz#e2f4cef0e219f463c179ab37463e4e1ecdccbafb"
   integrity sha512-2qHaIQr2VLRFoxe2nASzsV6ef4yOOH+Fi9FBOVH6cqeSgUnoyySPZkxzLuzd+RYOQTRpROA0ztTMqxROKSb/nA==
 
+pngjs@^3.4.0:
+  version "3.4.0"
+  resolved "https://registry.yarnpkg.com/pngjs/-/pngjs-3.4.0.tgz#99ca7d725965fb655814eaf65f38f12bbdbf555f"
+  integrity sha512-NCrCHhWmnQklfH4MtJMRjZ2a8c80qXeMlQMv2uVp9ISJMTt562SbGd6n2oq0PaPgKm7Z6pL9E2UlLIhC+SHL3w==
+
 posix-character-classes@^0.1.0:
   version "0.1.1"
   resolved "https://registry.yarnpkg.com/posix-character-classes/-/posix-character-classes-0.1.1.tgz#01eac0fe3b5af71a2a6c02feabb8c1fef7e00eab"