Merge pull request #252 from linz/feat/configure-temp-folder

Feat/configure temp folder

packages/_infra/src/cogify/index.ts

@@ -3,6 +3,9 @@ import iam = require('@aws-cdk/aws-iam');
 import batch = require('@aws-cdk/aws-batch');
 import ec2 = require('@aws-cdk/aws-ec2');
 import ecrAssets = require('@aws-cdk/aws-ecr-assets');
+import { Env } from '@basemaps/shared';
+import { ScratchData } from './mount.folder';
+import { createHash } from 'crypto';
 
 /**
  * Cogification infrastructure
@@ -50,31 +53,42 @@ export class CogBuilderStack extends cdk.Stack {
         const vpc = ec2.Vpc.fromLookup(this, 'AlbVpc', { tags: { default: 'true' } });
         const sg = new ec2.SecurityGroup(this, 'CogBatchSecurity', { vpc });
 
-        const launchTemplateName = 'CogBatchLaunchTemplate';
-        new ec2.CfnLaunchTemplate(this, 'CogBatchLaunchTemplate', {
-            launchTemplateData: {
+        const launchTemplateData = {
+            /**
+             * This only resizes the HOST's file system, which default's to 22GB
+             *
+             * Each container is still limited to 8GB of storage but this allows
+             * more containers to be run on each host
+             */
+            blockDeviceMappings: [
+                {
+                    deviceName: '/dev/xvdcz',
+                    ebs: {
+                        volumeSize: 128,
+                        volumeType: 'gp2',
+                    },
+                },
                 /**
-                 * This only resizes the HOST's file system, which default's to 22GB
-                 *
-                 * Each container is still limited to 8GB of storage but this allows
-                 * more containers to be run on each hos
-                 *
-                 * TODO we could mount a new drive `/dev/sda` and then share that
-                 * to all the containers, a file system will need be created on boot `mkfs.ext4`
-                 * Which could be done inside of a `launchTemplateData.userData` bash script
+                 * Provide a scratch folder for more temporary storage
+                 * This will be mounted into each container
                  */
-                blockDeviceMappings: [
-                    {
-                        deviceName: '/dev/xvdcz',
-                        ebs: {
-                            volumeSize: 128,
-                            volumeType: 'gp2',
-                        },
+                {
+                    deviceName: `/dev/${ScratchData.Device}`,
+                    ebs: {
+                        volumeSize: 256,
+                        volumeType: 'gp2',
                     },
-                ],
-            },
-            launchTemplateName,
-        });
+                },
+            ],
+            // Make a file system and mount the folder
+            userData: ScratchData.UserData,
+        };
+        const launchTemplateDataId = createHash('sha256')
+            .update(JSON.stringify(launchTemplateData))
+            .digest('hex')
+            .substr(0, 10);
+        const launchTemplateName = `CogBatchLaunchTemplate-${launchTemplateDataId}`;
+        new ec2.CfnLaunchTemplate(this, 'CogBatchLaunchTemplate', { launchTemplateData, launchTemplateName });
 
         const computeEnv = new batch.CfnComputeEnvironment(this, 'CogBatchCompute', {
             type: 'MANAGED',
@@ -118,7 +132,14 @@ export class CogBuilderStack extends cdk.Stack {
                  * Eg a instance with 8192MB allocates 7953MB usable
                  */
                 memory: 3900,
-                environment: [],
+                environment: [
+                    {
+                        name: Env.TempFolder,
+                        value: ScratchData.Folder,
+                    },
+                ],
+                mountPoints: [{ containerPath: ScratchData.Folder, sourceVolume: 'scratch' }],
+                volumes: [{ name: 'scratch', host: { sourcePath: ScratchData.Folder } }],
             },
         });
 

packages/_infra/src/cogify/mount.folder.ts

@@ -0,0 +1,23 @@
+const MountDevice = `xvdz`;
+const MountFolder = '/scratch';
+const MountFolderScript = `MIME-Version: 1.0
+Content-Type: multipart/mixed; boundary="==MYBOUNDARY=="
+
+--==MYBOUNDARY==
+Content-Type: text/x-shellscript; charset="us-ascii"
+
+#!/bin/bash
+mkfs.ext4 /dev/${MountDevice}
+mkdir ${MountFolder}
+mount /dev/${MountDevice} ${MountFolder}
+service docker restart
+
+--==MYBOUNDARY==`;
+
+export const ScratchData = {
+    /** Device name `xvdz` */
+    Device: MountDevice,
+    /** Folder where device is mounted @default /scratch */
+    Folder: MountFolder,
+    UserData: Buffer.from(MountFolderScript).toString('base64'),
+};

packages/cog/src/cli/actions/action.cog.ts

@@ -11,6 +11,7 @@ import { buildCogForQuadKey, CogJob } from '../../cog';
 import { FileOperator } from '../../file/file';
 import { FileOperatorSimple } from '../../file/file.local';
 import { buildWarpedVrt } from '../../cog.vrt';
+import { makeTempFolder } from '../../file/temp.folder';
 
 export class ActionCogCreate extends CommandLineAction {
     private job?: CommandLineStringParameter;
@@ -79,17 +80,19 @@ export class ActionCogCreate extends CommandLineAction {
         }
         const targetPath = FileOperator.join(job.output.path, `${job.id}/${quadKey}.tiff`);
         const outputFs = FileOperator.create(job.output);
+
+        const outputExists = await outputFs.exists(targetPath);
+        logger.info({ targetPath, outputExists }, 'CheckExists');
         // Output file exists don't try and overwrite it
-        if (await outputFs.exists(targetPath)) {
+        if (outputExists) {
             logger.warn({ targetPath }, 'OutputExists');
             return;
         }
 
-        const tmpFolder = `/tmp/basemaps-${job.id}-${processId}`;
+        const tmpFolder = await makeTempFolder(`basemaps-${job.id}-${processId}`);
 
         const tmpTiff = FileOperator.join(tmpFolder, `${quadKey}.tiff`);
         const tmpVrt = FileOperator.join(tmpFolder, `${job.id}.vrt`);
-        await fs.mkdir(tmpFolder, { recursive: true });
 
         try {
             logger.info({ path: job.output.vrt.path }, 'FetchVrt');

packages/cog/src/cli/actions/action.job.ts

@@ -1,24 +1,25 @@
 import { EPSG, LogConfig } from '@basemaps/shared';
+import { CogSource } from '@cogeotiff/core';
+import { CogSourceAwsS3 } from '@cogeotiff/source-aws';
+import { CogSourceFile } from '@cogeotiff/source-file';
 import {
     CommandLineAction,
     CommandLineFlagParameter,
     CommandLineIntegerParameter,
     CommandLineStringParameter,
 } from '@microsoft/ts-command-line';
+import { createReadStream, promises as fs } from 'fs';
 import * as Mercator from 'global-mercator';
-import { promises as fs, createReadStream } from 'fs';
+import { basename } from 'path';
 import * as ulid from 'ulid';
 import { CogBuilder } from '../../builder';
 import { CogJob } from '../../cog';
 import { buildVrtForTiffs, VrtOptions } from '../../cog.vrt';
-import { FileOperator } from '../../file/file';
 import { TileCover } from '../../cover';
+import { FileOperator } from '../../file/file';
 import { FileConfig } from '../../file/file.config';
 import { FileOperatorS3 } from '../../file/file.s3';
-import { CogSource } from '@cogeotiff/core';
-import { CogSourceAwsS3 } from '@cogeotiff/source-aws';
-import { CogSourceFile } from '@cogeotiff/source-file';
-import { basename } from 'path';
+import { makeTempFolder } from '../../file/temp.folder';
 
 const ProcessId = ulid.ulid();
 
@@ -173,8 +174,7 @@ export class ActionJobCreate extends CommandLineAction {
             quadkeys: metadata.covering,
         };
 
-        const tmpFolder = `/tmp/basemaps-${job.id}`;
-        await fs.mkdir(tmpFolder, { recursive: true });
+        const tmpFolder = await makeTempFolder(`basemaps-${job.id}`);
         try {
             // Local file systems need directories to be created before writing to them
             if (!FileOperatorS3.isS3(outputFs)) {

packages/cog/src/file/file.s3.ts

@@ -81,11 +81,17 @@ export class FileOperatorS3 implements FileProcessor {
             .promise();
     }
 
-    async exists(): Promise<boolean> {
-        // TODO does heading a missing object error
-        // const opts = FileOperatorS3.parse(filePath);
-        // await s3.headObject({ Bucket: opts.bucket, Key: opts.key }).promise();
-        throw new Error('Not yet implemented');
+    async exists(filePath: string): Promise<boolean> {
+        const opts = FileOperatorS3.parse(filePath);
+        try {
+            await this.s3.headObject({ Bucket: opts.bucket, Key: opts.key }).promise();
+            return true;
+        } catch (e) {
+            if (e.code == 'NotFound') {
+                return false;
+            }
+            throw e;
+        }
     }
 
     readStream(filePath: string): Readable {

packages/cog/src/file/temp.folder.ts

@@ -0,0 +1,12 @@
+import * as path from 'path';
+import { Env } from '@basemaps/shared';
+import { promises as fs } from 'fs';
+
+/** Make a temp folder inside TEMP_FOLDER's path */
+export async function makeTempFolder(folder: string): Promise<string> {
+    const tempPath = Env.get(Env.TempFolder, '/tmp');
+    const folderPath = path.join(tempPath, folder);
+
+    await fs.mkdir(folderPath, { recursive: true });
+    return folderPath;
+}

packages/shared/src/const.ts

@@ -21,6 +21,9 @@ export const Env = {
     /** How many tiffs to load at one time */
     TiffConcurrency: 'TIFF_CONCURRENCY',
 
+    /** Temporary folder used for processing, @default /tmp */
+    TempFolder: 'TEMP_FOLDER',
+
     /** Batch Index offset used to controll mutliple batch jobs */
     BatchIndex: 'AWS_BATCH_JOB_ARRAY_INDEX',