fix(lambda-analytics): do not track invalid api keys BM-642 (#2392)

* fix(lambda-analytics): do not track invalid api keys BM-642 * refactor: fixup lint * refactor: mix missing imports * refactor: formatting
linz · Aug 1, 2022 · 9f84285 · 9f84285
1 parent 02bcc42
commit 9f84285
Show file tree

Hide file tree

Showing 8 changed files with 65 additions and 63 deletions.
diff --git a/packages/lambda-analytics/package.json b/packages/lambda-analytics/package.json
@@ -18,6 +18,8 @@
   },
   "license": "MIT",
   "dependencies": {
+    "@basemaps/config": "^6.32.1",
+    "@basemaps/geo": "^6.32.1",
     "@basemaps/shared": "^6.32.1"
   },
   "scripts": {

diff --git a/packages/lambda-analytics/src/__tests__/file.process.test.ts b/packages/lambda-analytics/src/__tests__/file.process.test.ts
@@ -2,9 +2,10 @@ import { LogConfig } from '@basemaps/shared';
 import o from 'ospec';
 import { FileProcess } from '../file.process.js';
 import { LogStats } from '../stats.js';
+import { ulid } from 'ulid';
 
-const DevApiKey = 'dThisIsNotAKey';
-const ClientApiKey = 'cThisIsNotAKey';
+const DevApiKey = 'd' + ulid().toLowerCase();
+const ClientApiKey = 'c' + ulid().toLowerCase();
 
 export const ExampleLogs = `#Version: 1.0
 #Fields: date time x-edge-location sc-bytes c-ip cs-method cs(Host) cs-uri-stem sc-status cs(Referer) cs(User-Agent) cs-uri-query cs(Cookie) x-edge-result-type x-edge-request-id x-host-header cs-protocol cs-bytes time-taken x-forwarded-for ssl-protocol ssl-cipher x-edge-response-result-type cs-protocol-version fle-status fle-encrypted-fields c-port time-to-first-byte x-edge-detailed-result-type sc-content-type sc-content-len sc-range-start sc-range-end
@@ -54,7 +55,7 @@ o.spec('FileProcess', () => {
     o(apiStats?.apiType).equals('d');
     o(apiStats?.total).equals(1);
     o(apiStats?.cache).deepEquals({ hit: 1, miss: 0 });
-    o(apiStats?.projection).deepEquals({ 2193: 0, 3857: 1 });
+    o(apiStats?.tileMatrix).deepEquals({ WebMercatorQuad: 1 });
   });
 
   o('should extract and track a bunch of hits', async () => {
@@ -71,15 +72,15 @@ o.spec('FileProcess', () => {
     o(devStats?.total).equals(3);
     o(devStats?.apiType).equals('d');
     o(devStats?.cache).deepEquals({ hit: 2, miss: 1 });
-    o(devStats?.projection).deepEquals({ 2193: 0, 3857: 3 });
+    o(devStats?.tileMatrix).deepEquals({ WebMercatorQuad: 3 });
     o(devStats?.extension).deepEquals({ webp: 1, jpeg: 1, png: 1, wmts: 0, other: 0, pbf: 0 });
-    o(devStats?.tileSet).deepEquals({ aerial: 2, aerialIndividual: 0, topo50: 1, direct: 0 });
+    o(devStats?.tileSet).deepEquals({ aerial: 2, topo50: 1 });
 
     o(clientStats?.total).equals(2);
     o(clientStats?.apiType).equals('c');
     o(clientStats?.cache).deepEquals({ hit: 2, miss: 0 });
-    o(clientStats?.projection).deepEquals({ 2193: 0, 3857: 2 });
+    o(clientStats?.tileMatrix).deepEquals({ WebMercatorQuad: 2 });
     o(clientStats?.extension).deepEquals({ webp: 0, jpeg: 0, png: 0, wmts: 1, other: 0, pbf: 1 });
-    o(clientStats?.tileSet).deepEquals({ aerial: 0, aerialIndividual: 0, topo50: 2, direct: 0 });
+    o(clientStats?.tileSet).deepEquals({ topo50: 2 });
   });
 });
diff --git a/packages/lambda-analytics/src/file.process.ts b/packages/lambda-analytics/src/file.process.ts
@@ -1,4 +1,4 @@
-import { LogType, getUrlHost, fsa } from '@basemaps/shared';
+import { LogType, getUrlHost, fsa, isValidApiKey } from '@basemaps/shared';
 import { createInterface, Interface } from 'readline';
 import { createGunzip } from 'zlib';
 import { LogStats } from './stats.js';
@@ -30,14 +30,13 @@ export const FileProcess = {
 
       // Ignore requests which are not tile requests
       if (!uri.startsWith('/v1')) continue;
-      if (!query.startsWith('api=')) {
-        logger.debug({ uri, query }, 'NoApiKey');
-        continue;
+      const search = new URLSearchParams(query);
+      const apiKey = search.get('api');
+      const apiValid = isValidApiKey(apiKey);
+      if (apiValid.valid || apiValid.message === 'expired') {
+        stats.track(apiKey as string, referer, uri.toLowerCase(), parseInt(status), hit);
       }
-      // TODO This could be switched to a QueryString parser
-      const endIndex = query.indexOf('&');
-      const apiKey = query.slice('api='.length, endIndex === -1 ? query.length : endIndex);
-      stats.track(apiKey, referer, uri.toLowerCase(), parseInt(status), hit);
+      // TODO should we track non apikeys
     }
   },
 };
diff --git a/packages/lambda-analytics/src/stats.ts b/packages/lambda-analytics/src/stats.ts
@@ -1,4 +1,5 @@
-import { createHash } from 'crypto';
+import { sha256base58 } from '@basemaps/config';
+import { TileMatrixSets } from '@basemaps/geo';
 
 /** Changing this number will cause all the statistics to be recomputed */
 export const RollupVersion = 1;
@@ -26,17 +27,17 @@ export interface TileRequestStats {
   status: Record<number, number>;
   /** Tile file extensions used */
   extension: { webp: number; jpeg: number; png: number; wmts: number; pbf: number; other: number };
-  /** Projections used */
-  projection: { 2193: number; 3857: number };
+  /** Tile Matrixes used */
+  tileMatrix: Record<string, number>;
   /** Tilesets accessed */
-  tileSet: { aerial: number; aerialIndividual: number; topo50: number; direct: number };
+  tileSet: Record<string, number>;
   /** How was this rollup generated */
   generated: { v: number; hash?: string; version?: string };
 }
 
 function newStat(timestamp: string, api: string, referer: string | undefined): TileRequestStats {
   return {
-    statId: timestamp + '_' + createHash('sha3-256').update(`${api}_${referer}`).digest('hex'),
+    statId: timestamp + '_' + sha256base58(`${api}_${referer}`),
     timestamp,
     api,
     referer,
@@ -45,8 +46,8 @@ function newStat(timestamp: string, api: string, referer: string | undefined): T
     status: {},
     cache: { hit: 0, miss: 0 },
     extension: { webp: 0, jpeg: 0, png: 0, wmts: 0, pbf: 0, other: 0 },
-    projection: { 2193: 0, 3857: 0 },
-    tileSet: { aerial: 0, aerialIndividual: 0, topo50: 0, direct: 0 },
+    tileSet: {},
+    tileMatrix: {},
     generated: {
       v: RollupVersion,
       hash: process.env.GIT_HASH,
@@ -74,24 +75,19 @@ function track(stat: TileRequestStats, uri: string, status: number, isHit: boole
     stat.extension.wmts++;
   } else stat.extension.other++;
 
-  const [, , , tileSet, projection] = uri.split('/');
+  const [, , , tileSet, projectionStr] = uri.split('/');
   // no projection means this url is weirdly formatted
-  if (projection == null) return;
+  if (projectionStr == null) return;
+
+  const tileMatrix = TileMatrixSets.find(projectionStr);
+  if (tileMatrix == null) return;
 
   // Projection
-  if (projection.includes('3857')) stat.projection['3857']++;
-  else if (projection.includes('2193')) stat.projection['2193']++;
-  else return; // Unknown projection this is likely not a tile
+  stat.tileMatrix[tileMatrix.identifier] = (stat.tileMatrix[tileMatrix.identifier] ?? 0) + 1;
 
   // Tile set
-  if (tileSet === 'aerial') stat.tileSet.aerial++;
-  else if (tileSet === 'topo50') stat.tileSet.topo50++;
-  // TODO do we want to get the real names for these
-  else if (tileSet.startsWith('aerial:')) stat.tileSet.aerialIndividual++;
-  else if (tileSet.startsWith('01')) stat.tileSet.direct++;
-  else {
-    // TODO do we care about these other tile sets
-  }
+  if (tileSet.startsWith('01')) stat.tileSet['byId'] = (stat.tileSet['byId'] ?? 0) + 1;
+  else stat.tileSet[tileSet] = (stat.tileSet[tileSet] ?? 0) + 1;
 }
 
 export class LogStats {

diff --git a/packages/lambda-analytics/tsconfig.json b/packages/lambda-analytics/tsconfig.json
@@ -5,5 +5,5 @@
     "outDir": "./build"
   },
   "include": ["src/**/*"],
-  "references": [{ "path": "../shared" }]
+  "references": [{ "path": "../config" }, { "path": "../geo" }, { "path": "../shared" }]
 }
diff --git a/packages/lambda-tiler/src/util/validate.ts b/packages/lambda-tiler/src/util/validate.ts
@@ -1,8 +1,7 @@
 import { ImageFormat, TileMatrixSet, TileMatrixSets, VectorFormat } from '@basemaps/geo';
-import { Const, Projection } from '@basemaps/shared';
+import { Const, isValidApiKey, Projection } from '@basemaps/shared';
 import { getImageFormat } from '@basemaps/tiler';
 import { LambdaHttpRequest, LambdaHttpResponse } from '@linzjs/lambda';
-import * as ulid from 'ulid';
 import { TileXyzGet } from '../routes/tile.xyz';
 
 export interface TileXyz {
@@ -16,31 +15,6 @@ export interface TileMatrixRequest {
   Params: { tileMatrix?: string };
 }
 
-const OneHourMs = 60 * 60 * 1000;
-const OneDayMs = 24 * OneHourMs;
-const MaxApiAgeMs = 91 * OneDayMs;
-
-export interface ApiKeyStatus {
-  valid: boolean;
-  message: 'ok' | 'malformed' | 'missing' | 'expired';
-}
-
-export function isValidApiKey(apiKey?: string | null): ApiKeyStatus {
-  if (apiKey == null) return { valid: false, message: 'missing' };
-  if (!apiKey.startsWith('c') && !apiKey.startsWith('d')) return { valid: false, message: 'malformed' };
-  const ulidId = apiKey.slice(1).toUpperCase();
-  try {
-    const ulidTime = ulid.decodeTime(ulidId);
-    if (apiKey.startsWith('d')) return { valid: true, message: 'ok' };
-
-    if (Date.now() - ulidTime > MaxApiAgeMs) return { valid: false, message: 'expired' };
-  } catch (e) {
-    return { valid: false, message: 'malformed' };
-  }
-
-  return { valid: true, message: 'ok' };
-}
-
 export const Validate = {
   /**
    * Validate that the api key exists and is valid

diff --git a/packages/lambda-tiler/tsconfig.json b/packages/lambda-tiler/tsconfig.json
@@ -6,6 +6,7 @@
   },
   "include": ["src/**/*"],
   "references": [
+    { "path": "../config" },
     { "path": "../shared" },
     { "path": "../geo" },
     { "path": "../tiler" },

diff --git a/packages/shared/src/api.ts b/packages/shared/src/api.ts
@@ -4,6 +4,7 @@ const hasLocalStorage = (): boolean => typeof localStorage !== 'undefined';
 export const OneDayMs = 24 * 60 * 60 * 1000;
 /** Generate a new api key for the user every 30 days */
 const ApiKeyExpireMs = 30 * OneDayMs;
+const ApiKeyMaxAgeMs = 91 * OneDayMs;
 
 function newApiKey(): string {
   const newKey = 'c' + ulid().toLowerCase();
@@ -27,3 +28,31 @@ export function getApiKey(): string {
     return newApiKey();
   }
 }
+
+export type ApiKeyStatus = ApiKeyStatusValid | ApiKeyStatusInvalid;
+
+export interface ApiKeyStatusValid {
+  valid: true;
+  key: string;
+}
+
+export interface ApiKeyStatusInvalid {
+  valid: false;
+  message: 'malformed' | 'missing' | 'expired';
+}
+
+export function isValidApiKey(apiKey?: string | null): ApiKeyStatus {
+  if (apiKey == null) return { valid: false, message: 'missing' };
+  if (!apiKey.startsWith('c') && !apiKey.startsWith('d')) return { valid: false, message: 'malformed' };
+  const ulidId = apiKey.slice(1).toUpperCase();
+  try {
+    const ulidTime = decodeTime(ulidId);
+    if (apiKey.startsWith('d')) return { valid: true, key: apiKey };
+
+    if (Date.now() - ulidTime > ApiKeyMaxAgeMs) return { valid: false, message: 'expired' };
+  } catch (e) {
+    return { valid: false, message: 'malformed' };
+  }
+
+  return { valid: true, key: apiKey };
+}