From 9f84285ed203bf3443f288b20482cb18d6b13c40 Mon Sep 17 00:00:00 2001 From: Blayne Chard Date: Mon, 1 Aug 2022 13:25:45 +1200 Subject: [PATCH] fix(lambda-analytics): do not track invalid api keys BM-642 (#2392) * fix(lambda-analytics): do not track invalid api keys BM-642 * refactor: fixup lint * refactor: mix missing imports * refactor: formatting --- packages/lambda-analytics/package.json | 2 ++ .../src/__tests__/file.process.test.ts | 15 ++++---- packages/lambda-analytics/src/file.process.ts | 15 ++++---- packages/lambda-analytics/src/stats.ts | 36 +++++++++---------- packages/lambda-analytics/tsconfig.json | 2 +- packages/lambda-tiler/src/util/validate.ts | 28 +-------------- packages/lambda-tiler/tsconfig.json | 1 + packages/shared/src/api.ts | 29 +++++++++++++++ 8 files changed, 65 insertions(+), 63 deletions(-) diff --git a/packages/lambda-analytics/package.json b/packages/lambda-analytics/package.json index c29bdd364..7264a7faa 100644 --- a/packages/lambda-analytics/package.json +++ b/packages/lambda-analytics/package.json @@ -18,6 +18,8 @@ }, "license": "MIT", "dependencies": { + "@basemaps/config": "^6.32.1", + "@basemaps/geo": "^6.32.1", "@basemaps/shared": "^6.32.1" }, "scripts": { diff --git a/packages/lambda-analytics/src/__tests__/file.process.test.ts b/packages/lambda-analytics/src/__tests__/file.process.test.ts index 228fc9a24..181968963 100644 --- a/packages/lambda-analytics/src/__tests__/file.process.test.ts +++ b/packages/lambda-analytics/src/__tests__/file.process.test.ts @@ -2,9 +2,10 @@ import { LogConfig } from '@basemaps/shared'; import o from 'ospec'; import { FileProcess } from '../file.process.js'; import { LogStats } from '../stats.js'; +import { ulid } from 'ulid'; -const DevApiKey = 'dThisIsNotAKey'; -const ClientApiKey = 'cThisIsNotAKey'; +const DevApiKey = 'd' + ulid().toLowerCase(); +const ClientApiKey = 'c' + ulid().toLowerCase(); export const ExampleLogs = `#Version: 1.0 #Fields: date time x-edge-location sc-bytes c-ip cs-method cs(Host) cs-uri-stem sc-status cs(Referer) cs(User-Agent) cs-uri-query cs(Cookie) x-edge-result-type x-edge-request-id x-host-header cs-protocol cs-bytes time-taken x-forwarded-for ssl-protocol ssl-cipher x-edge-response-result-type cs-protocol-version fle-status fle-encrypted-fields c-port time-to-first-byte x-edge-detailed-result-type sc-content-type sc-content-len sc-range-start sc-range-end @@ -54,7 +55,7 @@ o.spec('FileProcess', () => { o(apiStats?.apiType).equals('d'); o(apiStats?.total).equals(1); o(apiStats?.cache).deepEquals({ hit: 1, miss: 0 }); - o(apiStats?.projection).deepEquals({ 2193: 0, 3857: 1 }); + o(apiStats?.tileMatrix).deepEquals({ WebMercatorQuad: 1 }); }); o('should extract and track a bunch of hits', async () => { @@ -71,15 +72,15 @@ o.spec('FileProcess', () => { o(devStats?.total).equals(3); o(devStats?.apiType).equals('d'); o(devStats?.cache).deepEquals({ hit: 2, miss: 1 }); - o(devStats?.projection).deepEquals({ 2193: 0, 3857: 3 }); + o(devStats?.tileMatrix).deepEquals({ WebMercatorQuad: 3 }); o(devStats?.extension).deepEquals({ webp: 1, jpeg: 1, png: 1, wmts: 0, other: 0, pbf: 0 }); - o(devStats?.tileSet).deepEquals({ aerial: 2, aerialIndividual: 0, topo50: 1, direct: 0 }); + o(devStats?.tileSet).deepEquals({ aerial: 2, topo50: 1 }); o(clientStats?.total).equals(2); o(clientStats?.apiType).equals('c'); o(clientStats?.cache).deepEquals({ hit: 2, miss: 0 }); - o(clientStats?.projection).deepEquals({ 2193: 0, 3857: 2 }); + o(clientStats?.tileMatrix).deepEquals({ WebMercatorQuad: 2 }); o(clientStats?.extension).deepEquals({ webp: 0, jpeg: 0, png: 0, wmts: 1, other: 0, pbf: 1 }); - o(clientStats?.tileSet).deepEquals({ aerial: 0, aerialIndividual: 0, topo50: 2, direct: 0 }); + o(clientStats?.tileSet).deepEquals({ topo50: 2 }); }); }); diff --git a/packages/lambda-analytics/src/file.process.ts b/packages/lambda-analytics/src/file.process.ts index 434db4193..103273138 100644 --- a/packages/lambda-analytics/src/file.process.ts +++ b/packages/lambda-analytics/src/file.process.ts @@ -1,4 +1,4 @@ -import { LogType, getUrlHost, fsa } from '@basemaps/shared'; +import { LogType, getUrlHost, fsa, isValidApiKey } from '@basemaps/shared'; import { createInterface, Interface } from 'readline'; import { createGunzip } from 'zlib'; import { LogStats } from './stats.js'; @@ -30,14 +30,13 @@ export const FileProcess = { // Ignore requests which are not tile requests if (!uri.startsWith('/v1')) continue; - if (!query.startsWith('api=')) { - logger.debug({ uri, query }, 'NoApiKey'); - continue; + const search = new URLSearchParams(query); + const apiKey = search.get('api'); + const apiValid = isValidApiKey(apiKey); + if (apiValid.valid || apiValid.message === 'expired') { + stats.track(apiKey as string, referer, uri.toLowerCase(), parseInt(status), hit); } - // TODO This could be switched to a QueryString parser - const endIndex = query.indexOf('&'); - const apiKey = query.slice('api='.length, endIndex === -1 ? query.length : endIndex); - stats.track(apiKey, referer, uri.toLowerCase(), parseInt(status), hit); + // TODO should we track non apikeys } }, }; diff --git a/packages/lambda-analytics/src/stats.ts b/packages/lambda-analytics/src/stats.ts index 3dcbbfc21..2de157bb4 100644 --- a/packages/lambda-analytics/src/stats.ts +++ b/packages/lambda-analytics/src/stats.ts @@ -1,4 +1,5 @@ -import { createHash } from 'crypto'; +import { sha256base58 } from '@basemaps/config'; +import { TileMatrixSets } from '@basemaps/geo'; /** Changing this number will cause all the statistics to be recomputed */ export const RollupVersion = 1; @@ -26,17 +27,17 @@ export interface TileRequestStats { status: Record; /** Tile file extensions used */ extension: { webp: number; jpeg: number; png: number; wmts: number; pbf: number; other: number }; - /** Projections used */ - projection: { 2193: number; 3857: number }; + /** Tile Matrixes used */ + tileMatrix: Record; /** Tilesets accessed */ - tileSet: { aerial: number; aerialIndividual: number; topo50: number; direct: number }; + tileSet: Record; /** How was this rollup generated */ generated: { v: number; hash?: string; version?: string }; } function newStat(timestamp: string, api: string, referer: string | undefined): TileRequestStats { return { - statId: timestamp + '_' + createHash('sha3-256').update(`${api}_${referer}`).digest('hex'), + statId: timestamp + '_' + sha256base58(`${api}_${referer}`), timestamp, api, referer, @@ -45,8 +46,8 @@ function newStat(timestamp: string, api: string, referer: string | undefined): T status: {}, cache: { hit: 0, miss: 0 }, extension: { webp: 0, jpeg: 0, png: 0, wmts: 0, pbf: 0, other: 0 }, - projection: { 2193: 0, 3857: 0 }, - tileSet: { aerial: 0, aerialIndividual: 0, topo50: 0, direct: 0 }, + tileSet: {}, + tileMatrix: {}, generated: { v: RollupVersion, hash: process.env.GIT_HASH, @@ -74,24 +75,19 @@ function track(stat: TileRequestStats, uri: string, status: number, isHit: boole stat.extension.wmts++; } else stat.extension.other++; - const [, , , tileSet, projection] = uri.split('/'); + const [, , , tileSet, projectionStr] = uri.split('/'); // no projection means this url is weirdly formatted - if (projection == null) return; + if (projectionStr == null) return; + + const tileMatrix = TileMatrixSets.find(projectionStr); + if (tileMatrix == null) return; // Projection - if (projection.includes('3857')) stat.projection['3857']++; - else if (projection.includes('2193')) stat.projection['2193']++; - else return; // Unknown projection this is likely not a tile + stat.tileMatrix[tileMatrix.identifier] = (stat.tileMatrix[tileMatrix.identifier] ?? 0) + 1; // Tile set - if (tileSet === 'aerial') stat.tileSet.aerial++; - else if (tileSet === 'topo50') stat.tileSet.topo50++; - // TODO do we want to get the real names for these - else if (tileSet.startsWith('aerial:')) stat.tileSet.aerialIndividual++; - else if (tileSet.startsWith('01')) stat.tileSet.direct++; - else { - // TODO do we care about these other tile sets - } + if (tileSet.startsWith('01')) stat.tileSet['byId'] = (stat.tileSet['byId'] ?? 0) + 1; + else stat.tileSet[tileSet] = (stat.tileSet[tileSet] ?? 0) + 1; } export class LogStats { diff --git a/packages/lambda-analytics/tsconfig.json b/packages/lambda-analytics/tsconfig.json index 5b3761dd8..cef90e6d5 100644 --- a/packages/lambda-analytics/tsconfig.json +++ b/packages/lambda-analytics/tsconfig.json @@ -5,5 +5,5 @@ "outDir": "./build" }, "include": ["src/**/*"], - "references": [{ "path": "../shared" }] + "references": [{ "path": "../config" }, { "path": "../geo" }, { "path": "../shared" }] } diff --git a/packages/lambda-tiler/src/util/validate.ts b/packages/lambda-tiler/src/util/validate.ts index 77115bf69..961a5ff4a 100644 --- a/packages/lambda-tiler/src/util/validate.ts +++ b/packages/lambda-tiler/src/util/validate.ts @@ -1,8 +1,7 @@ import { ImageFormat, TileMatrixSet, TileMatrixSets, VectorFormat } from '@basemaps/geo'; -import { Const, Projection } from '@basemaps/shared'; +import { Const, isValidApiKey, Projection } from '@basemaps/shared'; import { getImageFormat } from '@basemaps/tiler'; import { LambdaHttpRequest, LambdaHttpResponse } from '@linzjs/lambda'; -import * as ulid from 'ulid'; import { TileXyzGet } from '../routes/tile.xyz'; export interface TileXyz { @@ -16,31 +15,6 @@ export interface TileMatrixRequest { Params: { tileMatrix?: string }; } -const OneHourMs = 60 * 60 * 1000; -const OneDayMs = 24 * OneHourMs; -const MaxApiAgeMs = 91 * OneDayMs; - -export interface ApiKeyStatus { - valid: boolean; - message: 'ok' | 'malformed' | 'missing' | 'expired'; -} - -export function isValidApiKey(apiKey?: string | null): ApiKeyStatus { - if (apiKey == null) return { valid: false, message: 'missing' }; - if (!apiKey.startsWith('c') && !apiKey.startsWith('d')) return { valid: false, message: 'malformed' }; - const ulidId = apiKey.slice(1).toUpperCase(); - try { - const ulidTime = ulid.decodeTime(ulidId); - if (apiKey.startsWith('d')) return { valid: true, message: 'ok' }; - - if (Date.now() - ulidTime > MaxApiAgeMs) return { valid: false, message: 'expired' }; - } catch (e) { - return { valid: false, message: 'malformed' }; - } - - return { valid: true, message: 'ok' }; -} - export const Validate = { /** * Validate that the api key exists and is valid diff --git a/packages/lambda-tiler/tsconfig.json b/packages/lambda-tiler/tsconfig.json index d0bfd4355..f1f286f03 100644 --- a/packages/lambda-tiler/tsconfig.json +++ b/packages/lambda-tiler/tsconfig.json @@ -6,6 +6,7 @@ }, "include": ["src/**/*"], "references": [ + { "path": "../config" }, { "path": "../shared" }, { "path": "../geo" }, { "path": "../tiler" }, diff --git a/packages/shared/src/api.ts b/packages/shared/src/api.ts index 2e2b6c717..78ef0d53d 100644 --- a/packages/shared/src/api.ts +++ b/packages/shared/src/api.ts @@ -4,6 +4,7 @@ const hasLocalStorage = (): boolean => typeof localStorage !== 'undefined'; export const OneDayMs = 24 * 60 * 60 * 1000; /** Generate a new api key for the user every 30 days */ const ApiKeyExpireMs = 30 * OneDayMs; +const ApiKeyMaxAgeMs = 91 * OneDayMs; function newApiKey(): string { const newKey = 'c' + ulid().toLowerCase(); @@ -27,3 +28,31 @@ export function getApiKey(): string { return newApiKey(); } } + +export type ApiKeyStatus = ApiKeyStatusValid | ApiKeyStatusInvalid; + +export interface ApiKeyStatusValid { + valid: true; + key: string; +} + +export interface ApiKeyStatusInvalid { + valid: false; + message: 'malformed' | 'missing' | 'expired'; +} + +export function isValidApiKey(apiKey?: string | null): ApiKeyStatus { + if (apiKey == null) return { valid: false, message: 'missing' }; + if (!apiKey.startsWith('c') && !apiKey.startsWith('d')) return { valid: false, message: 'malformed' }; + const ulidId = apiKey.slice(1).toUpperCase(); + try { + const ulidTime = decodeTime(ulidId); + if (apiKey.startsWith('d')) return { valid: true, key: apiKey }; + + if (Date.now() - ulidTime > ApiKeyMaxAgeMs) return { valid: false, message: 'expired' }; + } catch (e) { + return { valid: false, message: 'malformed' }; + } + + return { valid: true, key: apiKey }; +}