Skip to content

Commit

Permalink
feat(lambda-tiler): log cache hit percentages (#2368)
Browse files Browse the repository at this point in the history 
* feat(lambda-tiler): log cache hit percentages

* feat(lambda-tiler): log source cache stats

* refactor: cleanup

* refactor: only one version of pino
  • Loading branch information
@blacha
blacha committed Jul 26, 2022
1 parent 93e6e2a commit 3f7bf0c
Show file tree
Hide file tree
Showing 6 changed files with 100 additions and 62 deletions.
2 changes: 1 addition & 1 deletion packages/lambda-cog/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@
"@basemaps/lambda": "^6.7.0",
"@basemaps/shared": "^6.31.0",
"@chunkd/fs": "^8.4.0",
"@linzjs/lambda": "^3.2.0"
"@linzjs/lambda": "^3.2.1"
},
"bundle": {
"entry": "src/index.ts",
Expand Down
2 changes: 1 addition & 1 deletion packages/lambda-tiler/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@
"@cogeotiff/core": "^7.0.0",
"@cotar/core": "^5.4.0",
"@linzjs/geojson": "^6.28.1",
"@linzjs/lambda": "^3.2.0",
"@linzjs/lambda": "^3.2.1",
"p-limit": "^4.0.0",
"path-to-regexp": "^6.1.0",
"pixelmatch": "^5.1.0",
Expand Down
11 changes: 11 additions & 0 deletions packages/lambda-tiler/src/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import { styleJsonGet } from './routes/tile.style.json.js';
import { wmtsCapabilitiesGet } from './routes/tile.wmts.js';
import { tileXyzGet } from './routes/tile.xyz.js';
import { versionGet } from './routes/version.js';
import { CoSources } from './util/source.cache.js';
import { St } from './util/source.tracer.js';

export const handler = lf.http(LogConfig.get());
Expand All @@ -29,6 +30,16 @@ handler.router.hook('response', (req, res) => {
req.set('requestCount', St.requests.length);
}

// Log the source cache hit/miss ratio
req.set('sources', {
hits: CoSources.cache.hits,
misses: CoSources.cache.misses,
size: CoSources.cache.currentSize,
resets: CoSources.cache.resets,
cacheA: CoSources.cache.cacheA.size,
cacheB: CoSources.cache.cacheB.size,
});

// Ensure CORS response headers are set
res.header('Access-Control-Allow-Origin', '*');
});
Expand Down
17 changes: 15 additions & 2 deletions packages/lambda-tiler/src/util/swapping.lru.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,10 @@ export class SwappingLru<T extends { size: number }> {
cacheB: Map<string, T> = new Map();
maxSize: number;

hits = 0;
misses = 0;
resets = 0;

_lastCheckedAt = -1;

constructor(maxSize: number) {
Expand All @@ -11,10 +15,18 @@ export class SwappingLru<T extends { size: number }> {

get(id: string): T | null {
const cacheA = this.cacheA.get(id);
if (cacheA) return cacheA;
if (cacheA) {
this.hits++;
return cacheA;
}

const cacheB = this.cacheB.get(id);
if (cacheB == null) return null;
if (cacheB == null) {
this.misses++;
return null;
}

this.hits++;
// If a object is still useful move it into the main cache
this.cacheA.set(id, cacheB);
this.cacheB.delete(id);
Expand All @@ -37,6 +49,7 @@ export class SwappingLru<T extends { size: number }> {
this._lastCheckedAt = Date.now();
if (this.maxSize <= 0) return;
if (this.currentSize <= this.maxSize) return;
this.resets++;
this.cacheB = this.cacheA;
this.cacheA = new Map();
}
Expand Down
2 changes: 1 addition & 1 deletion packages/shared/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@
"@linzjs/metrics": "^6.28.1",
"aws-sdk": "^2.890.0",
"entities": "^4.3.0",
"pino": "^7.5.0",
"pino": "^8.3.0",
"proj4": "^2.8.0",
"sax": "^1.2.4",
"source-map-support": "^0.5.19",
Expand Down
128 changes: 71 additions & 57 deletions yarn.lock
View file
Original file line number Diff line number Diff line change
Expand Up @@ -945,14 +945,14 @@
npmlog "^6.0.2"
write-file-atomic "^3.0.3"

"@linzjs/lambda@^3.2.0":
version "3.2.0"
resolved "https://registry.yarnpkg.com/@linzjs/lambda/-/lambda-3.2.0.tgz#fd41931a2703a688e595a90fa435aefbcdbb0a20"
integrity sha512-65jW+CD79MrtXTxc1hLV6kARZ6ZkNCgpTX+9h/mtEMr1gmMp5hL5kWzloRL8g/qthNv0YEheTiQCabS93eNAyQ==
"@linzjs/lambda@^3.2.1":
version "3.2.1"
resolved "https://registry.yarnpkg.com/@linzjs/lambda/-/lambda-3.2.1.tgz#61264b4a8ee5c3d0a7fc7687bd7f4c1ddb6bf4c8"
integrity sha512-gtNeumdN0/aqS0WJlwNoN9otfcL1kVGCppU9hLaco+sK+uX9rUdzOiiaiCCtJ3qODUwZwv7onP8lTpgSdm5hCQ==
dependencies:
"@linzjs/metrics" "^6.21.1"
find-my-way "^7.0.0"
pino "^7.9.1"
pino "^8.3.0"
ulid "^2.3.0"

"@linzjs/lui@^10.11.3":
Expand Down Expand Up @@ -1675,6 +1675,13 @@ abbrev@1:
resolved "https://registry.yarnpkg.com/abbrev/-/abbrev-1.1.1.tgz#f8f2c887ad10bf67f634f005b6987fed3179aac8"
integrity sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==

abort-controller@^3.0.0:
version "3.0.0"
resolved "https://registry.yarnpkg.com/abort-controller/-/abort-controller-3.0.0.tgz#eaf54d53b62bae4138e809ca225c8439a6efb392"
integrity sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==
dependencies:
event-target-shim "^5.0.0"

abstract-logging@^2.0.0:
version "2.0.1"
resolved "https://registry.yarnpkg.com/abstract-logging/-/abstract-logging-2.0.1.tgz#6b0c371df212db7129b57d2e7fcf282b8bf1c839"
Expand Down Expand Up @@ -3023,16 +3030,6 @@ duplexer@^0.1.1:
resolved "https://registry.yarnpkg.com/duplexer/-/duplexer-0.1.1.tgz#ace6ff808c1ce66b57d1ebf97977acb02334cfc1"
integrity sha1-rOb/gIwc5mtX0ev5eXessCM0z8E=

duplexify@^4.1.2:
version "4.1.2"
resolved "https://registry.yarnpkg.com/duplexify/-/duplexify-4.1.2.tgz#18b4f8d28289132fa0b9573c898d9f903f81c7b0"
integrity sha512-fz3OjcNCHmRP12MJoZMPglx8m4rrFP8rovnk4vT8Fs+aonZoCwGg10dSsQsfP/E62eZcPTMSMP6686fu9Qlqtw==
dependencies:
end-of-stream "^1.4.1"
inherits "^2.0.3"
readable-stream "^3.1.1"
stream-shift "^1.0.0"

earcut@^2.2.3:
version "2.2.3"
resolved "https://registry.yarnpkg.com/earcut/-/earcut-2.2.3.tgz#d44ced2ff5a18859568e327dd9c7d46b16f55cf4"
Expand Down Expand Up @@ -3415,6 +3412,11 @@ etag@~1.8.1:
resolved "https://registry.yarnpkg.com/etag/-/etag-1.8.1.tgz#41ae2eeb65efa62268aebfea83ac7d79299b0887"
integrity sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc=

event-target-shim@^5.0.0:
version "5.0.1"
resolved "https://registry.yarnpkg.com/event-target-shim/-/event-target-shim-5.0.1.tgz#5d4d3ebdf9583d63a5333ce2deb7480ab2b05789"
integrity sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==

eventemitter3@^4.0.4:
version "4.0.7"
resolved "https://registry.yarnpkg.com/eventemitter3/-/eventemitter3-4.0.7.tgz#2de9b68f6528d5644ef5c59526a1b4a07306169f"
Expand Down Expand Up @@ -3529,6 +3531,11 @@ fast-redact@^3.0.0:
resolved "https://registry.yarnpkg.com/fast-redact/-/fast-redact-3.0.0.tgz#ac2f9e36c9f4976f5db9fb18c6ffbaf308cf316d"
integrity sha512-a/S/Hp6aoIjx7EmugtzLqXmcNsyFszqbt6qQ99BdG61QjBZF6shNis0BYR6TsZOQ1twYc0FN2Xdhwwbv6+KD0w==

fast-redact@^3.1.1:
version "3.1.1"
resolved "https://registry.yarnpkg.com/fast-redact/-/fast-redact-3.1.1.tgz#790fcff8f808c2e12fabbfb2be5cb2deda448fa0"
integrity sha512-odVmjC8x8jNeMZ3C+rPMESzXVSEU8tSWSHv9HFxP2mm89G/1WwqhrerJDQm9Zus8X6aoRgQDThKqptdNA6bt+A==

fast-safe-stringify@^2.0.8:
version "2.0.8"
resolved "https://registry.yarnpkg.com/fast-safe-stringify/-/fast-safe-stringify-2.0.8.tgz#dc2af48c46cf712b683e849b2bbd446b32de936f"
Expand Down Expand Up @@ -5568,10 +5575,10 @@ object-assign@^4.0.1, object-assign@^4.1.1:
resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.1.tgz#2109adc7965887cfc05cbbd442cac8bfbb360863"
integrity sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM=

on-exit-leak-free@^0.2.0:
version "0.2.0"
resolved "https://registry.yarnpkg.com/on-exit-leak-free/-/on-exit-leak-free-0.2.0.tgz#b39c9e3bf7690d890f4861558b0d7b90a442d209"
integrity sha512-dqaz3u44QbRXQooZLTUKU41ZrzYrcvLISVgbrzbyCMxpmSLJvZ3ZamIJIZ29P6OhZIkNIQKosdeM6t1LYbA9hg==
on-exit-leak-free@^2.1.0:
version "2.1.0"
resolved "https://registry.yarnpkg.com/on-exit-leak-free/-/on-exit-leak-free-2.1.0.tgz#5c703c968f7e7f851885f6459bf8a8a57edc9cc4"
integrity sha512-VuCaZZAjReZ3vUwgOB8LxAosIurDiAW0s13rI1YwmaP++jvcxP77AWoQvenZebpCA2m8WC1/EosPYPMjnRAp/w==

on-finished@~2.3.0:
version "2.3.0"
Expand Down Expand Up @@ -5959,23 +5966,23 @@ pinkie@^2.0.0:
resolved "https://registry.yarnpkg.com/pinkie/-/pinkie-2.0.4.tgz#72556b80cfa0d48a974e80e77248e80ed4f7f870"
integrity sha1-clVrgM+g1IqXToDnckjoDtT3+HA=

pino-abstract-transport@v0.5.0:
version "0.5.0"
resolved "https://registry.yarnpkg.com/pino-abstract-transport/-/pino-abstract-transport-0.5.0.tgz#4b54348d8f73713bfd14e3dc44228739aa13d9c0"
integrity sha512-+KAgmVeqXYbTtU2FScx1XS3kNyfZ5TrXY07V96QnUSFqo2gAqlvmaxH67Lj7SWazqsMabf+58ctdTcBgnOLUOQ==
pino-abstract-transport@v1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/pino-abstract-transport/-/pino-abstract-transport-1.0.0.tgz#cc0d6955fffcadb91b7b49ef220a6cc111d48bb3"
integrity sha512-c7vo5OpW4wIS42hUVcT5REsL8ZljsUfBjqV/e2sFxmFEFZiq1XLUp5EYLtuDH6PEHq9W1egWqRbnLUP5FuZmOA==
dependencies:
duplexify "^4.1.2"
readable-stream "^4.0.0"
split2 "^4.0.0"

pino-std-serializers@^3.1.0:
version "3.1.0"
resolved "https://registry.yarnpkg.com/pino-std-serializers/-/pino-std-serializers-3.1.0.tgz#fe62d339aeef359d9d7bdf61f4e4b2d7be8c73d4"
integrity sha512-Fk1pxhX6tuW4ozRDFw5Mz/IHQV5wXYXZwjG/gOpTNPCbYEMeNW9VnKAEu1428CwAQVupFruOr1vkC/ufmcwedA==

pino-std-serializers@^4.0.0:
version "4.0.0"
resolved "https://registry.yarnpkg.com/pino-std-serializers/-/pino-std-serializers-4.0.0.tgz#1791ccd2539c091ae49ce9993205e2cd5dbba1e2"
integrity sha512-cK0pekc1Kjy5w9V2/n+8MkZwusa6EyyxfeQCB799CQRhRt/CqYKiWs5adeu8Shve2ZNffvfC/7J64A2PJo1W/Q==
pino-std-serializers@^6.0.0:
version "6.0.0"
resolved "https://registry.yarnpkg.com/pino-std-serializers/-/pino-std-serializers-6.0.0.tgz#4c20928a1bafca122fdc2a7a4a171ca1c5f9c526"
integrity sha512-mMMOwSKrmyl+Y12Ri2xhH1lbzQxwwpuru9VjyJpgFIH4asSj88F2csdMwN6+M5g1Ll4rmsYghHLQJw81tgZ7LQ==

pino@^6.13.0:
version "6.14.0"
Expand All @@ -5990,22 +5997,22 @@ pino@^6.13.0:
quick-format-unescaped "^4.0.3"
sonic-boom "^1.0.2"

pino@^7.5.0, pino@^7.9.1:
version "7.11.0"
resolved "https://registry.yarnpkg.com/pino/-/pino-7.11.0.tgz#0f0ea5c4683dc91388081d44bff10c83125066f6"
integrity sha512-dMACeu63HtRLmCG8VKdy4cShCPKaYDR4youZqoSWLxl5Gu99HUw8bw75thbPv9Nip+H+QYX8o3ZJbTdVZZ2TVg==
pino@^8.3.0:
version "8.3.0"
resolved "https://registry.yarnpkg.com/pino/-/pino-8.3.0.tgz#13e400df4ef39ce716b33b8e5486fb8e24a30c45"
integrity sha512-CUOOU5sYgyLVijf7qsOWkPLOdzIXaVE9Oyl6zAkjgJn30uGBFUtrRaaCzbtOSJ1I6BqfanBqBttbazEJIwHkJg==
dependencies:
atomic-sleep "^1.0.0"
fast-redact "^3.0.0"
on-exit-leak-free "^0.2.0"
pino-abstract-transport v0.5.0
pino-std-serializers "^4.0.0"
process-warning "^1.0.0"
fast-redact "^3.1.1"
on-exit-leak-free "^2.1.0"
pino-abstract-transport v1.0.0
pino-std-serializers "^6.0.0"
process-warning "^2.0.0"
quick-format-unescaped "^4.0.3"
real-require "^0.1.0"
safe-stable-stringify "^2.1.0"
sonic-boom "^2.2.1"
thread-stream "^0.15.1"
safe-stable-stringify "^2.3.1"
sonic-boom "^3.1.0"
thread-stream "^2.0.0"

pixelmatch@^5.1.0:
version "5.2.1"
Expand Down Expand Up @@ -6132,6 +6139,11 @@ process-warning@^1.0.0:
resolved "https://registry.yarnpkg.com/process-warning/-/process-warning-1.0.0.tgz#980a0b25dc38cd6034181be4b7726d89066b4616"
integrity sha512-du4wfLyj4yCZq1VupnVSZmRsPJsNuxoDQFdCFHLaYiEbFBD7QE0a+I4D7hOxrVnh78QE/YipFAj9lXHiXocV+Q==

process-warning@^2.0.0:
version "2.0.0"
resolved "https://registry.yarnpkg.com/process-warning/-/process-warning-2.0.0.tgz#341dbeaac985b90a04ebcd844d50097c7737b2ee"
integrity sha512-+MmoAXoUX+VTHAlwns0h+kFUWFs/3FZy+ZuchkgjyOu3oioLAo2LB5aCfKPh2+P9O18i3m43tUEv3YqttSy0Ww==

proj4@^2.8.0:
version "2.8.0"
resolved "https://registry.yarnpkg.com/proj4/-/proj4-2.8.0.tgz#b2cb8f3ccd56d4dcc7c3e46155cd02caa804b170"
Expand Down Expand Up @@ -6445,6 +6457,13 @@ readable-stream@^2.0.0, readable-stream@~2.3.6:
string_decoder "~1.1.1"
util-deprecate "~1.0.1"

readable-stream@^4.0.0:
version "4.1.0"
resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-4.1.0.tgz#280d0a29f559d3fb684a277254e02b6f61ae0631"
integrity sha512-sVisi3+P2lJ2t0BPbpK629j8wRW06yKGJUcaLAGXPAUhyUxVJm7VsCTit1PFgT4JHUDMrGNR+ZjSKpzGaRF3zw==
dependencies:
abort-controller "^3.0.0"

readdir-scoped-modules@^1.1.0:
version "1.1.0"
resolved "https://registry.yarnpkg.com/readdir-scoped-modules/-/readdir-scoped-modules-1.1.0.tgz#8d45407b4f870a0dcaebc0e28670d18e74514309"
Expand Down Expand Up @@ -6681,10 +6700,10 @@ safe-regex2@^2.0.0:
dependencies:
ret "~0.2.0"

safe-stable-stringify@^2.1.0:
version "2.3.0"
resolved "https://registry.yarnpkg.com/safe-stable-stringify/-/safe-stable-stringify-2.3.0.tgz#d09eb692386d7faa846d78922605c67cc0ab9c1c"
integrity sha512-VFlmNrvZ44a0QnRY2yfEIUhbMh8BjTFWf2mRG/8mCEAKTfQYV8xxfn6P+00OLej0gznC5C+hfgcEF7AGrN5Stw==
safe-stable-stringify@^2.3.1:
version "2.3.1"
resolved "https://registry.yarnpkg.com/safe-stable-stringify/-/safe-stable-stringify-2.3.1.tgz#ab67cbe1fe7d40603ca641c5e765cb942d04fc73"
integrity sha512-kYBSfT+troD9cDA85VDnHZ1rpHC50O0g1e6WlGHVCz/g+JS+9WKLj+XwFYyR8UbrZN8ll9HUpDAAddY58MGisg==

"safer-buffer@>= 2.1.2 < 3", "safer-buffer@>= 2.1.2 < 3.0.0":
version "2.1.2"
Expand Down Expand Up @@ -6938,10 +6957,10 @@ sonic-boom@^1.0.2:
atomic-sleep "^1.0.0"
flatstr "^1.0.12"

sonic-boom@^2.2.1:
version "2.4.0"
resolved "https://registry.yarnpkg.com/sonic-boom/-/sonic-boom-2.4.0.tgz#f896898174023b1212b201a7d815a3aea6975055"
integrity sha512-2jSXeHjrovvsrnQ0tvG6eUjDdtMBxtYDAHr18mGDM/Pno/Wdw+pQ1AXUWKCZgriZ9MvCVdUEXXOUfVEwPcrieg==
sonic-boom@^3.1.0:
version "3.1.0"
resolved "https://registry.yarnpkg.com/sonic-boom/-/sonic-boom-3.1.0.tgz#c79f4298ae841f236f3bc0d6c1225d39d51f8eb2"
integrity sha512-qVr246G5307nz5JmhtsvHudxNMrRTWTCmTg+tA4gHQJnVyx/SsyQnky/5peC23B8etvlKli9P6sNXWQGDxoskQ==
dependencies:
atomic-sleep "^1.0.0"

Expand Down Expand Up @@ -7076,11 +7095,6 @@ ssri@^9.0.0:
resolved "https://registry.yarnpkg.com/statuses/-/statuses-1.5.0.tgz#161c7dac177659fd9811f43771fa99381478628c"
integrity sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow=

stream-shift@^1.0.0:
version "1.0.1"
resolved "https://registry.yarnpkg.com/stream-shift/-/stream-shift-1.0.1.tgz#d7088281559ab2778424279b0877da3c392d5a3d"
integrity sha512-AiisoFqQ0vbGcZgQPY1cdP2I76glaVA/RauYR4G4thNFgkTqr90yXTo4LYX60Jl+sIlPNHHdGSwo01AvbKUSVQ==

string-argv@~0.3.1:
version "0.3.1"
resolved "https://registry.yarnpkg.com/string-argv/-/string-argv-0.3.1.tgz#95e2fbec0427ae19184935f816d74aaa4c5c19da"
Expand Down Expand Up @@ -7313,10 +7327,10 @@ text-table@^0.2.0:
resolved "https://registry.yarnpkg.com/text-table/-/text-table-0.2.0.tgz#7f5ee823ae805207c00af2df4a84ec3fcfa570b4"
integrity sha1-f17oI66AUgfACvLfSoTsP8+lcLQ=

thread-stream@^0.15.1:
version "0.15.2"
resolved "https://registry.yarnpkg.com/thread-stream/-/thread-stream-0.15.2.tgz#fb95ad87d2f1e28f07116eb23d85aba3bc0425f4"
integrity sha512-UkEhKIg2pD+fjkHQKyJO3yoIvAP3N6RlNFt2dUhcS1FGvCD1cQa1M/PGknCLFIyZdtJOWQjejp7bdNqmN7zwdA==
thread-stream@^2.0.0:
version "2.0.0"
resolved "https://registry.yarnpkg.com/thread-stream/-/thread-stream-2.0.0.tgz#1ecb450324ebb1bb284d4398b8af15cb6905028c"
integrity sha512-tnbzCbIrA4Khq5SJt/Fyz5DlE8pUnPR3//nWv+cqdRktvAl2NuC9O08HHq2Ifa10bhkvHLuzcesNjaH15EgTXA==
dependencies:
real-require "^0.1.0"

Expand Down

0 comments on commit 3f7bf0c

Please sign in to comment.