diff --git a/packages/lambda-cog/package.json b/packages/lambda-cog/package.json index 96812c255..8ef261002 100644 --- a/packages/lambda-cog/package.json +++ b/packages/lambda-cog/package.json @@ -26,7 +26,7 @@ "@basemaps/lambda": "^6.7.0", "@basemaps/shared": "^6.31.0", "@chunkd/fs": "^8.4.0", - "@linzjs/lambda": "^3.2.0" + "@linzjs/lambda": "^3.2.1" }, "bundle": { "entry": "src/index.ts", diff --git a/packages/lambda-tiler/package.json b/packages/lambda-tiler/package.json index a94a2f726..9ad146d0d 100644 --- a/packages/lambda-tiler/package.json +++ b/packages/lambda-tiler/package.json @@ -32,7 +32,7 @@ "@cogeotiff/core": "^7.0.0", "@cotar/core": "^5.4.0", "@linzjs/geojson": "^6.28.1", - "@linzjs/lambda": "^3.2.0", + "@linzjs/lambda": "^3.2.1", "p-limit": "^4.0.0", "path-to-regexp": "^6.1.0", "pixelmatch": "^5.1.0", diff --git a/packages/lambda-tiler/src/index.ts b/packages/lambda-tiler/src/index.ts index a978ec1b9..2f20c5e70 100644 --- a/packages/lambda-tiler/src/index.ts +++ b/packages/lambda-tiler/src/index.ts @@ -11,6 +11,7 @@ import { styleJsonGet } from './routes/tile.style.json.js'; import { wmtsCapabilitiesGet } from './routes/tile.wmts.js'; import { tileXyzGet } from './routes/tile.xyz.js'; import { versionGet } from './routes/version.js'; +import { CoSources } from './util/source.cache.js'; import { St } from './util/source.tracer.js'; export const handler = lf.http(LogConfig.get()); @@ -29,6 +30,16 @@ handler.router.hook('response', (req, res) => { req.set('requestCount', St.requests.length); } + // Log the source cache hit/miss ratio + req.set('sources', { + hits: CoSources.cache.hits, + misses: CoSources.cache.misses, + size: CoSources.cache.currentSize, + resets: CoSources.cache.resets, + cacheA: CoSources.cache.cacheA.size, + cacheB: CoSources.cache.cacheB.size, + }); + // Ensure CORS response headers are set res.header('Access-Control-Allow-Origin', '*'); }); diff --git a/packages/lambda-tiler/src/util/swapping.lru.ts b/packages/lambda-tiler/src/util/swapping.lru.ts index 23fc744fd..04d64d6d8 100644 --- a/packages/lambda-tiler/src/util/swapping.lru.ts +++ b/packages/lambda-tiler/src/util/swapping.lru.ts @@ -3,6 +3,10 @@ export class SwappingLru { cacheB: Map = new Map(); maxSize: number; + hits = 0; + misses = 0; + resets = 0; + _lastCheckedAt = -1; constructor(maxSize: number) { @@ -11,10 +15,18 @@ export class SwappingLru { get(id: string): T | null { const cacheA = this.cacheA.get(id); - if (cacheA) return cacheA; + if (cacheA) { + this.hits++; + return cacheA; + } const cacheB = this.cacheB.get(id); - if (cacheB == null) return null; + if (cacheB == null) { + this.misses++; + return null; + } + + this.hits++; // If a object is still useful move it into the main cache this.cacheA.set(id, cacheB); this.cacheB.delete(id); @@ -37,6 +49,7 @@ export class SwappingLru { this._lastCheckedAt = Date.now(); if (this.maxSize <= 0) return; if (this.currentSize <= this.maxSize) return; + this.resets++; this.cacheB = this.cacheA; this.cacheA = new Map(); } diff --git a/packages/shared/package.json b/packages/shared/package.json index b2582147a..d599eddd0 100644 --- a/packages/shared/package.json +++ b/packages/shared/package.json @@ -31,7 +31,7 @@ "@linzjs/metrics": "^6.28.1", "aws-sdk": "^2.890.0", "entities": "^4.3.0", - "pino": "^7.5.0", + "pino": "^8.3.0", "proj4": "^2.8.0", "sax": "^1.2.4", "source-map-support": "^0.5.19", diff --git a/yarn.lock b/yarn.lock index 9b44b3433..54435da1b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -945,14 +945,14 @@ npmlog "^6.0.2" write-file-atomic "^3.0.3" -"@linzjs/lambda@^3.2.0": - version "3.2.0" - resolved "https://registry.yarnpkg.com/@linzjs/lambda/-/lambda-3.2.0.tgz#fd41931a2703a688e595a90fa435aefbcdbb0a20" - integrity sha512-65jW+CD79MrtXTxc1hLV6kARZ6ZkNCgpTX+9h/mtEMr1gmMp5hL5kWzloRL8g/qthNv0YEheTiQCabS93eNAyQ== +"@linzjs/lambda@^3.2.1": + version "3.2.1" + resolved "https://registry.yarnpkg.com/@linzjs/lambda/-/lambda-3.2.1.tgz#61264b4a8ee5c3d0a7fc7687bd7f4c1ddb6bf4c8" + integrity sha512-gtNeumdN0/aqS0WJlwNoN9otfcL1kVGCppU9hLaco+sK+uX9rUdzOiiaiCCtJ3qODUwZwv7onP8lTpgSdm5hCQ== dependencies: "@linzjs/metrics" "^6.21.1" find-my-way "^7.0.0" - pino "^7.9.1" + pino "^8.3.0" ulid "^2.3.0" "@linzjs/lui@^10.11.3": @@ -1675,6 +1675,13 @@ abbrev@1: resolved "https://registry.yarnpkg.com/abbrev/-/abbrev-1.1.1.tgz#f8f2c887ad10bf67f634f005b6987fed3179aac8" integrity sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q== +abort-controller@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/abort-controller/-/abort-controller-3.0.0.tgz#eaf54d53b62bae4138e809ca225c8439a6efb392" + integrity sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg== + dependencies: + event-target-shim "^5.0.0" + abstract-logging@^2.0.0: version "2.0.1" resolved "https://registry.yarnpkg.com/abstract-logging/-/abstract-logging-2.0.1.tgz#6b0c371df212db7129b57d2e7fcf282b8bf1c839" @@ -3023,16 +3030,6 @@ duplexer@^0.1.1: resolved "https://registry.yarnpkg.com/duplexer/-/duplexer-0.1.1.tgz#ace6ff808c1ce66b57d1ebf97977acb02334cfc1" integrity sha1-rOb/gIwc5mtX0ev5eXessCM0z8E= -duplexify@^4.1.2: - version "4.1.2" - resolved "https://registry.yarnpkg.com/duplexify/-/duplexify-4.1.2.tgz#18b4f8d28289132fa0b9573c898d9f903f81c7b0" - integrity sha512-fz3OjcNCHmRP12MJoZMPglx8m4rrFP8rovnk4vT8Fs+aonZoCwGg10dSsQsfP/E62eZcPTMSMP6686fu9Qlqtw== - dependencies: - end-of-stream "^1.4.1" - inherits "^2.0.3" - readable-stream "^3.1.1" - stream-shift "^1.0.0" - earcut@^2.2.3: version "2.2.3" resolved "https://registry.yarnpkg.com/earcut/-/earcut-2.2.3.tgz#d44ced2ff5a18859568e327dd9c7d46b16f55cf4" @@ -3415,6 +3412,11 @@ etag@~1.8.1: resolved "https://registry.yarnpkg.com/etag/-/etag-1.8.1.tgz#41ae2eeb65efa62268aebfea83ac7d79299b0887" integrity sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc= +event-target-shim@^5.0.0: + version "5.0.1" + resolved "https://registry.yarnpkg.com/event-target-shim/-/event-target-shim-5.0.1.tgz#5d4d3ebdf9583d63a5333ce2deb7480ab2b05789" + integrity sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ== + eventemitter3@^4.0.4: version "4.0.7" resolved "https://registry.yarnpkg.com/eventemitter3/-/eventemitter3-4.0.7.tgz#2de9b68f6528d5644ef5c59526a1b4a07306169f" @@ -3529,6 +3531,11 @@ fast-redact@^3.0.0: resolved "https://registry.yarnpkg.com/fast-redact/-/fast-redact-3.0.0.tgz#ac2f9e36c9f4976f5db9fb18c6ffbaf308cf316d" integrity sha512-a/S/Hp6aoIjx7EmugtzLqXmcNsyFszqbt6qQ99BdG61QjBZF6shNis0BYR6TsZOQ1twYc0FN2Xdhwwbv6+KD0w== +fast-redact@^3.1.1: + version "3.1.1" + resolved "https://registry.yarnpkg.com/fast-redact/-/fast-redact-3.1.1.tgz#790fcff8f808c2e12fabbfb2be5cb2deda448fa0" + integrity sha512-odVmjC8x8jNeMZ3C+rPMESzXVSEU8tSWSHv9HFxP2mm89G/1WwqhrerJDQm9Zus8X6aoRgQDThKqptdNA6bt+A== + fast-safe-stringify@^2.0.8: version "2.0.8" resolved "https://registry.yarnpkg.com/fast-safe-stringify/-/fast-safe-stringify-2.0.8.tgz#dc2af48c46cf712b683e849b2bbd446b32de936f" @@ -5568,10 +5575,10 @@ object-assign@^4.0.1, object-assign@^4.1.1: resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.1.tgz#2109adc7965887cfc05cbbd442cac8bfbb360863" integrity sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM= -on-exit-leak-free@^0.2.0: - version "0.2.0" - resolved "https://registry.yarnpkg.com/on-exit-leak-free/-/on-exit-leak-free-0.2.0.tgz#b39c9e3bf7690d890f4861558b0d7b90a442d209" - integrity sha512-dqaz3u44QbRXQooZLTUKU41ZrzYrcvLISVgbrzbyCMxpmSLJvZ3ZamIJIZ29P6OhZIkNIQKosdeM6t1LYbA9hg== +on-exit-leak-free@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/on-exit-leak-free/-/on-exit-leak-free-2.1.0.tgz#5c703c968f7e7f851885f6459bf8a8a57edc9cc4" + integrity sha512-VuCaZZAjReZ3vUwgOB8LxAosIurDiAW0s13rI1YwmaP++jvcxP77AWoQvenZebpCA2m8WC1/EosPYPMjnRAp/w== on-finished@~2.3.0: version "2.3.0" @@ -5959,12 +5966,12 @@ pinkie@^2.0.0: resolved "https://registry.yarnpkg.com/pinkie/-/pinkie-2.0.4.tgz#72556b80cfa0d48a974e80e77248e80ed4f7f870" integrity sha1-clVrgM+g1IqXToDnckjoDtT3+HA= -pino-abstract-transport@v0.5.0: - version "0.5.0" - resolved "https://registry.yarnpkg.com/pino-abstract-transport/-/pino-abstract-transport-0.5.0.tgz#4b54348d8f73713bfd14e3dc44228739aa13d9c0" - integrity sha512-+KAgmVeqXYbTtU2FScx1XS3kNyfZ5TrXY07V96QnUSFqo2gAqlvmaxH67Lj7SWazqsMabf+58ctdTcBgnOLUOQ== +pino-abstract-transport@v1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/pino-abstract-transport/-/pino-abstract-transport-1.0.0.tgz#cc0d6955fffcadb91b7b49ef220a6cc111d48bb3" + integrity sha512-c7vo5OpW4wIS42hUVcT5REsL8ZljsUfBjqV/e2sFxmFEFZiq1XLUp5EYLtuDH6PEHq9W1egWqRbnLUP5FuZmOA== dependencies: - duplexify "^4.1.2" + readable-stream "^4.0.0" split2 "^4.0.0" pino-std-serializers@^3.1.0: @@ -5972,10 +5979,10 @@ pino-std-serializers@^3.1.0: resolved "https://registry.yarnpkg.com/pino-std-serializers/-/pino-std-serializers-3.1.0.tgz#fe62d339aeef359d9d7bdf61f4e4b2d7be8c73d4" integrity sha512-Fk1pxhX6tuW4ozRDFw5Mz/IHQV5wXYXZwjG/gOpTNPCbYEMeNW9VnKAEu1428CwAQVupFruOr1vkC/ufmcwedA== -pino-std-serializers@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/pino-std-serializers/-/pino-std-serializers-4.0.0.tgz#1791ccd2539c091ae49ce9993205e2cd5dbba1e2" - integrity sha512-cK0pekc1Kjy5w9V2/n+8MkZwusa6EyyxfeQCB799CQRhRt/CqYKiWs5adeu8Shve2ZNffvfC/7J64A2PJo1W/Q== +pino-std-serializers@^6.0.0: + version "6.0.0" + resolved "https://registry.yarnpkg.com/pino-std-serializers/-/pino-std-serializers-6.0.0.tgz#4c20928a1bafca122fdc2a7a4a171ca1c5f9c526" + integrity sha512-mMMOwSKrmyl+Y12Ri2xhH1lbzQxwwpuru9VjyJpgFIH4asSj88F2csdMwN6+M5g1Ll4rmsYghHLQJw81tgZ7LQ== pino@^6.13.0: version "6.14.0" @@ -5990,22 +5997,22 @@ pino@^6.13.0: quick-format-unescaped "^4.0.3" sonic-boom "^1.0.2" -pino@^7.5.0, pino@^7.9.1: - version "7.11.0" - resolved "https://registry.yarnpkg.com/pino/-/pino-7.11.0.tgz#0f0ea5c4683dc91388081d44bff10c83125066f6" - integrity sha512-dMACeu63HtRLmCG8VKdy4cShCPKaYDR4youZqoSWLxl5Gu99HUw8bw75thbPv9Nip+H+QYX8o3ZJbTdVZZ2TVg== +pino@^8.3.0: + version "8.3.0" + resolved "https://registry.yarnpkg.com/pino/-/pino-8.3.0.tgz#13e400df4ef39ce716b33b8e5486fb8e24a30c45" + integrity sha512-CUOOU5sYgyLVijf7qsOWkPLOdzIXaVE9Oyl6zAkjgJn30uGBFUtrRaaCzbtOSJ1I6BqfanBqBttbazEJIwHkJg== dependencies: atomic-sleep "^1.0.0" - fast-redact "^3.0.0" - on-exit-leak-free "^0.2.0" - pino-abstract-transport v0.5.0 - pino-std-serializers "^4.0.0" - process-warning "^1.0.0" + fast-redact "^3.1.1" + on-exit-leak-free "^2.1.0" + pino-abstract-transport v1.0.0 + pino-std-serializers "^6.0.0" + process-warning "^2.0.0" quick-format-unescaped "^4.0.3" real-require "^0.1.0" - safe-stable-stringify "^2.1.0" - sonic-boom "^2.2.1" - thread-stream "^0.15.1" + safe-stable-stringify "^2.3.1" + sonic-boom "^3.1.0" + thread-stream "^2.0.0" pixelmatch@^5.1.0: version "5.2.1" @@ -6132,6 +6139,11 @@ process-warning@^1.0.0: resolved "https://registry.yarnpkg.com/process-warning/-/process-warning-1.0.0.tgz#980a0b25dc38cd6034181be4b7726d89066b4616" integrity sha512-du4wfLyj4yCZq1VupnVSZmRsPJsNuxoDQFdCFHLaYiEbFBD7QE0a+I4D7hOxrVnh78QE/YipFAj9lXHiXocV+Q== +process-warning@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/process-warning/-/process-warning-2.0.0.tgz#341dbeaac985b90a04ebcd844d50097c7737b2ee" + integrity sha512-+MmoAXoUX+VTHAlwns0h+kFUWFs/3FZy+ZuchkgjyOu3oioLAo2LB5aCfKPh2+P9O18i3m43tUEv3YqttSy0Ww== + proj4@^2.8.0: version "2.8.0" resolved "https://registry.yarnpkg.com/proj4/-/proj4-2.8.0.tgz#b2cb8f3ccd56d4dcc7c3e46155cd02caa804b170" @@ -6445,6 +6457,13 @@ readable-stream@^2.0.0, readable-stream@~2.3.6: string_decoder "~1.1.1" util-deprecate "~1.0.1" +readable-stream@^4.0.0: + version "4.1.0" + resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-4.1.0.tgz#280d0a29f559d3fb684a277254e02b6f61ae0631" + integrity sha512-sVisi3+P2lJ2t0BPbpK629j8wRW06yKGJUcaLAGXPAUhyUxVJm7VsCTit1PFgT4JHUDMrGNR+ZjSKpzGaRF3zw== + dependencies: + abort-controller "^3.0.0" + readdir-scoped-modules@^1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/readdir-scoped-modules/-/readdir-scoped-modules-1.1.0.tgz#8d45407b4f870a0dcaebc0e28670d18e74514309" @@ -6681,10 +6700,10 @@ safe-regex2@^2.0.0: dependencies: ret "~0.2.0" -safe-stable-stringify@^2.1.0: - version "2.3.0" - resolved "https://registry.yarnpkg.com/safe-stable-stringify/-/safe-stable-stringify-2.3.0.tgz#d09eb692386d7faa846d78922605c67cc0ab9c1c" - integrity sha512-VFlmNrvZ44a0QnRY2yfEIUhbMh8BjTFWf2mRG/8mCEAKTfQYV8xxfn6P+00OLej0gznC5C+hfgcEF7AGrN5Stw== +safe-stable-stringify@^2.3.1: + version "2.3.1" + resolved "https://registry.yarnpkg.com/safe-stable-stringify/-/safe-stable-stringify-2.3.1.tgz#ab67cbe1fe7d40603ca641c5e765cb942d04fc73" + integrity sha512-kYBSfT+troD9cDA85VDnHZ1rpHC50O0g1e6WlGHVCz/g+JS+9WKLj+XwFYyR8UbrZN8ll9HUpDAAddY58MGisg== "safer-buffer@>= 2.1.2 < 3", "safer-buffer@>= 2.1.2 < 3.0.0": version "2.1.2" @@ -6938,10 +6957,10 @@ sonic-boom@^1.0.2: atomic-sleep "^1.0.0" flatstr "^1.0.12" -sonic-boom@^2.2.1: - version "2.4.0" - resolved "https://registry.yarnpkg.com/sonic-boom/-/sonic-boom-2.4.0.tgz#f896898174023b1212b201a7d815a3aea6975055" - integrity sha512-2jSXeHjrovvsrnQ0tvG6eUjDdtMBxtYDAHr18mGDM/Pno/Wdw+pQ1AXUWKCZgriZ9MvCVdUEXXOUfVEwPcrieg== +sonic-boom@^3.1.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/sonic-boom/-/sonic-boom-3.1.0.tgz#c79f4298ae841f236f3bc0d6c1225d39d51f8eb2" + integrity sha512-qVr246G5307nz5JmhtsvHudxNMrRTWTCmTg+tA4gHQJnVyx/SsyQnky/5peC23B8etvlKli9P6sNXWQGDxoskQ== dependencies: atomic-sleep "^1.0.0" @@ -7076,11 +7095,6 @@ ssri@^9.0.0: resolved "https://registry.yarnpkg.com/statuses/-/statuses-1.5.0.tgz#161c7dac177659fd9811f43771fa99381478628c" integrity sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow= -stream-shift@^1.0.0: - version "1.0.1" - resolved "https://registry.yarnpkg.com/stream-shift/-/stream-shift-1.0.1.tgz#d7088281559ab2778424279b0877da3c392d5a3d" - integrity sha512-AiisoFqQ0vbGcZgQPY1cdP2I76glaVA/RauYR4G4thNFgkTqr90yXTo4LYX60Jl+sIlPNHHdGSwo01AvbKUSVQ== - string-argv@~0.3.1: version "0.3.1" resolved "https://registry.yarnpkg.com/string-argv/-/string-argv-0.3.1.tgz#95e2fbec0427ae19184935f816d74aaa4c5c19da" @@ -7313,10 +7327,10 @@ text-table@^0.2.0: resolved "https://registry.yarnpkg.com/text-table/-/text-table-0.2.0.tgz#7f5ee823ae805207c00af2df4a84ec3fcfa570b4" integrity sha1-f17oI66AUgfACvLfSoTsP8+lcLQ= -thread-stream@^0.15.1: - version "0.15.2" - resolved "https://registry.yarnpkg.com/thread-stream/-/thread-stream-0.15.2.tgz#fb95ad87d2f1e28f07116eb23d85aba3bc0425f4" - integrity sha512-UkEhKIg2pD+fjkHQKyJO3yoIvAP3N6RlNFt2dUhcS1FGvCD1cQa1M/PGknCLFIyZdtJOWQjejp7bdNqmN7zwdA== +thread-stream@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/thread-stream/-/thread-stream-2.0.0.tgz#1ecb450324ebb1bb284d4398b8af15cb6905028c" + integrity sha512-tnbzCbIrA4Khq5SJt/Fyz5DlE8pUnPR3//nWv+cqdRktvAl2NuC9O08HHq2Ifa10bhkvHLuzcesNjaH15EgTXA== dependencies: real-require "^0.1.0"