From 96fe16fd3ccb5b4f4893b4f2147057de5edecf79 Mon Sep 17 00:00:00 2001 From: Harry Waye Date: Wed, 26 Apr 2023 14:09:13 +0100 Subject: [PATCH] chore(recordings): use cooperative-sticky rebalance strategy (#15260) Revert "revert(recordings): use cooperative-sticky rebalance strategy (#15211)" This reverts commit a40f01138eeb07aa97984f10e577a8e7394daa97. --- plugin-server/functional_tests/kafka.ts | 2 +- plugin-server/package.json | 2 +- plugin-server/pnpm-lock.yaml | 487 +++++++++++++++--- plugin-server/src/kafka/admin.ts | 2 +- plugin-server/src/kafka/batch-consumer.ts | 19 +- plugin-server/src/kafka/config.ts | 2 +- plugin-server/src/kafka/consumer.ts | 2 +- plugin-server/src/kafka/producer.ts | 2 +- .../blob-ingester/offset-manager.ts | 2 +- .../session-recordings-blob-consumer.ts | 2 +- .../session-recordings-consumer.ts | 2 +- .../blob-ingester/offset-manager.test.ts | 2 +- .../session-recordings-consumer.test.ts | 2 +- 13 files changed, 456 insertions(+), 72 deletions(-) diff --git a/plugin-server/functional_tests/kafka.ts b/plugin-server/functional_tests/kafka.ts index 8455d85628335..2144c42881635 100644 --- a/plugin-server/functional_tests/kafka.ts +++ b/plugin-server/functional_tests/kafka.ts @@ -1,6 +1,6 @@ import { CompressionCodecs, CompressionTypes } from 'kafkajs' import SnappyCodec from 'kafkajs-snappy' -import { HighLevelProducer } from 'node-rdkafka' +import { HighLevelProducer } from 'node-rdkafka-acosom' import { defaultConfig } from '../src/config/config' diff --git a/plugin-server/package.json b/plugin-server/package.json index 2babe7e1ff9d8..90df874817d2b 100644 --- a/plugin-server/package.json +++ b/plugin-server/package.json @@ -70,7 +70,7 @@ "lru-cache": "^6.0.0", "luxon": "^1.27.0", "node-fetch": "^2.6.1", - "node-rdkafka": "^2.15.0", + "node-rdkafka-acosom": "2.16.1", "node-schedule": "^2.1.0", "pg": "^8.6.0", "pino": "^8.6.0", diff --git a/plugin-server/pnpm-lock.yaml b/plugin-server/pnpm-lock.yaml index 33dd4832a0817..c1e9ec85c6212 100644 --- a/plugin-server/pnpm-lock.yaml +++ b/plugin-server/pnpm-lock.yaml @@ -76,7 +76,7 @@ specifiers: lru-cache: ^6.0.0 luxon: ^1.27.0 node-fetch: ^2.6.1 - node-rdkafka: ^2.15.0 + node-rdkafka-acosom: 2.16.1 node-schedule: ^2.1.0 parse-prometheus-text-format: ^1.1.1 pg: ^8.6.0 @@ -131,7 +131,7 @@ dependencies: lru-cache: 6.0.0 luxon: 1.28.0 node-fetch: 2.6.7 - node-rdkafka: 2.15.0 + node-rdkafka-acosom: 2.16.1_mwhvu7sfp6vq5ryuwb6hlbjfka node-schedule: 2.1.0 pg: 8.8.0 pino: 8.7.0 @@ -2523,7 +2523,6 @@ packages: engines: {node: '>=12'} dependencies: '@jridgewell/trace-mapping': 0.3.9 - dev: true /@dabh/diagnostics/2.0.3: resolution: {integrity: sha512-hrlQOIi7hAfzsMqlGSFyVucrx38O+j6wiGOf//H2ecvIEqYN4ADBSS2iLMh5UFyDunCNniUIPk/q3riFv45xRA==} @@ -3048,12 +3047,10 @@ packages: get-package-type: 0.1.0 js-yaml: 3.14.1 resolve-from: 5.0.0 - dev: true /@istanbuljs/schema/0.1.3: resolution: {integrity: sha512-ZXRY4jNvVgSVQ8DL3LTcakaAtXwTVUxE81hslsyD2AtoXW/wVob10HkOJ1X/pAlcI7D+2YoZKg5do8G/w6RYgA==} engines: {node: '>=8'} - dev: true /@jest/console/28.1.3: resolution: {integrity: sha512-QPAkP5EwKdK/bxIr6C1I4Vs0rm2nHiANzj/Z5X2JQkrZo6IqvC4ldZ9K95tF0HdidhA8Bo6egxSzUFPYKcEXLw==} @@ -3324,7 +3321,6 @@ packages: dependencies: '@jridgewell/resolve-uri': 3.1.0 '@jridgewell/sourcemap-codec': 1.4.14 - dev: true /@maxmind/geoip2-node/3.5.0: resolution: {integrity: sha512-WG2TNxMwDWDOrljLwyZf5bwiEYubaHuICvQRlgz74lE9OZA/z4o+ZT6OisjDBAZh/yRJVNK6mfHqmP5lLlAwsA==} @@ -3558,7 +3554,6 @@ packages: cpu: [arm64] os: [darwin] requiresBuild: true - dev: true optional: true /@swc/core-darwin-x64/1.3.14: @@ -3567,7 +3562,6 @@ packages: cpu: [x64] os: [darwin] requiresBuild: true - dev: true optional: true /@swc/core-linux-arm-gnueabihf/1.3.14: @@ -3576,7 +3570,6 @@ packages: cpu: [arm] os: [linux] requiresBuild: true - dev: true optional: true /@swc/core-linux-arm64-gnu/1.3.14: @@ -3585,7 +3578,6 @@ packages: cpu: [arm64] os: [linux] requiresBuild: true - dev: true optional: true /@swc/core-linux-arm64-musl/1.3.14: @@ -3594,7 +3586,6 @@ packages: cpu: [arm64] os: [linux] requiresBuild: true - dev: true optional: true /@swc/core-linux-x64-gnu/1.3.14: @@ -3603,7 +3594,6 @@ packages: cpu: [x64] os: [linux] requiresBuild: true - dev: true optional: true /@swc/core-linux-x64-musl/1.3.14: @@ -3612,7 +3602,6 @@ packages: cpu: [x64] os: [linux] requiresBuild: true - dev: true optional: true /@swc/core-win32-arm64-msvc/1.3.14: @@ -3621,7 +3610,6 @@ packages: cpu: [arm64] os: [win32] requiresBuild: true - dev: true optional: true /@swc/core-win32-ia32-msvc/1.3.14: @@ -3630,7 +3618,6 @@ packages: cpu: [ia32] os: [win32] requiresBuild: true - dev: true optional: true /@swc/core-win32-x64-msvc/1.3.14: @@ -3639,7 +3626,6 @@ packages: cpu: [x64] os: [win32] requiresBuild: true - dev: true optional: true /@swc/core/1.3.14: @@ -3658,7 +3644,6 @@ packages: '@swc/core-win32-arm64-msvc': 1.3.14 '@swc/core-win32-ia32-msvc': 1.3.14 '@swc/core-win32-x64-msvc': 1.3.14 - dev: true /@swc/jest/0.2.23_@swc+core@1.3.14: resolution: {integrity: sha512-ZLj17XjHbPtNsgqjm83qizENw05emLkKGu3WuPUttcy9hkngl0/kcc7fDbcSBpADS0GUtsO+iKPjZFWVAtJSlA==} @@ -3693,19 +3678,15 @@ packages: /@tsconfig/node10/1.0.9: resolution: {integrity: sha512-jNsYVVxU8v5g43Erja32laIDHXeoNvFEpX33OK4d6hljo3jDhCBDhx5dhCCTMWUojscpAagGiRkBKxpdl9fxqA==} - dev: true /@tsconfig/node12/1.0.11: resolution: {integrity: sha512-cqefuRsh12pWyGsIoBKJA9luFu3mRxCA+ORZvA4ktLSzIuCUtWVxGIuXigEwO5/ywWFMZ2QEGKWvkZG1zDMTag==} - dev: true /@tsconfig/node14/1.0.3: resolution: {integrity: sha512-ysT8mhdixWK6Hw3i1V2AeRqZ5WfXg1G43mqoYlM2nc6388Fq5jcXyr5mRsqViLx/GJYdoL0bfXD8nmF+Zn/Iow==} - dev: true /@tsconfig/node16/1.0.3: resolution: {integrity: sha512-yOlFc+7UtL/89t2ZhjPvvB/DeAr3r+Dq58IgzsFkOAvVC6NMJXmCGjbptdXdR9qsX7pKcTL+s87FtYREi2dEEQ==} - dev: true /@types/adm-zip/0.4.34: resolution: {integrity: sha512-8ToYLLAYhkRfcmmljrKi22gT2pqu7hGMDtORP1emwIEGmgUTZOsaDjzWFzW5N2frcFRz/50CWt4zA1CxJ73pmQ==} @@ -4235,10 +4216,21 @@ packages: normalize-path: 3.0.0 picomatch: 2.3.1 + /append-transform/2.0.0: + resolution: {integrity: sha512-7yeyCEurROLQJFv5Xj4lEGTy0borxepjFv1g22oAdqFu//SrAlDl1O1Nxx15SH1RoliUml6p8dwJW9jvZughhg==} + engines: {node: '>=8'} + dependencies: + default-require-extensions: 3.0.1 + dev: false + /aproba/2.0.0: resolution: {integrity: sha512-lYe4Gx7QT+MKGbDsA+Z+he/Wtef0BiwDOlK/XkBrdfsh9J/jPPXbX0tE9x9cl27Tmu5gg3QUbUrQYa/y+KOHPQ==} dev: false + /archy/1.0.0: + resolution: {integrity: sha512-Xg+9RwCg/0p32teKdGMPTPnVXKD0w3DfHnFTficozsAgsvq2XenPJq/MYpzzQ/v8zrOyJn6Ds39VA4JIDwFfqw==} + dev: false + /are-we-there-yet/3.0.1: resolution: {integrity: sha512-QZW4EDmGwlYur0Yyf/b2uGucHQMa8aFUP7eu9ddR73vvhFyt4V0Vl3QHPcTNJ8l6qYOBdxgXdnBXQrHilfRQBg==} engines: {node: ^12.13.0 || ^14.15.0 || >=16.0.0} @@ -4249,13 +4241,11 @@ packages: /arg/4.1.3: resolution: {integrity: sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==} - dev: true /argparse/1.0.10: resolution: {integrity: sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==} dependencies: sprintf-js: 1.0.3 - dev: true /argparse/2.0.1: resolution: {integrity: sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==} @@ -4333,6 +4323,11 @@ packages: tslib: 2.4.1 dev: false + /async-hook-domain/2.0.4: + resolution: {integrity: sha512-14LjCmlK1PK8eDtTezR6WX8TMaYNIzBIsd2D1sGoGjgx0BuNMMoSdk7i/drlbtamy0AWv9yv2tkB+ASdmeqFIw==} + engines: {node: '>=10'} + dev: false + /async-retry/1.3.3: resolution: {integrity: sha512-wfr/jstw9xNi/0teMHrRW7dsz3Lt5ARhYNZ2ewpadnhaIp5mbALhOAP+EAdsC7t4Z6wqsDVv9+W6gm1Dk9mEyw==} dependencies: @@ -4540,6 +4535,11 @@ packages: resolution: {integrity: sha512-rA2CrUl1+6yKrn+XgLs8Hdy18OER1UW146nM+ixzhQXDY+Bd3ySkyIJGwF2a4I45JwbvF1mDL/nWkqBwpOcdBA==} dev: false + /bind-obj-methods/3.0.0: + resolution: {integrity: sha512-nLEaaz3/sEzNSyPWRsN9HNsqwk1AUyECtGj+XwGdIi3xABnEqecvXtIJ0wehQXuuER5uZ/5fTs2usONgYjG+iw==} + engines: {node: '>=10'} + dev: false + /bindings/1.5.0: resolution: {integrity: sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==} dependencies: @@ -4743,7 +4743,6 @@ packages: /buffer-from/1.1.2: resolution: {integrity: sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==} - dev: true /buffer-writer/2.0.0: resolution: {integrity: sha512-a7ZpuTZU1TRtnwyCNW3I5dc0wWNC3VR9S++Ewyk2HHZdrO3CQJqSpd+95Us590V6AL7JqUAH2IwZ/398PmNFgw==} @@ -4847,6 +4846,16 @@ packages: resolution: {integrity: sha512-WF0LihfemtesFcJgO7xfOoOcnWzY/QHR4qeDqV44jPU3HTI54+LnfXK3SA27AVVGCdZFgjjFFaqUA9Jx7dMJZA==} dev: true + /caching-transform/4.0.0: + resolution: {integrity: sha512-kpqOvwXnjjN44D89K5ccQC+RUrsy7jB/XLlRrx0D7/2HNcTPqzsb6XgYoErwko6QsV184CA2YgS1fxDiiDZMWA==} + engines: {node: '>=8'} + dependencies: + hasha: 5.2.2 + make-dir: 3.1.0 + package-hash: 4.0.0 + write-file-atomic: 3.0.3 + dev: false + /call-bind/1.0.2: resolution: {integrity: sha512-7O+FbCihrB5WGbFYesctwmTKae6rOiIzmz1icreWJ+0aA7LJfuqhEso2T9ncpcFtzMQtzXf2QGGueWJGTYsqrA==} dependencies: @@ -4877,7 +4886,6 @@ packages: /camelcase/5.3.1: resolution: {integrity: sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==} engines: {node: '>=6'} - dev: true /camelcase/6.3.0: resolution: {integrity: sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==} @@ -4963,6 +4971,14 @@ packages: engines: {node: '>=6'} dev: false + /cliui/6.0.0: + resolution: {integrity: sha512-t6wbgtoCXvAzst7QgXxJYqPt0usEfbgQdftEPbLL/cvv6HPE5VgvqCuAIDR0NgU52ds6rFwqrgakNLrHEjCbrQ==} + dependencies: + string-width: 4.2.3 + strip-ansi: 6.0.1 + wrap-ansi: 6.2.0 + dev: false + /cliui/7.0.4: resolution: {integrity: sha512-OcRE68cOsVMXp1Yvonl/fzkQOyjLSu/8bhPDfQt0e0/Eb283TKP20Fs2MqoPsr9SwA595rRCA+QMzYc9nBP+JQ==} dependencies: @@ -5065,6 +5081,10 @@ packages: engines: {node: '>= 6'} dev: true + /commondir/1.0.1: + resolution: {integrity: sha512-W9pAhw0ja1Edb5GVdIF1mjZw/ASI0AlShXM83UUGe2DVr5TdAPEA1OA8m/g8zWp9x6On7gqufY+FatDbC3MDQg==} + dev: false + /compressible/2.0.18: resolution: {integrity: sha512-AF3r7P5dWxL8MxyITRMlORQNaOA2IkAFaTr4k7BUumjPtRpGDTZpl0Pb1XCO6JeDCBdp126Cgs9sMxqSjgYyRg==} engines: {node: '>= 0.6'} @@ -5194,7 +5214,6 @@ packages: /create-require/1.1.1: resolution: {integrity: sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==} - dev: true /cron-parser/3.5.0: resolution: {integrity: sha512-wyVZtbRs6qDfFd8ap457w3XVntdvqcwBGxBoTvJQH9KGVKL/fB+h2k3C8AqiVxvUQKN1Ps/Ns46CNViOpVDhfQ==} @@ -5211,7 +5230,6 @@ packages: path-key: 3.1.1 shebang-command: 2.0.0 which: 2.0.2 - dev: true /crypto-browserify/3.12.0: resolution: {integrity: sha512-fz4spIh+znjO2VjL+IdhEpRJ3YN6sMzITSBijk6FK2UvTqruSQW+/cCZTSNsMiZNvUeq0CqurF+dAbyiGOY6Wg==} @@ -5396,6 +5414,11 @@ packages: dependencies: ms: 2.1.2 + /decamelize/1.2.0: + resolution: {integrity: sha512-z2S+W9X73hAUUki+N+9Za2lBlun89zigOyGrsax+KUQ6wKW4ZoWpEYBkGhQjwAjjDCkWxhY0VKEhk8wzY7F5cA==} + engines: {node: '>=0.10.0'} + dev: false + /dedent/0.7.0: resolution: {integrity: sha512-Q6fKUPqnAHAyhiUgFU7BUzLiv0kd8saH9al7tnu5Q/okj6dnupxyTgFIBjVzJATdfIAm9NAsvXNzjaKa+bxVyA==} dev: true @@ -5408,6 +5431,13 @@ packages: engines: {node: '>=0.10.0'} dev: true + /default-require-extensions/3.0.1: + resolution: {integrity: sha512-eXTJmRbm2TIt9MgWTsOH1wEuhew6XGZcMeGKCtLedIg/NCsg1iBePXkceTdK4Fii7pzmN9tGsZhKzZ4h7O/fxw==} + engines: {node: '>=8'} + dependencies: + strip-bom: 4.0.0 + dev: false + /default-user-agent/1.0.0: resolution: {integrity: sha512-bDF7bg6OSNcSwFWPu4zYKpVkJZQYVrAANMYB8bc9Szem1D0yKdm4sa/rOCs2aC9+2GMqQ7KnwtZRvDhmLF0dXw==} engines: {node: '>= 0.10.0'} @@ -5504,7 +5534,6 @@ packages: /diff/4.0.2: resolution: {integrity: sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==} engines: {node: '>=0.3.1'} - dev: true /diffie-hellman/5.0.3: resolution: {integrity: sha512-kqag/Nl+f3GwyK25fhUMYj81BUOrZ9IuJsjIcDE5icNM9FJHAVm3VcUDxdLPoQtTuUylWm6ZIknYJwwaPxsUzg==} @@ -5694,6 +5723,10 @@ packages: is-symbol: 1.0.4 dev: true + /es6-error/4.1.1: + resolution: {integrity: sha512-Um/+FxMr9CISWh0bi5Zv0iOD+4cFh5qLeks1qhAopKVAJw3drgKbKySikp7wGhDL0HPeaja0P5ULZrxLkniUVg==} + dev: false + /escalade/3.1.1: resolution: {integrity: sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==} engines: {node: '>=6'} @@ -6065,6 +6098,10 @@ packages: resolution: {integrity: sha512-39F7TBIV0G7gTelxwbEqnwhp90eqCPON1k0NwNfwhgKn4Co4ybUbj2pECcXT0B3ztRKZ7Pw1JujUUgmQJHcVAQ==} dev: false + /events-to-array/1.1.2: + resolution: {integrity: sha512-inRWzRY7nG+aXZxBzEqYKB3HPgwflZRopAjDCHv0whhRx+MTUr1ei0ICZUypdyE0HRm4L2d5VEcIqLD6yl+BFA==} + dev: false + /events/1.1.1: resolution: {integrity: sha512-kEcvvCBByWXGnZy6JUlgAp2gBIUjfCAV6P6TgT1/aaQKcmuAEC4OZTV1I4EWQLz2gxZw76atuVyvHhTxvi0Flw==} engines: {node: '>=0.4.x'} @@ -6232,13 +6269,21 @@ packages: dependencies: to-regex-range: 5.0.1 + /find-cache-dir/3.3.2: + resolution: {integrity: sha512-wXZV5emFEjrridIgED11OoUKLxiYjAcqot/NJdAkOhlJ+vGzwhOAfcG5OX1jP+S0PcjEn8bdMJv+g2jwQ3Onig==} + engines: {node: '>=8'} + dependencies: + commondir: 1.0.1 + make-dir: 3.1.0 + pkg-dir: 4.2.0 + dev: false + /find-up/4.1.0: resolution: {integrity: sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==} engines: {node: '>=8'} dependencies: locate-path: 5.0.0 path-exists: 4.0.0 - dev: true /find-up/5.0.0: resolution: {integrity: sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==} @@ -6248,6 +6293,10 @@ packages: path-exists: 4.0.0 dev: true + /findit/2.0.0: + resolution: {integrity: sha512-ENZS237/Hr8bjczn5eKuBohLgaD0JyUd0arxretR1f9RO46vZHA1b2y0VorgGV3WaOT3c+78P8h7v4JGJ1i/rg==} + dev: false + /flat-cache/3.0.4: resolution: {integrity: sha512-dm9s5Pw7Jc0GvMYbshN6zchCA9RgQlzzEZX3vylR9IqFfS8XciblUXOKfW6SiuJ0e13eDYZoZV5wdrev7P3Nwg==} engines: {node: ^10.12.0 || >=12.0.0} @@ -6287,7 +6336,6 @@ packages: dependencies: cross-spawn: 7.0.3 signal-exit: 3.0.7 - dev: true /form-data/3.0.1: resolution: {integrity: sha512-RHkBKtLWUVwd7SqRIvCZMEvAMoGUp0XU+seQiZejj0COz3RI3hWP4sCv3gZWWLjJTd7rGwcsF5eKZGii0r/hbg==} @@ -6314,6 +6362,14 @@ packages: pause-stream: 0.0.11 dev: false + /fromentries/1.3.2: + resolution: {integrity: sha512-cHEpEQHUg0f8XdtZCc2ZAhrHzKzT0MrFUTcvx+hfxYu7rGMDc5SKoXFh+n4YigxsHXRzc6OrCshdR1bWH6HHyg==} + dev: false + + /fs-exists-cached/1.0.0: + resolution: {integrity: sha512-kSxoARUDn4F2RPXX48UXnaFKwVU7Ivd/6qpzZL29MCDmr9sTvybv4gFCp+qaI4fM9m0z9fgz/yJvi56GAz+BZg==} + dev: false + /fs-extra/10.1.0: resolution: {integrity: sha512-oRXApq54ETRj4eMiFzGnHWGy+zo5raudjuxN0b8H7s/RU2oW0Wvsx9O0ACRN/kRq9E8Vu/ReskGB5o3ji+FzHQ==} engines: {node: '>=12'} @@ -6364,6 +6420,10 @@ packages: /function-bind/1.1.1: resolution: {integrity: sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==} + /function-loop/2.0.1: + resolution: {integrity: sha512-ktIR+O6i/4h+j/ZhZJNdzeI4i9lEPeEK6UPR2EVyTVBqOwcU3Za9xYKLH64ZR9HmcROyRrOkizNyjjtWJzDDkQ==} + dev: false + /function.prototype.name/1.1.5: resolution: {integrity: sha512-uN7m/BzVKQnCUF/iW8jYea67v++2u7m5UgENbHRtdDVclOUP+FMPlCNdmk0h/ysGyo2tavMJEDqJAkJdRa1vMA==} engines: {node: '>= 0.4'} @@ -6472,7 +6532,6 @@ packages: /get-package-type/0.1.0: resolution: {integrity: sha512-pjzuKtY64GYfWizNAJ0fr9VqttZkNiK2iS430LtIHzjBEr6bX8Am2zm4sW4Ro5wjWW5cAlRL1qAMTcXbjNAO2Q==} engines: {node: '>=8.0.0'} - dev: true /get-stream/6.0.1: resolution: {integrity: sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg==} @@ -6749,6 +6808,14 @@ packages: inherits: 2.0.4 minimalistic-assert: 1.0.1 + /hasha/5.2.2: + resolution: {integrity: sha512-Hrp5vIK/xr5SkeN2onO32H0MgNZ0f17HRNH39WfL0SYUNOTZ5Lz1TJ8Pajo/87dYGEFlLMm7mIc/k/s6Bvz9HQ==} + engines: {node: '>=8'} + dependencies: + is-stream: 2.0.1 + type-fest: 0.8.1 + dev: false + /hdr-histogram-js/2.0.3: resolution: {integrity: sha512-Hkn78wwzWHNCp2uarhzQ2SGFLU3JY8SBDDd3TAABK4fc30wm+MuPOrg5QVFVfkKOQd6Bfz3ukJEI+q9sXEkK1g==} dependencies: @@ -6795,7 +6862,6 @@ packages: /html-escaper/2.0.2: resolution: {integrity: sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg==} - dev: true /htmlescape/1.1.1: resolution: {integrity: sha512-eVcrzgbR4tim7c7soKQKtxa/kQM4TzjnlU83rcZ9bHU6t31ehfV7SktN6McWgwPWg+JYMA/O3qpGxBvFq1z2Jg==} @@ -7217,6 +7283,11 @@ packages: call-bind: 1.0.2 dev: true + /is-windows/1.0.2: + resolution: {integrity: sha512-eXK1UInq2bPmjyX6e3VHIzMLobc4J94i4AWn+Hpq3OU5KkrRC96OAcR3PRJ/pGu6m8TRnBHP9dkXQVsT/COVIA==} + engines: {node: '>=0.10.0'} + dev: false + /is-wsl/1.1.0: resolution: {integrity: sha512-gfygJYZ2gLTDlmbWMI0CE2MwnFzSN/2SZfkMlItC4K/JBlsWVDB0bO6XhqcY13YXE7iMcAJnzTCJjPiTeJJ0Mw==} engines: {node: '>=4'} @@ -7246,7 +7317,25 @@ packages: /istanbul-lib-coverage/3.2.0: resolution: {integrity: sha512-eOeJ5BHCmHYvQK7xt9GkdHuzuCGS1Y6g9Gvnx3Ym33fz/HpLRYxiS0wHNr+m/MBC8B647Xt608vCDEvhl9c6Mw==} engines: {node: '>=8'} - dev: true + + /istanbul-lib-hook/3.0.0: + resolution: {integrity: sha512-Pt/uge1Q9s+5VAZ+pCo16TYMWPBIl+oaNIjgLQxcX0itS6ueeaA+pEfThZpH8WxhFgCiEb8sAJY6MdUKgiIWaQ==} + engines: {node: '>=8'} + dependencies: + append-transform: 2.0.0 + dev: false + + /istanbul-lib-instrument/4.0.3: + resolution: {integrity: sha512-BXgQl9kf4WTCPCCpmFGoJkz/+uhvm7h7PFKUYxh7qarQd3ER33vHG//qaE8eN25l07YqZPpHXU9I09l/RD5aGQ==} + engines: {node: '>=8'} + dependencies: + '@babel/core': 7.20.2 + '@istanbuljs/schema': 0.1.3 + istanbul-lib-coverage: 3.2.0 + semver: 6.3.0 + transitivePeerDependencies: + - supports-color + dev: false /istanbul-lib-instrument/5.2.1: resolution: {integrity: sha512-pzqtp31nLv/XFOzXGuvhCb8qhjmTVo5vjVk19XE4CRlSWz0KoeJ3bw9XsA7nOp9YBf4qHjwBxkDzKcME/J29Yg==} @@ -7261,6 +7350,18 @@ packages: - supports-color dev: true + /istanbul-lib-processinfo/2.0.3: + resolution: {integrity: sha512-NkwHbo3E00oybX6NGJi6ar0B29vxyvNwoC7eJ4G4Yq28UfY758Hgn/heV8VRFhevPED4LXfFz0DQ8z/0kw9zMg==} + engines: {node: '>=8'} + dependencies: + archy: 1.0.0 + cross-spawn: 7.0.3 + istanbul-lib-coverage: 3.2.0 + p-map: 3.0.0 + rimraf: 3.0.2 + uuid: 8.3.2 + dev: false + /istanbul-lib-report/3.0.0: resolution: {integrity: sha512-wcdi+uAKzfiGT2abPpKZ0hSU1rGQjUQnLvtY5MpQ7QCTahD3VODhcu4wcfY1YtkGaDD5yuydOLINXsfbus9ROw==} engines: {node: '>=8'} @@ -7268,7 +7369,6 @@ packages: istanbul-lib-coverage: 3.2.0 make-dir: 3.1.0 supports-color: 7.2.0 - dev: true /istanbul-lib-source-maps/4.0.1: resolution: {integrity: sha512-n3s8EwkdFIJCG3BPKBYvskgXGoy88ARzvegkitk60NxRdwltLOTaH7CUiMRXvwYorl0Q712iEjcWB+fK/MrWVw==} @@ -7279,7 +7379,6 @@ packages: source-map: 0.6.1 transitivePeerDependencies: - supports-color - dev: true /istanbul-reports/3.1.5: resolution: {integrity: sha512-nUsEMa9pBt/NOHqbcbeJEgqIlY/K7rVWUX6Lql2orY5e9roQOthbR3vtY4zzf2orPELg80fnxxk9zUyPlgwD1w==} @@ -7287,7 +7386,13 @@ packages: dependencies: html-escaper: 2.0.2 istanbul-lib-report: 3.0.0 - dev: true + + /jackspeak/1.4.2: + resolution: {integrity: sha512-GHeGTmnuaHnvS+ZctRB01bfxARuu9wW83ENbuiweu07SFcVlZrJpcshSre/keGT7YGBhLHg/+rXCNSrsEHKU4Q==} + engines: {node: '>=8'} + dependencies: + cliui: 7.0.4 + dev: false /jest-changed-files/28.1.3: resolution: {integrity: sha512-esaOfUWJXk2nfZt9SPyC8gA1kNfdKLkQWyzsMlqq8msYSlNKfmZxfRgZn4Cd4MGVUF+7v6dBs0d5TOAKa7iIiA==} @@ -7723,7 +7828,6 @@ packages: dependencies: argparse: 1.0.10 esprima: 4.0.1 - dev: true /js-yaml/4.1.0: resolution: {integrity: sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==} @@ -7961,6 +8065,25 @@ packages: type-check: 0.4.0 dev: true + /libtap/1.4.0: + resolution: {integrity: sha512-STLFynswQ2A6W14JkabgGetBNk6INL1REgJ9UeNKw5llXroC2cGLgKTqavv0sl8OLVztLLipVKMcQ7yeUcqpmg==} + engines: {node: '>=10'} + dependencies: + async-hook-domain: 2.0.4 + bind-obj-methods: 3.0.0 + diff: 4.0.2 + function-loop: 2.0.1 + minipass: 3.3.4 + own-or: 1.0.0 + own-or-env: 1.0.2 + signal-exit: 3.0.7 + stack-utils: 2.0.5 + tap-parser: 11.0.2 + tap-yaml: 1.0.2 + tcompare: 5.0.7 + trivial-deferred: 1.1.2 + dev: false + /lines-and-columns/1.2.4: resolution: {integrity: sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==} @@ -7975,7 +8098,6 @@ packages: engines: {node: '>=8'} dependencies: p-locate: 4.1.0 - dev: true /locate-path/6.0.0: resolution: {integrity: sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==} @@ -8000,6 +8122,10 @@ packages: resolution: {integrity: sha512-C5N2Z3DgnnKr0LOpv/hKCgKdb7ZZwafIrsesve6lmzvZIRZRGaZ/l6Q8+2W7NaT+ZwO3fFlSCzCzrDCFdJfZ4g==} dev: false + /lodash.flattendeep/4.4.0: + resolution: {integrity: sha512-uHaJFihxmJcEX3kT4I23ABqKKalJ/zDrDg0lsFtc1h+3uw49SIJ5beyhx5ExVRti3AvKoOJngIj7xz3oylPdWQ==} + dev: false + /lodash.includes/4.3.0: resolution: {integrity: sha512-W3Bx6mdkRTGtlJISOvVD/lbqjTlPPUDTMnlXZFnVwi9NKJ6tiAk6LVdlhZMm17VZisqhKcgzpO5Wz91PCt5b0w==} dev: false @@ -8121,7 +8247,6 @@ packages: /make-error/1.3.6: resolution: {integrity: sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==} - dev: true /make-fetch-happen/10.2.1: resolution: {integrity: sha512-NgOPbRiaQM10DYXvN3/hhGVI2M5MtITFryzBGxHM5p4wnFxsVCbxkrBrDsk+EZ5OB4jEOT7AjDxtdF+KVEFT7w==} @@ -8546,13 +8671,27 @@ packages: resolution: {integrity: sha512-O5lz91xSOeoXP6DulyHfllpq+Eg00MWitZIbtPfoSEvqIHdl5gfcY6hYzDWnj0qD5tz52PI08u9qUvSVeUBeHw==} dev: true - /node-rdkafka/2.15.0: - resolution: {integrity: sha512-LXeliPuuWC9QaeQL48RWbXz1AUl6M0JAbZJN800VHYMXO+jO7GBc97rtClBau+fMj2qxFiPw+d0IovIIcgaVdw==} + /node-preload/0.2.1: + resolution: {integrity: sha512-RM5oyBy45cLEoHqCeh+MNuFAxO0vTFBLskvQbOKnEE7YTTSN4tbN8QWDIPQ6L+WvKsB/qLEGpYe2ZZ9d4W9OIQ==} + engines: {node: '>=8'} + dependencies: + process-on-spawn: 1.0.0 + dev: false + + /node-rdkafka-acosom/2.16.1_mwhvu7sfp6vq5ryuwb6hlbjfka: + resolution: {integrity: sha512-DuzWrv2u0M97LCYWY6W/opX331bUIFwLnOZ31147u2DfrGyqZjuMZAOv70TXFQmaxyGHZ84ipQzVLNWbBbZl0g==} engines: {node: '>=6.0.0'} requiresBuild: true dependencies: bindings: 1.5.0 nan: 2.17.0 + tap: 16.3.4_mwhvu7sfp6vq5ryuwb6hlbjfka + transitivePeerDependencies: + - coveralls + - flow-remove-types + - supports-color + - ts-node + - typescript dev: false /node-releases/2.0.6: @@ -8613,6 +8752,42 @@ packages: engines: {node: '>=0.10.0'} dev: true + /nyc/15.1.0: + resolution: {integrity: sha512-jMW04n9SxKdKi1ZMGhvUTHBN0EICCRkHemEoE5jm6mTYcqcdas0ATzgUgejlQUHMvpnOZqGB5Xxsv9KxJW1j8A==} + engines: {node: '>=8.9'} + hasBin: true + dependencies: + '@istanbuljs/load-nyc-config': 1.1.0 + '@istanbuljs/schema': 0.1.3 + caching-transform: 4.0.0 + convert-source-map: 1.9.0 + decamelize: 1.2.0 + find-cache-dir: 3.3.2 + find-up: 4.1.0 + foreground-child: 2.0.0 + get-package-type: 0.1.0 + glob: 7.2.3 + istanbul-lib-coverage: 3.2.0 + istanbul-lib-hook: 3.0.0 + istanbul-lib-instrument: 4.0.3 + istanbul-lib-processinfo: 2.0.3 + istanbul-lib-report: 3.0.0 + istanbul-lib-source-maps: 4.0.1 + istanbul-reports: 3.1.5 + make-dir: 3.1.0 + node-preload: 0.2.1 + p-map: 3.0.0 + process-on-spawn: 1.0.0 + resolve-from: 5.0.0 + rimraf: 3.0.2 + signal-exit: 3.0.7 + spawn-wrap: 2.0.0 + test-exclude: 6.0.0 + yargs: 15.4.1 + transitivePeerDependencies: + - supports-color + dev: false + /object-assign/4.1.1: resolution: {integrity: sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==} engines: {node: '>=0.10.0'} @@ -8682,6 +8857,11 @@ packages: is-wsl: 2.2.0 dev: false + /opener/1.5.2: + resolution: {integrity: sha512-ur5UIdyw5Y7yEj9wLzhqXiy6GZ3Mwx0yGI+5sMn2r0N0v3cKJvUmFH5yPP+WXh9e0xfyzyJX95D8l088DNFj7A==} + hasBin: true + dev: false + /opn/5.5.0: resolution: {integrity: sha512-PqHpggC9bLV0VeWcdKhkpxY+3JTzetLSqTCWL/z/tFIbI6G8JCjondXklT1JinczLz2Xib62sSp0T/gKT4KksA==} engines: {node: '>=4'} @@ -8734,6 +8914,16 @@ packages: minimist: 1.2.7 dev: false + /own-or-env/1.0.2: + resolution: {integrity: sha512-NQ7v0fliWtK7Lkb+WdFqe6ky9XAzYmlkXthQrBbzlYbmFKoAYbDDcwmOm6q8kOuwSRXW8bdL5ORksploUJmWgw==} + dependencies: + own-or: 1.0.0 + dev: false + + /own-or/1.0.0: + resolution: {integrity: sha512-NfZr5+Tdf6MB8UI9GLvKRs4cXY8/yB0w3xtt84xFdWy8hkGjn+JFc60VhzS/hFRfbyxFcGYMTjnF4Me+RbbqrA==} + dev: false + /p-defer/3.0.0: resolution: {integrity: sha512-ugZxsxmtTln604yeYd29EGrNhazN2lywetzpKhfmQjW/VJmhpDmWbiX+h0zL8V91R0UXkhb3KtPmyq9PZw3aYw==} engines: {node: '>=8'} @@ -8756,7 +8946,6 @@ packages: engines: {node: '>=6'} dependencies: p-try: 2.2.0 - dev: true /p-limit/3.1.0: resolution: {integrity: sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==} @@ -8769,7 +8958,6 @@ packages: engines: {node: '>=8'} dependencies: p-limit: 2.3.0 - dev: true /p-locate/5.0.0: resolution: {integrity: sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==} @@ -8783,6 +8971,13 @@ packages: engines: {node: '>=6'} dev: false + /p-map/3.0.0: + resolution: {integrity: sha512-d3qXVTF/s+W+CdJ5A29wywV2n8CQQYahlgz2bFiA+4eVNJbHJodPZ+/gXwPGh0bOqA+j8S+6+ckmvLGPk1QpxQ==} + engines: {node: '>=8'} + dependencies: + aggregate-error: 3.1.0 + dev: false + /p-map/4.0.0: resolution: {integrity: sha512-/bjOqmgETBYB5BoEeGVea8dmvHb2m9GLy1E9W43yeyfP6QQCZGFNa+XRceJEuDB6zqr+gKpIAmlLebMpykw/MQ==} engines: {node: '>=10'} @@ -8800,7 +8995,6 @@ packages: /p-try/2.2.0: resolution: {integrity: sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==} engines: {node: '>=6'} - dev: true /pac-proxy-agent/5.0.0: resolution: {integrity: sha512-CcFG3ZtnxO8McDigozwE3AqAw15zDvGH+OjXO4kzf7IkEKkQ4gxQ+3sdF50WmhQ4P/bVusXcqNE2S3XrNURwzQ==} @@ -8828,6 +9022,16 @@ packages: netmask: 2.0.2 dev: false + /package-hash/4.0.0: + resolution: {integrity: sha512-whdkPIooSu/bASggZ96BWVvZTRMOFxnyUG5PnTSGKoJE2gd5mbVNmR2Nj20QFzxYYgAXpoqC+AiXzl+UMRh7zQ==} + engines: {node: '>=8'} + dependencies: + graceful-fs: 4.2.10 + hasha: 5.2.2 + lodash.flattendeep: 4.4.0 + release-zalgo: 1.0.0 + dev: false + /packet-reader/1.0.0: resolution: {integrity: sha512-HAKu/fG3HpHFO0AA8WE8q2g+gBJaZ9MG7fcKk+IJPLTGAD6Psw4443l+9DGRbOIh3/aXr7Phy0TjilYivJo5XQ==} dev: false @@ -8884,7 +9088,6 @@ packages: /path-exists/4.0.0: resolution: {integrity: sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==} engines: {node: '>=8'} - dev: true /path-is-absolute/1.0.1: resolution: {integrity: sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==} @@ -8893,7 +9096,6 @@ packages: /path-key/3.1.1: resolution: {integrity: sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==} engines: {node: '>=8'} - dev: true /path-parse/1.0.7: resolution: {integrity: sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==} @@ -9046,7 +9248,6 @@ packages: engines: {node: '>=8'} dependencies: find-up: 4.1.0 - dev: true /postgres-array/2.0.0: resolution: {integrity: sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==} @@ -9120,6 +9321,13 @@ packages: resolution: {integrity: sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==} dev: true + /process-on-spawn/1.0.0: + resolution: {integrity: sha512-1WsPDsUSMmZH5LeMLegqkPDrsGgsWwk1Exipy2hvB0o/F0ASzbpIctSCcZIK1ykJvtTJULEH+20WOFjMvGnCTg==} + engines: {node: '>=8'} + dependencies: + fromentries: 1.3.2 + dev: false + /process-warning/2.0.0: resolution: {integrity: sha512-+MmoAXoUX+VTHAlwns0h+kFUWFs/3FZy+ZuchkgjyOu3oioLAo2LB5aCfKPh2+P9O18i3m43tUEv3YqttSy0Ww==} dev: false @@ -9473,6 +9681,13 @@ packages: jsesc: 0.5.0 dev: false + /release-zalgo/1.0.0: + resolution: {integrity: sha512-gUAyHVHPPC5wdqX/LG4LWtRYtgjxyX78oanFNTMMyFEfOqdC54s3eE82imuWKbOeqYht2CrNf64Qb8vgmmtZGA==} + engines: {node: '>=4'} + dependencies: + es6-error: 4.1.1 + dev: false + /remove-trailing-separator/1.1.0: resolution: {integrity: sha512-/hS+Y0u3aOfIETiaiirUFwDBDzmXPvO+jAfKTitUngIPzdKc6Z0LoFjM/CK5PL4C+eKwHohlHAb6H0VFfmmUsw==} dev: false @@ -9486,6 +9701,10 @@ packages: engines: {node: '>=0.10.0'} dev: true + /require-main-filename/2.0.0: + resolution: {integrity: sha512-NKN5kMDylKuldxYLSUfrbo5Tuzh4hd+2E8NPPX02mZtn1VuREQToYe/ZdlJy+J3uCpfaiGF05e7B8W0iXbQHmg==} + dev: false + /requires-port/1.0.0: resolution: {integrity: sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ==} dev: false @@ -9510,7 +9729,6 @@ packages: /resolve-from/5.0.0: resolution: {integrity: sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw==} engines: {node: '>=8'} - dev: true /resolve.exports/1.1.0: resolution: {integrity: sha512-J1l+Zxxp4XK3LUDZ9m60LRJF/mAe4z6a4xyabPHk7pvK5t35dACV32iIjJDFeWZFfZlO29w6SZ67knR0tHzJtQ==} @@ -9671,12 +9889,10 @@ packages: engines: {node: '>=8'} dependencies: shebang-regex: 3.0.0 - dev: true /shebang-regex/3.0.0: resolution: {integrity: sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==} engines: {node: '>=8'} - dev: true /shell-quote/1.8.0: resolution: {integrity: sha512-QHsz8GgQIGKlRi24yFc6a6lN69Idnx634w49ay6+jA5yFh7a1UY+4Rp6HPx/L/1zcEDPEij8cIsiqR6bQsE5VQ==} @@ -9826,7 +10042,6 @@ packages: dependencies: buffer-from: 1.1.2 source-map: 0.6.1 - dev: true /source-map/0.5.7: resolution: {integrity: sha512-LbrmJOMUSdEVxIKvdcJzQC+nQhe8FUZQTXQy6+I75skNgn3OoQ0DZA8YnFa7gp8tqtL3KPf1kmo0R5DoApeSGQ==} @@ -9842,13 +10057,24 @@ packages: deprecated: Please use @jridgewell/sourcemap-codec instead dev: true + /spawn-wrap/2.0.0: + resolution: {integrity: sha512-EeajNjfN9zMnULLwhZZQU3GWBoFNkbngTUPfaawT4RkMiviTxcX0qfhVbGey39mfctfDHkWtuecgQ8NJcyQWHg==} + engines: {node: '>=8'} + dependencies: + foreground-child: 2.0.0 + is-windows: 1.0.2 + make-dir: 3.1.0 + rimraf: 3.0.2 + signal-exit: 3.0.7 + which: 2.0.2 + dev: false + /split2/4.1.0: resolution: {integrity: sha512-VBiJxFkxiXRlUIeyMQi8s4hgvKCSjtknJv/LVYbrgALPwf5zSKmEwV9Lst25AkvMDnvxODugjdl6KZgwKM1WYQ==} engines: {node: '>= 10.x'} /sprintf-js/1.0.3: resolution: {integrity: sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==} - dev: true /ssri/9.0.1: resolution: {integrity: sha512-o57Wcn66jMQvfHG1FlYbWeZWW/dHZhJXjpIcTfXldXEk5nz5lStPo3mK0OJQfGR3RbZUlbISexbljkJzuEj/8Q==} @@ -9866,7 +10092,6 @@ packages: engines: {node: '>=10'} dependencies: escape-string-regexp: 2.0.0 - dev: true /standard-as-callback/2.1.0: resolution: {integrity: sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==} @@ -10006,7 +10231,6 @@ packages: /strip-bom/4.0.0: resolution: {integrity: sha512-3xurFv5tEgii33Zi8Jtp55wEIILR9eh34FAW00PZf+JnSsTmV/ioewSgQl97JHvgjoRGwPShsWm+IdrxB35d0w==} engines: {node: '>=8'} - dev: true /strip-final-newline/2.0.0: resolution: {integrity: sha512-BrpvfNAE3dcvq7ll3xVumzjKjZQ5tI1sEUIKr3Uoks0XUl45St3FlatVqef9prk4jRDzhW6WZg+3bk93y6pLjA==} @@ -10086,6 +10310,91 @@ packages: resolution: {integrity: sha512-y3JaeRSplks6NYQuCOj3ZFMO3j60rTwbuKCvZxsAraGYH2epusatvZ0baZYA01WsGqJBq/Dl6vOrMUJqyMj8kA==} dev: false + /tap-mocha-reporter/5.0.3: + resolution: {integrity: sha512-6zlGkaV4J+XMRFkN0X+yuw6xHbE9jyCZ3WUKfw4KxMyRGOpYSRuuQTRJyWX88WWuLdVTuFbxzwXhXuS2XE6o0g==} + engines: {node: '>= 8'} + hasBin: true + dependencies: + color-support: 1.1.3 + debug: 4.3.4 + diff: 4.0.2 + escape-string-regexp: 2.0.0 + glob: 7.2.3 + tap-parser: 11.0.2 + tap-yaml: 1.0.2 + unicode-length: 2.1.0 + transitivePeerDependencies: + - supports-color + dev: false + + /tap-parser/11.0.2: + resolution: {integrity: sha512-6qGlC956rcORw+fg7Fv1iCRAY8/bU9UabUAhs3mXRH6eRmVZcNPLheSXCYaVaYeSwx5xa/1HXZb1537YSvwDZg==} + engines: {node: '>= 8'} + hasBin: true + dependencies: + events-to-array: 1.1.2 + minipass: 3.3.4 + tap-yaml: 1.0.2 + dev: false + + /tap-yaml/1.0.2: + resolution: {integrity: sha512-GegASpuqBnRNdT1U+yuUPZ8rEU64pL35WPBpCISWwff4dErS2/438barz7WFJl4Nzh3Y05tfPidZnH+GaV1wMg==} + dependencies: + yaml: 1.10.2 + dev: false + + /tap/16.3.4_mwhvu7sfp6vq5ryuwb6hlbjfka: + resolution: {integrity: sha512-SAexdt2ZF4XBgye6TPucFI2y7VE0qeFXlXucJIV1XDPCs+iJodk0MYacr1zR6Ycltzz7PYg8zrblDXKbAZM2LQ==} + engines: {node: '>=12'} + hasBin: true + peerDependencies: + coveralls: ^3.1.1 + flow-remove-types: '>=2.112.0' + ts-node: '>=8.5.2' + typescript: '>=3.7.2' + peerDependenciesMeta: + coveralls: + optional: true + flow-remove-types: + optional: true + ts-node: + optional: true + typescript: + optional: true + dependencies: + chokidar: 3.5.3 + findit: 2.0.0 + foreground-child: 2.0.0 + fs-exists-cached: 1.0.0 + glob: 7.2.3 + isexe: 2.0.0 + istanbul-lib-processinfo: 2.0.3 + jackspeak: 1.4.2 + libtap: 1.4.0 + minipass: 3.3.4 + mkdirp: 1.0.4 + nyc: 15.1.0 + opener: 1.5.2 + rimraf: 3.0.2 + signal-exit: 3.0.7 + source-map-support: 0.5.21 + tap-mocha-reporter: 5.0.3 + tap-parser: 11.0.2 + tap-yaml: 1.0.2 + tcompare: 5.0.7 + ts-node: 10.9.1_uusaoo756f3l7x5w4ayk4pz43y + typescript: 4.8.4 + which: 2.0.2 + transitivePeerDependencies: + - supports-color + dev: false + bundledDependencies: + - ink + - treport + - '@types/react' + - '@isaacs/import-jsx' + - react + /tar/6.1.12: resolution: {integrity: sha512-jU4TdemS31uABHd+Lt5WEYJuzn+TJTCBLljvIAHZOz6M9Os5pJ4dD+vRFLxPa/n3T0iEFzpi+0x1UfuDZYbRMw==} engines: {node: '>=10'} @@ -10098,6 +10407,13 @@ packages: yallist: 4.0.0 dev: false + /tcompare/5.0.7: + resolution: {integrity: sha512-d9iddt6YYGgyxJw5bjsN7UJUO1kGOtjSlNy/4PoGYAjQS5pAT/hzIoLf1bZCw+uUxRmZJh7Yy1aA7xKVRT9B4w==} + engines: {node: '>=10'} + dependencies: + diff: 4.0.2 + dev: false + /tdigest/0.1.2: resolution: {integrity: sha512-+G0LLgjjo9BZX2MfdvPfH+MKLCrxlXSYec5DaPYP1fe6Iyhf0/fSmJ0bFiZ1F8BT6cGXl2LpltQptzjXKWEkKA==} dependencies: @@ -10137,7 +10453,6 @@ packages: '@istanbuljs/schema': 0.1.3 glob: 7.2.3 minimatch: 3.1.2 - dev: true /text-hex/1.0.0: resolution: {integrity: sha512-uuVGNWzgJ4yhRaNSiubPY7OjISw4sw4E5Uv0wbjp+OzcbmVU/rsT8ujgcXJhn9ypzsgr5vlzpPqP+MBBKcGvbg==} @@ -10262,6 +10577,11 @@ packages: resolution: {integrity: sha512-XrHUvV5HpdLmIj4uVMxHggLbFSZYIn7HEWsqePZcI50pco+MPqJ50wMGY794X7AOOhxOBAjbkqfAbEe/QMp2Lw==} dev: false + /trivial-deferred/1.1.2: + resolution: {integrity: sha512-vDPiDBC3hyP6O4JrJYMImW3nl3c03Tsj9fEXc7Qc/XKa1O7gf5ZtFfIR/E0dun9SnDHdwjna1Z2rSzYgqpxh/g==} + engines: {node: '>= 8'} + dev: false + /ts-node-dev/2.0.0_uusaoo756f3l7x5w4ayk4pz43y: resolution: {integrity: sha512-ywMrhCfH6M75yftYvrvNarLEY+SUXtUvU8/0Z6llrHQVBx12GiFk5sStF8UdfE/yfzk9IAq7O5EEbTQsxlBI8w==} engines: {node: '>=0.8.0'} @@ -10320,7 +10640,6 @@ packages: typescript: 4.8.4 v8-compile-cache-lib: 3.0.1 yn: 3.1.1 - dev: true /tsconfig-paths/3.14.1: resolution: {integrity: sha512-fxDhWnFSLt3VuTwtvJt5fpwxBHg5AdKWMsgcPOOIilyjymcYVZoCQF8fvFRezCNfblEXmi+PcM1eYHeOAgXCOQ==} @@ -10402,6 +10721,11 @@ packages: engines: {node: '>=10'} dev: true + /type-fest/0.8.1: + resolution: {integrity: sha512-4dbzIzqvjtgiM5rw1k5rEHtBANKmdudhGyBEajN01fEyhaAIhsoKNy6y7+IN93IfpFtwY9iqi7kD+xwKhQsNJA==} + engines: {node: '>=8'} + dev: false + /type-fest/1.4.0: resolution: {integrity: sha512-yGSza74xk0UG8k+pLh5oeoYirvIiWo5t0/o3zHHAO2tRDiZcxWP7fywNlXhqb6/r6sWvwi+RsyQMWhVLe4BVuA==} engines: {node: '>=10'} @@ -10421,7 +10745,6 @@ packages: resolution: {integrity: sha512-QCh+85mCy+h0IGff8r5XWzOVSbBO+KfeYrMQh7NJ58QujwcE22u+NUSmUxqF+un70P9GXKxa2HCNiTTMJknyjQ==} engines: {node: '>=4.2.0'} hasBin: true - dev: true /uc.micro/1.0.6: resolution: {integrity: sha512-8Y75pvTYkLJW2hWQHXxoqRgV7qb9B+9vFEtidML+7koHUFapnVJAZ6cKs+Qjz5Aw3aZWHMC6u0wJE3At+nSGwA==} @@ -10481,6 +10804,12 @@ packages: engines: {node: '>=4'} dev: false + /unicode-length/2.1.0: + resolution: {integrity: sha512-4bV582zTV9Q02RXBxSUMiuN/KHo5w4aTojuKTNT96DIKps/SIawFp7cS5Mu25VuY1AioGXrmYyzKZUzh8OqoUw==} + dependencies: + punycode: 2.1.1 + dev: false + /unicode-match-property-ecmascript/2.0.0: resolution: {integrity: sha512-5kaZCrbp5mmbz5ulBkDkbY0SsPOjKqVS35VpL9ulMPfSl0J0Xsm+9Evphv9CoIZFwre7aJoa94AY6seMKGVN5Q==} engines: {node: '>=4'} @@ -10665,7 +10994,6 @@ packages: /v8-compile-cache-lib/3.0.1: resolution: {integrity: sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg==} - dev: true /v8-to-istanbul/9.0.1: resolution: {integrity: sha512-74Y4LqY74kLE6IFyIjPtkSTWzUZmj8tdHT9Ii/26dvQ6K9Dl2NbEfj0XgU2sHCtKgt5VupqhlO/5aWuqS+IY1w==} @@ -10725,6 +11053,10 @@ packages: is-symbol: 1.0.4 dev: true + /which-module/2.0.1: + resolution: {integrity: sha512-iBdZ57RDvnOR9AGBhML2vFZf7h8vmBjhoaZqODJBFWHVtKkDmKuHai3cx5PgVMrX5YDNp27AofYbAwctSS+vhQ==} + dev: false + /which-typed-array/1.1.9: resolution: {integrity: sha512-w9c4xkx6mPidwp7180ckYWfMmvxpjlZuIudNtDf4N/tTAUB8VJbX25qZoAsrtGuYNnGw3pa0AXgbGKRB8/EceA==} engines: {node: '>= 0.4'} @@ -10786,6 +11118,15 @@ packages: resolution: {integrity: sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ==} engines: {node: '>=0.10.0'} + /wrap-ansi/6.2.0: + resolution: {integrity: sha512-r6lPcBGxZXlIcymEu7InxDMhdW0KDxpLgoFLcguasxCaJ/SOIZwINatK9KY/tf+ZrlywOKU0UDj3ATXUBfxJXA==} + engines: {node: '>=8'} + dependencies: + ansi-styles: 4.3.0 + string-width: 4.2.3 + strip-ansi: 6.0.1 + dev: false + /wrap-ansi/7.0.0: resolution: {integrity: sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==} engines: {node: '>=10'} @@ -10869,6 +11210,10 @@ packages: resolution: {integrity: sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==} engines: {node: '>=0.4'} + /y18n/4.0.3: + resolution: {integrity: sha512-JKhqTOwSrqNA1NY5lSztJ1GrBiUodLMmIZuLiDaMRJ+itFd+ABVE8XBjOvIWL+rSqNDC74LCSFmlb/U4UZ4hJQ==} + dev: false + /y18n/5.0.8: resolution: {integrity: sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==} engines: {node: '>=10'} @@ -10885,6 +11230,14 @@ packages: engines: {node: '>= 6'} dev: false + /yargs-parser/18.1.3: + resolution: {integrity: sha512-o50j0JeToy/4K6OZcaQmW6lyXXKhq7csREXcDwk2omFPJEwUNOVtJKvmDr9EI1fAJZUyZcRF7kxGBWmRXudrCQ==} + engines: {node: '>=6'} + dependencies: + camelcase: 5.3.1 + decamelize: 1.2.0 + dev: false + /yargs-parser/20.2.9: resolution: {integrity: sha512-y11nGElTIV+CT3Zv9t7VKl+Q3hTQoT9a1Qzezhhl6Rp21gJ/IVTW7Z3y9EWXhuUBC2Shnf+DX0antecpAwSP8w==} engines: {node: '>=10'} @@ -10894,6 +11247,23 @@ packages: engines: {node: '>=12'} dev: true + /yargs/15.4.1: + resolution: {integrity: sha512-aePbxDmcYW++PaqBsJ+HYUFwCdv4LVvdnhBy78E57PIor8/OVvhMrADFFEDh8DHDFRv/O9i3lPhsENjO7QX0+A==} + engines: {node: '>=8'} + dependencies: + cliui: 6.0.0 + decamelize: 1.2.0 + find-up: 4.1.0 + get-caller-file: 2.0.5 + require-directory: 2.1.1 + require-main-filename: 2.0.0 + set-blocking: 2.0.0 + string-width: 4.2.3 + which-module: 2.0.1 + y18n: 4.0.3 + yargs-parser: 18.1.3 + dev: false + /yargs/16.2.0: resolution: {integrity: sha512-D1mvvtDG0L5ft/jGWkLpG1+m0eQxOfaBvTNELraWj22wSVUMWxZUvYgJYcKh6jGGIkJFhH4IZPQhR4TKpc8mBw==} engines: {node: '>=10'} @@ -10922,7 +11292,6 @@ packages: /yn/3.1.1: resolution: {integrity: sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==} engines: {node: '>=6'} - dev: true /yocto-queue/0.1.0: resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==} diff --git a/plugin-server/src/kafka/admin.ts b/plugin-server/src/kafka/admin.ts index 3aec4021cd19d..4f244ada0e162 100644 --- a/plugin-server/src/kafka/admin.ts +++ b/plugin-server/src/kafka/admin.ts @@ -1,4 +1,4 @@ -import { AdminClient, CODES, GlobalConfig, IAdminClient, LibrdKafkaError } from 'node-rdkafka' +import { AdminClient, CODES, GlobalConfig, IAdminClient, LibrdKafkaError } from 'node-rdkafka-acosom' import { status } from '../utils/status' diff --git a/plugin-server/src/kafka/batch-consumer.ts b/plugin-server/src/kafka/batch-consumer.ts index 7e796bba2c3bd..84de09d4e3d61 100644 --- a/plugin-server/src/kafka/batch-consumer.ts +++ b/plugin-server/src/kafka/batch-consumer.ts @@ -1,4 +1,4 @@ -import { GlobalConfig, KafkaConsumer, Message } from 'node-rdkafka' +import { GlobalConfig, KafkaConsumer, Message } from 'node-rdkafka-acosom' import { exponentialBuckets, Histogram } from 'prom-client' import { status } from '../utils/status' @@ -74,7 +74,22 @@ export const startBatchConsumer = async ({ 'enable.partition.eof': true, 'queued.min.messages': 100000, // 100000 is the default 'queued.max.messages.kbytes': 102400, // 1048576 is the default, we go smaller to reduce mem usage. - 'partition.assignment.strategy': 'roundrobin', // KafkaJS uses round robin, and we need to be compatible with it. + // Use cooperative-sticky rebalancing strategy, which is the + // [default](https://kafka.apache.org/documentation/#consumerconfigs_partition.assignment.strategy) + // in the Java Kafka Client. There its actually + // RangeAssignor,CooperativeStickyAssignor i.e. it mixes eager and + // cooperative strategies. This is however explicitly mentioned to not + // be supported in the [librdkafka library config + // docs](https://github.com/confluentinc/librdkafka/blob/master/CONFIGURATION.md#partitionassignmentstrategy) + // so we just use cooperative-sticky. If there are other consumer + // members with other strategies already running, you'll need to delete + // e.g. the replicaset for them if on k8s. + // + // See + // https://www.confluent.io/en-gb/blog/incremental-cooperative-rebalancing-in-kafka/ + // for details on the advantages of this rebalancing strategy as well as + // how it works. + 'partition.assignment.strategy': 'cooperative-sticky', rebalance_cb: true, offset_commit_cb: true, }) diff --git a/plugin-server/src/kafka/config.ts b/plugin-server/src/kafka/config.ts index 31261b261c2cd..2c92aa4ea8001 100644 --- a/plugin-server/src/kafka/config.ts +++ b/plugin-server/src/kafka/config.ts @@ -1,4 +1,4 @@ -import { GlobalConfig } from 'node-rdkafka' +import { GlobalConfig } from 'node-rdkafka-acosom' import { hostname } from 'os' import { KafkaConfig } from '../utils/db/hub' diff --git a/plugin-server/src/kafka/consumer.ts b/plugin-server/src/kafka/consumer.ts index 149f33a2d474a..b74b5e75bced7 100644 --- a/plugin-server/src/kafka/consumer.ts +++ b/plugin-server/src/kafka/consumer.ts @@ -7,7 +7,7 @@ import { Message, TopicPartition, TopicPartitionOffset, -} from 'node-rdkafka' +} from 'node-rdkafka-acosom' import { latestOffsetTimestampGauge } from '../main/ingestion-queues/metrics' import { status } from '../utils/status' diff --git a/plugin-server/src/kafka/producer.ts b/plugin-server/src/kafka/producer.ts index 055901f548711..f2f429244f1f1 100644 --- a/plugin-server/src/kafka/producer.ts +++ b/plugin-server/src/kafka/producer.ts @@ -3,7 +3,7 @@ import { HighLevelProducer as RdKafkaProducer, NumberNullUndefined, ProducerGlobalConfig, -} from 'node-rdkafka' +} from 'node-rdkafka-acosom' import { status } from '../utils/status' diff --git a/plugin-server/src/main/ingestion-queues/session-recording/blob-ingester/offset-manager.ts b/plugin-server/src/main/ingestion-queues/session-recording/blob-ingester/offset-manager.ts index 41a64b2c50807..b1971461281c1 100644 --- a/plugin-server/src/main/ingestion-queues/session-recording/blob-ingester/offset-manager.ts +++ b/plugin-server/src/main/ingestion-queues/session-recording/blob-ingester/offset-manager.ts @@ -16,7 +16,7 @@ * track everything in this single process */ -import { KafkaConsumer } from 'node-rdkafka' +import { KafkaConsumer } from 'node-rdkafka-acosom' import { status } from '../../../../utils/status' diff --git a/plugin-server/src/main/ingestion-queues/session-recording/session-recordings-blob-consumer.ts b/plugin-server/src/main/ingestion-queues/session-recording/session-recordings-blob-consumer.ts index e945d0886aebe..9b83a9592ec1e 100644 --- a/plugin-server/src/main/ingestion-queues/session-recording/session-recordings-blob-consumer.ts +++ b/plugin-server/src/main/ingestion-queues/session-recording/session-recordings-blob-consumer.ts @@ -1,5 +1,5 @@ import { mkdirSync, rmSync } from 'node:fs' -import { CODES, HighLevelProducer as RdKafkaProducer, Message } from 'node-rdkafka' +import { CODES, HighLevelProducer as RdKafkaProducer, Message } from 'node-rdkafka-acosom' import { KAFKA_SESSION_RECORDING_EVENTS } from '../../../config/kafka-topics' import { BatchConsumer, startBatchConsumer } from '../../../kafka/batch-consumer' diff --git a/plugin-server/src/main/ingestion-queues/session-recording/session-recordings-consumer.ts b/plugin-server/src/main/ingestion-queues/session-recording/session-recordings-consumer.ts index 3fe62dc36c777..f19ddfcf6fe47 100644 --- a/plugin-server/src/main/ingestion-queues/session-recording/session-recordings-consumer.ts +++ b/plugin-server/src/main/ingestion-queues/session-recording/session-recordings-consumer.ts @@ -1,5 +1,5 @@ import { PluginEvent } from '@posthog/plugin-scaffold' -import { HighLevelProducer as RdKafkaProducer, Message, NumberNullUndefined } from 'node-rdkafka' +import { HighLevelProducer as RdKafkaProducer, Message, NumberNullUndefined } from 'node-rdkafka-acosom' import { KAFKA_CLICKHOUSE_SESSION_RECORDING_EVENTS, diff --git a/plugin-server/tests/main/ingestion-queues/session-recording/blob-ingester/offset-manager.test.ts b/plugin-server/tests/main/ingestion-queues/session-recording/blob-ingester/offset-manager.test.ts index c6a544be0c544..d7db02670fc03 100644 --- a/plugin-server/tests/main/ingestion-queues/session-recording/blob-ingester/offset-manager.test.ts +++ b/plugin-server/tests/main/ingestion-queues/session-recording/blob-ingester/offset-manager.test.ts @@ -1,4 +1,4 @@ -import { KafkaConsumer } from 'node-rdkafka' +import { KafkaConsumer } from 'node-rdkafka-acosom' import { OffsetManager } from '../../../../../src/main/ingestion-queues/session-recording/blob-ingester/offset-manager' diff --git a/plugin-server/tests/main/ingestion-queues/session-recording/session-recordings-consumer.test.ts b/plugin-server/tests/main/ingestion-queues/session-recording/session-recordings-consumer.test.ts index dd2654750cedc..5ecb35692c592 100644 --- a/plugin-server/tests/main/ingestion-queues/session-recording/session-recordings-consumer.test.ts +++ b/plugin-server/tests/main/ingestion-queues/session-recording/session-recordings-consumer.test.ts @@ -1,4 +1,4 @@ -import LibrdKafkaError from 'node-rdkafka/lib/error' +import LibrdKafkaError from 'node-rdkafka-acosom/lib/error' import { Pool } from 'pg' import { defaultConfig } from '../../../../src/config/config'