{"level":30,"time":1731409383543,"pid":19,"hostname":"c94776d321b0","name":"qryn","reqId":"req-1zjk","req":{"method":"POST","url":"/loki/api/v1/push","hostname":"192.168.50.63:3100","remoteAddress":"192.168.7.240","remotePort":36594},"msg":"incoming request"} {"level":50,"time":1731409383612,"pid":19,"hostname":"c94776d321b0","name":"qryn","msg":"rawRequest error: INSERT INTO qryn.time_series(date, fingerprint, labels, name, type) FORMAT JSONEachRow"} {"level":50,"time":1731409383613,"pid":19,"hostname":"c94776d321b0","name":"qryn","reqId":"req-1zjk","err":"Database push error\nError: Database push error\n at Worker. (/app/lib/db/clickhouse.js:86:27)\n at Worker.emit (node:events:519:28)\n at MessagePort. (node:internal/worker:263:53)\n at [nodejs.internal.kHybridDispatch] (node:internal/event_target:820:20)\n at MessagePort. (node:internal/per_context/messageport:23:28)","msg":"Database push error"} {"level":30,"time":1731409383614,"pid":19,"hostname":"c94776d321b0","name":"qryn","reqId":"req-1zjk","res":{"statusCode":500},"responseTime":70.63062691688538,"msg":"request completed"} {"level":50,"time":1731409383612,"pid":19,"hostname":"c94776d321b0","name":"qryn","msg":"Code: 241. DB::Exception: Memory limit (total) exceeded: would use 1.11 GiB (attempt to allocate chunk of 4550387 bytes), current RSS 481.95 MiB, maximum: 921.60 MiB. OvercommitTracker decision: Query was selected to stop by OvercommitTracker. (MEMORY_LIMIT_EXCEEDED) (version 24.10.1.2812 (official build))\n"} axiosError AxiosError: Request failed with status code 500 at settle (/app/node_modules/axios/dist/node/axios.cjs:2019:12) at IncomingMessage.handleStreamEnd (/app/node_modules/axios/dist/node/axios.cjs:3135:11) at IncomingMessage.emit (node:events:531:35) at endReadableNT (node:internal/streams/readable:1696:12) at process.processTicksAndRejections (node:internal/process/task_queues:82:21) at Axios.request (/app/node_modules/axios/dist/node/axios.cjs:4287:41) at process.processTicksAndRejections (node:internal/process/task_queues:95:5) at async rawRequest (/app/lib/db/clickhouse.js:1403:12) at async TimeoutOrSizeThrottler._flush (/app/lib/db/throttler.js:92:5) at async TimeoutOrSizeThrottler.flush (/app/lib/db/throttler.js:74:7) at async /app/lib/db/throttler.js:162:13 at async Promise.all (index 0) at async Timeout._onTimeout (/app/lib/db/throttler.js:160:9) { code: 'ERR_BAD_RESPONSE', config: { transitional: { silentJSONPars 'Keep-Alive', 'content-type': 'text/plain; charset=UTF-8', 'x-clickhouse-server-display-name': 'eeb9c057b032', 'transfer-encoding': 'chunked', 'x-clickhouse-query-id': '0870a268-0e08-465a-a383-4d8533301b8e', 'x-clickhouse-timezone': 'UTC', 'x-clickhouse-exception-code': '241', 'keep-alive': 'timeout=10, max=9999', 'x-clickhouse-summary': '{"read_rows":"1","read_bytes":"192","written_rows":"1","written_bytes":"192","total_rows_to_read":"0","result_rows":"0","result_bytes":"0","elapsed_ns":"3358191"}' }, config: { transitional: [Object], adapter: [Array], transformRequest: [Array], transformResponse: [Array], timeout: 0, xsrfCookieName: 'XSRF-TOKEN', xsrfHeaderName: 'X-XSRF-TOKEN', maxContentLength: -1, maxBodyLength: Infinity, env: [Object], validateStatus: [Function: validateStatus], headers: [Object [AxiosHeaders]], method: 'post', url: 'http://q22 3a 22 32 30 32 34 2d 31 31 2d 31 32 22 2c 22 66 69 6e 67 65 72 70 72 69 6e 74 22 3a 22 31 32 34 37 31 37 37 35 34 35 38 34 38 31 ... 202 more bytes> }, request: ClientRequest { _events: [Object: null prototype] { abort: [Function (anonymous)], aborted: [Function (anonymous)], connect: [Function (anonymous)], error: [Function (anonymous)], socket: [Function (anonymous)], timeout: [Function (anonymous)], finish: [Function: requestOnFinish] }, _eventsCount: 7, _maxListeners: undefined, outputData: [], outputSize: 0, writable: true, destroyed: true, _last: false, chunkedEncoding: false, shouldKeepAlive: true, maxRequestsOnConnectionReached: false, _defaultKeepAlive: true, useChunkedEncodingByDefault: true, sendDate: false, _removedConnection: false, _removedContLen: false, _removedTE: false, strictContentLength: false, _contentLength: '252', _hasBody: true, _trailer: '', finished: true, _headerSent: true, _closed: true, socket: Socket { connecting: false, _hadError: false, _parent: null, _host: 'clickhouse-server', _closeAfterHandlingError: false, _events: [Object], _readableState: [ReadableState], _writableState: [WritableState], allowHalfOpen: false, _maxListeners: undefined, _eventsCount: 6, _sockname: null, _pendingData: null, _pendingEncoding: '', server: null, _server: null, timeout: 5000, parser: null, _httpMessage: null, [Symbol(async_id_symbol)]: -1, [Symbol(kHandle)]: [TCP], [Symbol(lastWriteQueueSize)]: 0, [Symbol(timeout)]: Timeout { _idleTimeout: 5000, _idlePrev: [TimersList], _idleNext: [TimersList], _idleStart: 120423016, _onTimeout: [Function: bound ], _timerArgs: undefined, _repeat: null, _destroyed: false, [Symbol(refed)]: false, [Symbol(kHasPrimitive)]: false, [Symbol(asyncId)]: 4240639, [Symbol(triggerId)]: 4240637 }, [Symbol(kBuffer)]: null, [Symbol(kBufferCb)]: null, [Symbol(kBufferGen)]: null, [Symbol(shapeMode)]: true, [Symbol(kCapture)]: false, [Symbol(kSetNoDelay)]: true, [Symbol(kSetKeepAlive)]: true, [Symbol(kSetKeepAliveInitialDelay)]: 1, [Symbol(kBytesRead)]: 0, [Symbol(kBytesWritten)]: 0 }, _header: 'POST /?database=qryn&query=INSERT%20INTO%20qryn.time_series(date%2C%20fingerprint%2C%20labels%2C%20name%2C%20type)%20FORMAT%20JSONEachRow HTTP/1.1\r\n' + 'Accept: application/json, text/plain, */*\r\n' + 'Content-Type: application/x-www-form-urlencoded\r\n' + 'User-Agent: axios/1.7.7\r\n' + 'Content-Length: 252\r\n' + 'Accept-Encoding: gzip, compress, deflate, br\r\n' + 'Host: clickhouse-server:8123\r\n' + 'Authorization: Basic cXJ5bjoxMjM0NTY3ODk=\r\n' + 'Connection: keep-alive\r\n' + '\r\n', _keepAliveTimeout: 0, _onPendingData: [Function: nop], agent: Agent { _events: [Object: null prototype], _eventsCount: 2, _maxListeners: undefined, defaultPort: 80, protocol: 'http:', options: [Object: null prototype], requests: [Object: null prototype] {}, sockets: [Object: null prototype] {}, freeSockets: [Object: null prototype], keepAliveMsecs: 1000, keepAlive: true, maxSockets: Infinity, maxFreeSockets: 256, scheduling: 'lifo', maxTotalSockets: Infinity, totalSocketCount: 1, [Symbol(shapeMode)]: false, [Symbol(kCapture)]: false }, socketPath: undefined, method: 'POST', maxHeaderSize: undefined, insecureHTTPParser: undefined, joinDuplicateHeaders: undefined, path: '/?database=qryn&query=INSERT%20INTO%20qryn.time_series(date%2C%20fingerprint%2C%20labels%2C%20name%2C%20type)%20FORMAT%20JSONEachRow', _ended: true, res: IncomingMessage { _events: [Object], _readableState: [ReadableState], _maxListeners: undefined, socket: null, httpVersionMajor: 1, httpVersionMinor: 1, httpVersion: '1.1', complete: true, rawHeaders: [Array], rawTrailers: [], joinDuplicateHeaders: undefined, aborted: false, upgrade: false, url: '', method: null, statusCode: 500, statusMessage: 'Internal Server Error', client: [Socket], _consuming: false, _dumped: false, req: [Circular *1], _eventsCount: 4, responseUrl: 'http://qryn:123456789@clickhouse-server:8123/?database=qryn&query=INSERT%20INTO%20qryn.time_series(date%2C%20fingerprint%2C%20labels%2C%20name%2C%20type)%20FORMAT%20JSONEachRow', redirects: [], [Symbol(shapeMode)]: true, [Symbol(kCapture)]: false, [Symbol(kHeaders)]: [Object], [Symbol(kHeadersCount)]: 20, [Symbol(kTrailers)]: null, [Symbol(kTrailersCount)]: 0 }, aborted: false, timeoutCb: null, upgradeOrConnect: false, parser: null, maxHeadersCount: null, reusedSocket: false, host: 'clickhouse-server', protocol: 'http:', _redirectable: Writable { _events: [Object], _writableState: [WritableState], _maxListeners: undefined, _options: [Object], _ended: true, _ending: true, _redirectCount: 0, _redirects: [], _requestBodyLength: 252, _requestBodyBuffers: [], _eventsCount: 3, _onNativeResponse: [Function (anonymous)], _currentRequest: [Circular *1], _currentUrl: 'http://qryn:123456789@clickhouse-server:8123/?database=qryn&query=INSERT%20INTO%20qryn.time_series(date%2C%20fingerprint%2C%20labels%2C%20name%2C%20type)%20FORMAT%20JSONEachRow', [Symbol(shapeMode)]: true, [Symbol(kCapture)]: false }, [Symbol(shapeMode)]: false, [Symbol(kCapture)]: false, [Symbol(kBytesWritten)]: 0, [Symbol(kNeedDrain)]: false, [Symbol(corked)]: 0, [Symbol(kOutHeaders)]: [Object: null prototype] { accept: [Array], 'content-type': [Array], 'user-agent': [Array], 'content-length': [Array], 'accept-encoding': [Array], host: [Array], authorization: [Array] }, [Symbol(errored)]: null, [Symbol(kHighWaterMark)]: 16384, [Symbol(kRejectNonStandardBodyWrites)]: false, [Symbol(kUniqueHeaders)]: null }, response: { status: 500, statusText: 'Internal Server Error', headers: Object [AxiosHeaders] { date: 'Tue, 12 Nov 2024 11:03:03 GMT', connection: 'Keep-Alive', 'content-type': 'text/plain; charset=UTF-8', 'x-clickhouse-server-display-name': 'eeb9c057b032', 'transfer-encoding': 'chunked', 'x-clickhouse-query-id': '0870a268-0e08-465a-a383-4d8533301b8e', 'x-clickhouse-timezone': 'UTC', 'x-clickhouse-exception-code': '241', 'keep-alive': 'timeout=10, max=9999', 'x-clickhouse-summary': '{"read_rows":"1","read_bytes":"192","written_rows":"1","written_bytes":"192","total_rows_to_read":"0","result_rows":"0","result_bytes":"0","elapsed_ns":"3358191"}' }, config: { transitional: [Object], adapter: [Array], transformRequest: [Array], transformResponse: [Array], timeout: 0, xsrfCookieName: 'XSRF-TOKEN', xsrfHeaderName: 'X-XSRF-TOKEN', maxContentLength: -1, maxBodyLength: Infinity, env: [Object], validateStatus: [Function: validateStatus], headers: [Object [AxiosHeaders]], method: 'post', url: 'http://qryn:123456789@clickhouse-server:8123/?database=qryn&query=INSERT%20INTO%20qryn.time_series(date%2C%20fingerprint%2C%20labels%2C%20name%2C%20type)%20FORMAT%20JSONEachRow', data: }, request: ClientRequest { _events: [Object: null prototype], _eventsCount: 7, _maxListeners: undefined, outputData: [], outputSize: 0, writable: true, destroyed: true, _last: false, chunkedEncoding: false, shouldKeepAlive: true, maxRequestsOnConnectionReached: false, _defaultKeepAlive: true, useChunkedEncodingByDefault: true, sendDate: false, _removedConnection: false, _removedContLen: false, _removedTE: false, strictContentLength: false, _contentLength: '252', _hasBody: true, _trailer: '', finished: true, _headerSent: true, _closed: true, socket: [Socket], _header: 'POST /?database=qryn&query=INSERT%20INTO%20qryn.time_series(date%2C%20fingerprint%2C%20labels%2C%20name%2C%20type)%20FORMAT%20JSONEachRow HTTP/1.1\r\n' + 'Accept: application/json, text/plain, */*\r\n' + 'Content-Type: application/x-www-form-urlencoded\r\n' + 'User-Agent: axios/1.7.7\r\n' + 'Content-Length: 252\r\n' + 'Accept-Encoding: gzip, compress, deflate, br\r\n' + 'Host: clickhouse-server:8123\r\n' + 'Authorization: Basic cXJ5bjoxMjM0NTY3ODk=\r\n' + 'Connection: keep-alive\r\n' + '\r\n', _keepAliveTimeout: 0, _onPendingData: [Function: nop], agent: [Agent], socketPath: undefined, method: 'POST', maxHeaderSize: undefined, insecureHTTPParser: undefined, joinDuplicateHeaders: undefined, path: '/?database=qryn&query=INSERT%20INTO%20qryn.time_series(date%2C%20fingerprint%2C%20labels%2C%20name%2C%20type)%20FORMAT%20JSONEachRow', _ended: true, res: [IncomingMessage], aborted: false, timeoutCb: null, upgradeOrConnect: false, parser: null, maxHeadersCount: null, reusedSocket: false, host: 'clickhouse-server', protocol: 'http:', _redirectable: [Writable], [Symbol(shapeMode)]: false, [Symbol(kCapture)]: false, [Symbol(kBytesWritten)]: 0, [Symbol(kNeedDrain)]: false, [Symbol(corked)]: 0, [Symbol(kOutHeaders)]: [Object: null prototype], [Symbol(errored)]: null, [Symbol(kHighWaterMark)]: 16384, [Symbol(kRejectNonStandardBodyWrites)]: false, [Symbol(kUniqueHeaders)]: null }, data: 'Code: 241. DB::Exception: Memory limit (total) exceeded: would use 1.11 GiB (attempt to allocate chunk of 4550387 bytes), current RSS 481.95 MiB, maximum: 921.60 MiB. OvercommitTracker decision: Query was selected to stop by OvercommitTracker. (MEMORY_LIMIT_EXCEEDED) (version 24.10.1.2812 (official build))\n' }, status: 500 } {"level":50,"time":1731409383613,"pid":19,"hostname":"c94776d321b0","name":"qryn","err":"AXIOS ERROR: AxiosError: Request failed with status code 500 Response data: Code: 241. DB::Exception: Memory limit (total) exceeded: would use 1.11 GiB (attempt to allocate chunk of 4550387 bytes), current RSS 481.95 MiB, maximum: 921.60 MiB. OvercommitTracker decision: Query was selected to stop by OvercommitTracker. (MEMORY_LIMIT_EXCEEDED) (version 24.10.1.2812 (official build))\n\nError: AXIOS ERROR: AxiosError: Request failed with status code 500 Response data: Code: 241. DB::Exception: Memory limit (total) exceeded: would use 1.11 GiB (attempt to allocate chunk of 4550387 bytes), current RSS 481.95 MiB, maximum: 921.60 MiB. OvercommitTracker decision: Query was selected to stop by OvercommitTracker. (MEMORY_LIMIT_EXCEEDED) (version 24.10.1.2812 (official build))\n\n at axiosError (/app/lib/db/throttler.js:29:14)\n at TimeoutOrSizeThrottler.flush (/app/lib/db/throttler.js:77:26)\n at process.processTicksAndRejections (node:internal/process/task_queues:95:5)\n at async /app/lib/db/throttler.js:162:13\n at async Promise.all (index 0)\n at async Timeout._onTimeout (/app/lib/db/throttler.js:160:9)","msg":"AXIOS ERROR"} {"level":50,"time":1731409383625,"pid":19,"hostname":"c94776d321b0","name":"qryn","msg":"rawRequest error: INSERT INTO qryn.samples_v3(fingerprint, timestamp_ns, value, string, type) FORMAT JSONEachRow"} {"level":50,"time":1731409383625,"pid":19,"hostname":"c94776d321b0","name":"qryn","msg":"Code: 241. DB::Exception: Memory limit (total) exceeded: would use 1.12 GiB (attempt to allocate chunk of 5030083 bytes), current RSS 483.02 MiB, maximum: 921.60 MiB. OvercommitTracker decision: Query was selected to stop by OvercommitTracker. (MEMORY_LIMIT_EXCEEDED) (version 24.10.1.2812 (official build))\n"} axiosError AxiosError: Request failed with status code 500 at settle (/app/node_modules/axios/dist/node/axios.cjs:2019:12) at IncomingMessage.handleStreamEnd (/app/node_modules/axios/dist/node/axios.cjs:3135:11) at IncomingMessage.emit (node:events:531:35) at endReadableNT (node:internal/streams/readable:1696:12) at process.processTicksAndRejections (node:internal/process/task_queues:82:21) at Axios.request (/app/node_modules/axios/dist/node/axios.cjs:4287:41) at process.processTicksAndRejections (node:internal/process/task_queues:95:5) at async rawRequest (/app/lib/db/clickhouse.js:1403:12) at async TimeoutOrSizeThrottler._flush (/app/lib/db/throttler.js:92:5) at async TimeoutOrSizeThrottler.flush (/app/lib/db/throttler.js:74:7) at async /app/lib/db/throttler.js:163:13 at async Promise.all (index 0) at async Timeout._onTimeout (/app/lib/db/throttler.js:160:9) { code: 'ERR_BAD_RESPONSE', config: { transitional: { silentJSONParsing: true, forcedJSONParsing: true, clarifyTimeoutError: false }, adapter: [ 'xhr', 'http', 'fetch' ], transformRequest: [ [Function: transformRequest] ], transformResponse: [ [Function: transformResponse] ], timeout: 0, xsrfCookieName: 'XSRF-TOKEN', xsrfHeaderName: 'X-XSRF-TOKEN', maxContentLength: -1, maxBodyLength: Infinity, env: { FormData: [Function], Blob: [class Blob] }, validateStatus: [Function: validateStatus], headers: Object [AxiosHeaders] { Accept: 'application/json, text/plain, */*', 'Content-Type': 'application/x-www-form-urlencoded', 'User-Agent': 'axios/1.7.7', 'Content-Length': '1096251', 'Accept-Encoding': 'gzip, compress, deflate, br' }, method: 'post', url: 'http://qryn:123456789@clickhouse-server:8123/?database=qryn&query=INSERT%20INTO%20qryn.samples_v3(fingerprint%2C%20timestamp_ns%2C%20value%2C%20string%2C%20type)%20FORMAT%20JSONEachRow', data: }, request: ClientRequest { _events: [Object: null prototype] { abort: [Function (anonymous)], aborted: [Function (anonymous)], connect: [Function (anonymous)], error: [Function (anonymous)], socket: [Function (anonymous)], timeout: [Function (anonymous)], finish: [Function: requestOnFinish] }, _eventsCount: 7, _maxListeners: undefined, outputData: [], outputSize: 0, writable: true, destroyed: true, _last: false, chunkedEncoding: false, shouldKeepAlive: true, maxRequestsOnConnectionReached: false, _defaultKeepAlive: true, useChunkedEncodingByDefault: true, sendDate: false, _removedConnection: false, _removedContLen: false, _removedTE: false, strictContentLength: false, _contentLength: '1096251', _hasBody: true, _trailer: '', finished: true, _headerSent: true, _closed: true, socket: Socket { connecting: false, _hadError: false, _parent: null, _host: 'clickhouse-server', _closeAfterHandlingError: false, _events: [Object], _readableState: [ReadableState], _writableState: [WritableState], allowHalfOpen: false, _maxListeners: undefined, _eventsCount: 6, _sockname: null, _pendingData: null, _pendingEncoding: '', server: null, _server: null, timeout: 5000, parser: null, _httpMessage: null, [Symbol(async_id_symbol)]: -1, [Symbol(kHandle)]: [TCP], [Symbol(lastWriteQueueSize)]: 0, [Symbol(timeout)]: Timeout { _idleTimeout: 5000, _idlePrev: [TimersList], _idleNext: [TimersList], _idleStart: 120423029, _onTimeout: [Function: bound ], _timerArgs: undefined, _repeat: null, _destroyed: false, [Symbol(refed)]: false, [Symbol(kHasPrimitive)]: false, [Symbol(asyncId)]: 4240656, [Symbol(triggerId)]: 4240654 }, [Symbol(kBuffer)]: null, [Symbol(kBufferCb)]: null, [Symbol(kBufferGen)]: null, [Symbol(shapeMode)]: true, [Symbol(kCapture)]: false, [Symbol(kSetNoDelay)]: true, [Symbol(kSetKeepAlive)]: true, [Symbol(kSetKeepAliveInitialDelay)]: 1, [Symbol(kBytesRead)]: 0, [Symbol(kBytesWritten)]: 0 }, _header: 'POST /?database=qryn&query=INSERT%20INTO%20qryn.samples_v3(fingerprint%2C%20timestamp_ns%2C%20value%2C%20string%2C%20type)%20FORMAT%20JSONEachRow HTTP/1.1\r\n' + 'Accept: application/json, text/plain, */*\r\n' + 'Content-Type: application/x-www-form-urlencoded\r\n' + 'User-Agent: axios/1.7.7\r\n' + 'Content-Length: 1096251\r\n' + 'Accept-Encoding: gzip, compress, deflate, br\r\n' + 'Host: clickhouse-server:8123\r\n' + 'Authorization: Basic cXJ5bjoxMjM0NTY3ODk=\r\n' + 'Connection: keep-alive\r\n' + '\r\n', _keepAliveTimeout: 0, _onPendingData: [Function: nop], agent: Agent { _events: [Object: null prototype], _eventsCount: 2, _maxListeners: undefined, defaultPort: 80, protocol: 'http:', options: [Object: null prototype], requests: [Object: null prototype] {}, sockets: [Object: null prototype] {}, freeSockets: [Object: null prototype], keepAliveMsecs: 1000, keepAlive: true, maxSockets: Infinity, maxFreeSockets: 256, scheduling: 'lifo', maxTotalSockets: Infinity, totalSocketCount: 1, [Symbol(shapeMode)]: false, [Symbol(kCapture)]: false }, socketPath: undefined, method: 'POST', maxHeaderSize: undefined, insecureHTTPParser: undefined, joinDuplicateHeaders: undefined, path: '/?database=qryn&query=INSERT%20INTO%20qryn.samples_v3(fingerprint%2C%20timestamp_ns%2C%20value%2C%20string%2C%20type)%20FORMAT%20JSONEachRow', _ended: true, res: IncomingMessage { _events: [Object], _readableState: [ReadableState], _maxListeners: undefined, socket: null, httpVersionMajor: 1, httpVersionMinor: 1, httpVersion: '1.1', complete: true, rawHeaders: [Array], rawTrailers: [], joinDuplicateHeaders: undefined, aborted: false, upgrade: false, url: '', method: null, statusCode: 500, statusMessage: 'Internal Server Error', client: [Socket], _consuming: false, _dumped: false, req: [Circular *1], _eventsCount: 4, responseUrl: 'http://qryn:123456789@clickhouse-server:8123/?database=qryn&query=INSERT%20INTO%20qryn.samples_v3(fingerprint%2C%20timestamp_ns%2C%20value%2C%20string%2C%20type)%20FORMAT%20JSONEachRow', redirects: [], [Symbol(shapeMode)]: true, [Symbol(kCapture)]: false, [Symbol(kHeaders)]: [Object], [Symbol(kHeadersCount)]: 20, [Symbol(kTrailers)]: null, [Symbol(kTrailersCount)]: 0 }, aborted: false, timeoutCb: null, upgradeOrConnect: false, parser: null, maxHeadersCount: null, reusedSocket: true, host: 'clickhouse-server', protocol: 'http:', _redirectable: Writable { _events: [Object], _writableState: [WritableState], _maxListeners: undefined, _options: [Object], _ended: true, _ending: true, _redirectCount: 0, _redirects: [], _requestBodyLength: 1096251, _requestBodyBuffers: [], _eventsCount: 3, _onNativeResponse: [Function (anonymous)], _currentRequest: [Circular *1], _currentUrl: 'http://qryn:123456789@clickhouse-server:8123/?database=qryn&query=INSERT%20INTO%20qryn.samples_v3(fingerprint%2C%20timestamp_ns%2C%20value%2C%20string%2C%20type)%20FORMAT%20JSONEachRow', [Symbol(shapeMode)]: true, [Symbol(kCapture)]: false }, [Symbol(shapeMode)]: false, [Symbol(kCapture)]: false, [Symbol(kBytesWritten)]: 0, [Symbol(kNeedDrain)]: false, [Symbol(corked)]: 0, [Symbol(kOutHeaders)]: [Object: null prototype] { accept: [Array], 'content-type': [Array], 'user-agent': [Array], 'content-length': [Array], 'accept-encoding': [Array], host: [Array], authorization: [Array] }, [Symbol(errored)]: null, [Symbol(kHighWaterMark)]: 16384, [Symbol(kRejectNonStandardBodyWrites)]: false, [Symbol(kUniqueHeaders)]: null }, response: { status: 500, statusText: 'Internal Server Error', headers: Object [AxiosHeaders] { date: 'Tue, 12 Nov 2024 11:03:03 GMT', connection: 'Keep-Alive', 'content-type': 'text/plain; charset=UTF-8', 'x-clickhouse-server-display-name': 'eeb9c057b032', 'transfer-encoding': 'chunked', 'x-clickhouse-query-id': '0d0fb35e-a8cc-470c-a65e-ef39db9f75b7', 'x-clickhouse-timezone': 'UTC', 'x-clickhouse-exception-code': '241', 'keep-alive': 'timeout=10, max=9998', 'x-clickhouse-summary': '{"read_rows":"307","read_bytes":"1009936","written_rows":"307","written_bytes":"1009936","total_rows_to_read":"0","result_rows":"0","result_bytes":"0","elapsed_ns":"8487345"}' }, config: { transitional: [Object], adapter: [Array], transformRequest: [Array], transformResponse: [Array], timeout: 0, xsrfCookieName: 'XSRF-TOKEN', xsrfHeaderName: 'X-XSRF-TOKEN', maxContentLength: -1, maxBodyLength: Infinity, env: [Object], validateStatus: [Function: validateStatus], headers: [Object [AxiosHeaders]], method: 'post', url: 'http://qryn:123456789@clickhouse-server:8123/?database=qryn&query=INSERT%20INTO%20qryn.samples_v3(fingerprint%2C%20timestamp_ns%2C%20value%2C%20string%2C%20type)%20FORMAT%20JSONEachRow', data: }, request: ClientRequest { _events: [Object: null prototype], _eventsCount: 7, _maxListeners: undefined, outputData: [], outputSize: 0, writable: true, destroyed: true, _last: false, chunkedEncoding: false, shouldKeepAlive: true, maxRequestsOnConnectionReached: false, _defaultKeepAlive: true, useChunkedEncodingByDefault: true, sendDate: false, _removedConnection: false, _removedContLen: false, _removedTE: false, strictContentLength: false, _contentLength: '1096251', _hasBody: true, _trailer: '', finished: true, _headerSent: true, _closed: true, socket: [Socket], _header: 'POST /?database=qryn&query=INSERT%20INTO%20qryn.samples_v3(fingerprint%2C%20timestamp_ns%2C%20value%2C%20string%2C%20type)%20FORMAT%20JSONEachRow HTTP/1.1\r\n' + 'Accept: application/json, text/plain, */*\r\n' + 'Content-Type: application/x-www-form-urlencoded\r\n' + 'User-Agent: axios/1.7.7\r\n' + 'Content-Length: 1096251\r\n' + 'Accept-Encoding: gzip, compress, deflate, br\r\n' + 'Host: clickhouse-server:8123\r\n' + 'Authorization: Basic cXJ5bjoxMjM0NTY3ODk=\r\n' + 'Connection: keep-alive\r\n' + '\r\n', _keepAliveTimeout: 0, _onPendingData: [Function: nop], agent: [Agent], socketPath: undefined, method: 'POST', maxHeaderSize: undefined, insecureHTTPParser: undefined, joinDuplicateHeaders: undefined, path: '/?database=qryn&query=INSERT%20INTO%20qryn.samples_v3(fingerprint%2C%20timestamp_ns%2C%20value%2C%20string%2C%20type)%20FORMAT%20JSONEachRow', _ended: true, res: [IncomingMessage], aborted: false, timeoutCb: null, upgradeOrConnect: false, parser: null, maxHeadersCount: null, reusedSocket: true, host: 'clickhouse-server', protocol: 'http:', _redirectable: [Writable], [Symbol(shapeMode)]: false, [Symbol(kCapture)]: false, [Symbol(kBytesWritten)]: 0, [Symbol(kNeedDrain)]: false, [Symbol(corked)]: 0, [Symbol(kOutHeaders)]: [Object: null prototype], [Symbol(errored)]: null, [Symbol(kHighWaterMark)]: 16384, [Symbol(kRejectNonStandardBodyWrites)]: false, [Symbol(kUniqueHeaders)]: null }, data: 'Code: 241. DB::Exception: Memory limit (total) exceeded: would use 1.12 GiB (attempt to allocate chunk of 5030083 bytes), current RSS 483.02 MiB, maximum: 921.60 MiB. OvercommitTracker decision: Query was selected to stop by OvercommitTracker. (MEMORY_LIMIT_EXCEEDED) (version 24.10.1.2812 (official build))\n' }, status: 500 } {"level":50,"time":1731409383625,"pid":19,"hostname":"c94776d321b0","name":"qryn","err":"AXIOS ERROR: AxiosError: Request failed with status code 500 Response data: Code: 241. DB::Exception: Memory limit (total) exceeded: would use 1.12 GiB (attempt to allocate chunk of 5030083 bytes), current RSS 483.02 MiB, maximum: 921.60 MiB. OvercommitTracker decision: Query was selected to stop by OvercommitTracker. (MEMORY_LIMIT_EXCEEDED) (version 24.10.1.2812 (official build))\n\nError: AXIOS ERROR: AxiosError: Request failed with status code 500 Response data: Code: 241. DB::Exception: Memory limit (total) exceeded: would use 1.12 GiB (attempt to allocate chunk of 5030083 bytes), current RSS 483.02 MiB, maximum: 921.60 MiB. OvercommitTracker decision: Query was selected to stop by OvercommitTracker. (MEMORY_LIMIT_EXCEEDED) (version 24.10.1.2812 (official build))\n\n at axiosError (/app/lib/db/throttler.js:29:14)\n at TimeoutOrSizeThrottler.flush (/app/lib/db/throttler.js:77:26)\n at process.processTicksAndRejections (node:internal/process/task_queues:95:5)\n at async /app/lib/db/throttler.js:163:13\n at async Promise.all (index 0)\n at async Timeout._onTimeout (/app/lib/db/throttler.js:160:9)","msg":"AXIOS ERROR"} {"level":30,"time":1731409384130,"pid":19,"hostname":"c94776d321b0","name":"qryn","reqId":"req-1zjl","req":{"method":"GET","url":"/loki/api/v1/labels?start=1731409084258000000&end=1731409384258000000","hostname":"127.0.0.1:3100","remoteAddress":"172.18.0.1","remotePort":59338},"msg":"incoming request"} SELECT DISTINCT key FROM time_series_gin WHERE date >= toDate(FROM_UNIXTIME(intDiv(1731409084258000000, 1000000000))) AND date <= toDate(FROM_UNIXTIME(intDiv(1731409384258000000, 1000000000))) AND type IN (0,0) FORMAT JSON {"level":30,"time":1731409384132,"pid":19,"hostname":"c94776d321b0","name":"qryn","reqId":"req-1zjm","req":{"method":"GET","url":"/loki/api/v1/query_range?direction=backward&end=1731409384274000000&query=sum+by+%28level%2C+detected_level%29+%28count_over_time%28%7Bfilename%3D%22%2Fvar%2Flog%2Fhitpaw-online%2Fcheck_hitpaw-online.log%22%2Chost%3D%22hitpaw-localweb-test%22%2Cjob%3D%22hitpaw-online-log%22%7D+%7C+drop+__error__%5B1s%5D%29%29&start=1731409084000000000&step=1000ms","hostname":"127.0.0.1:3100","remoteAddress":"172.18.0.1","remotePort":59330},"msg":"incoming request"} WITH idx_sel AS (select `sel_1`.`fingerprint` from (select `fingerprint` from `qryn`.`time_series_gin` where ((`key` = 'filename') and (`val` = '/var/log/hitpaw-online/check_hitpaw-online.log'))) as `sel_1` inner any join (select `fingerprint` from `qryn`.`time_series_gin` where ((`key` = 'host') and (`val` = 'hitpaw-localweb-test'))) as `sel_2` on `sel_1`.`fingerprint` = `sel_2`.`fingerprint` inner any join (select `fingerprint` from `qryn`.`time_series_gin` where ((`key` = 'job') and (`val` = 'hitpaw-online-log'))) as `sel_3` on `sel_1`.`fingerprint` = `sel_3`.`fingerprint`), rate_a AS (select `samples`.`string` as `string`,`samples`.`fingerprint` as `fingerprint`,intDiv(samples.timestamp_ns, 1000000) as `timestamp_ns`,arrayFilter(x -> x.1 NOT IN ('__error__'), JSONExtractKeysAndValues(time_series.labels, 'String')) as `labels` from qryn.samples_v3 as `samples` left any join (select `fingerprint`,`labels` from `qryn`.`time_series` as `time_series` where ((`time_series`.`fingerprint` in (idx_sel)) and (date >= toDate(fromUnixTimestamp(intDiv(1731409084000000000, 1000000000)))) and (date <= toDate(fromUnixTimestamp(intDiv(1731409385000000000, 1000000000)))))) AS time_series on `samples`.`fingerprint` = time_series.fingerprint where ((`samples`.`timestamp_ns` between 1731409084000000000 and 1731409385000000000) and (`samples`.`type` in (0,0))) and (samples.fingerprint IN idx_sel) order by `timestamp_ns` desc), agg_a AS (select labels as `labels`,intDiv(timestamp_ns, 1000) * 1000 as `timestamp_ns`,toFloat64(count(1)) as `value` from rate_a group by `labels`,`timestamp_ns` order by `labels` asc,`timestamp_ns` asc) select arrayFilter(x -> x.1 IN ('level','detected_level'), labels) as `labels`,`timestamp_ns`,sum(value) as `value` from agg_a group by `labels`,`timestamp_ns` order by `labels`,`timestamp_ns` {"level":30,"time":1731409384159,"pid":19,"hostname":"c94776d321b0","name":"qryn","reqId":"req-1zjn","req":{"method":"GET","url":"/loki/api/v1/query_range?direction=backward&end=1731409384274000000&limit=1000&query=%7Bfilename%3D%22%2Fvar%2Flog%2Fhitpaw-online%2Fcheck_hitpaw-online.log%22%2Chost%3D%22hitpaw-localweb-test%22%2Cjob%3D%22hitpaw-online-log%22%7D&start=1731409084274000000&step=200ms","hostname":"127.0.0.1:3100","remoteAddress":"172.18.0.1","remotePort":45648},"msg":"incoming request"} WITH idx_sel AS (select `sel_1`.`fingerprint` from (select `fingerprint` from `qryn`.`time_series_gin` where ((`key` = 'filename') and (`val` = '/var/log/hitpaw-online/check_hitpaw-online.log'))) as `sel_1` inner any join (select `fingerprint` from `qryn`.`time_series_gin` where ((`key` = 'host') and (`val` = 'hitpaw-localweb-test'))) as `sel_2` on `sel_1`.`fingerprint` = `sel_2`.`fingerprint` inner any join (select `fingerprint` from `qryn`.`time_series_gin` where ((`key` = 'job') and (`val` = 'hitpaw-online-log'))) as `sel_3` on `sel_1`.`fingerprint` = `sel_3`.`fingerprint`), sel_a AS (select `samples`.`string` as `string`,`samples`.`fingerprint` as `fingerprint`,samples.timestamp_ns as `timestamp_ns` from qryn.samples_v3 as `samples` where ((`samples`.`timestamp_ns` between 1731409084273000000 and 1731409384273000000) and (`samples`.`type` in (0,0))) and (samples.fingerprint IN idx_sel) order by `timestamp_ns` desc limit 1000) select JSONExtractKeysAndValues(time_series.labels, 'String') as `labels`,samples.* from sel_a as `samples` left any join (select `fingerprint`,`labels` from `qryn`.`time_series` as `time_series` where ((`time_series`.`fingerprint` in (idx_sel)) and (date >= toDate(fromUnixTimestamp(intDiv(1731409084273000000, 1000000000)))) and (date <= toDate(fromUnixTimestamp(intDiv(1731409384273000000, 1000000000)))))) AS time_series on `samples`.`fingerprint` = time_series.fingerprint order by `labels` desc,`timestamp_ns` desc {"level":30,"time":1731409384175,"pid":19,"hostname":"c94776d321b0","name":"qryn","reqId":"req-1zjl","res":{"statusCode":200},"responseTime":44.14421606063843,"msg":"request completed"} {"level":30,"time":1731409384271,"pid":19,"hostname":"c94776d321b0","name":"qryn","reqId":"req-1zjm","res":{"statusCode":500},"responseTime":138.80394804477692,"msg":"request completed"} {"level":50,"time":1731409384271,"pid":19,"hostname":"c94776d321b0","name":"qryn","reqId":"req-1zjm","err":"Error: Request failed with status code 500\nResponse: [500] {\"exception\": \"Code: 241. DB::Exception: Memory limit (total) exceeded: would use 1.15 GiB (attempt to allocate chunk of 0 bytes), current RSS 540.90 MiB, maximum: 921.60 MiB. OvercommitTracker decision: Query was selected to stop by OvercommitTracker.: While executing AggregatingTransform. (MEMORY_LIMIT_EXCEEDED) (version 24.10.1.2812 (official build))\"}\n{\"exception\": \"Code: 241. DB::Exception: Memory limit (total) exceeded: would use 1.15 GiB (attempt to allocate chunk of 0 bytes), current RSS 540.90 MiB, maximum: 921.60 MiB. OvercommitTracker decision: Query was selected to stop by OvercommitTracker.: While executing AggregatingTransform. (MEMORY_LIMIT_EXCEEDED) (version 24.10.1.2812 (official build))\"}\n\nAxiosError: Request failed with status code 500\n at settle (/app/node_modules/axios/dist/node/axios.cjs:2019:12)\n at RedirectableRequest.handleResponse (/app/node_modules/axios/dist/node/axios.cjs:3083:9)\n at RedirectableRequest.emit (node:events:519:28)\n at RedirectableRequest._processResponse (/app/node_modules/follow-redirects/index.js:399:10)\n at RedirectableRequest._onNativeResponse (/app/node_modules/follow-redirects/index.js:92:12)\n at Object.onceWrapper (node:events:634:26)\n at ClientRequest.emit (node:events:519:28)\n at HTTPParser.parserOnIncomingClient (node:_http_client:709:27)\n at HTTPParser.parserOnHeadersComplete (node:_http_common:119:17)\n at Socket.socketOnData (node:_http_client:551:22)\n at Axios.request (/app/node_modules/axios/dist/node/axios.cjs:4287:41)\n at process.processTicksAndRejections (node:internal/process/task_queues:95:5)\n at async queryFingerprintsScan (/app/lib/db/clickhouse.js:387:19)\n at async Object.handler (/app/lib/handlers/query_range.js:41:22)","msg":"Request failed with status code 500"} {"level":50,"time":1731409384271,"pid":19,"hostname":"c94776d321b0","name":"qryn","reqId":"req-1zjm","err":"Error: Request failed with status code 500\nResponse: [500] {\"exception\": \"Code: 241. DB::Exception: Memory limit (total) exceeded: would use 1.15 GiB (attempt to allocate chunk of 0 bytes), current RSS 540.90 MiB, maximum: 921.60 MiB. OvercommitTracker decision: Query was selected to stop by OvercommitTracker.: While executing AggregatingTransform. (MEMORY_LIMIT_EXCEEDED) (version 24.10.1.2812 (official build))\"}\n{\"exception\": \"Code: 241. DB::Exception: Memory limit (total) exceeded: would use 1.15 GiB (attempt to allocate chunk of 0 bytes), current RSS 540.90 MiB, maximum: 921.60 MiB. OvercommitTracker decision: Query was selected to stop by OvercommitTracker.: While executing AggregatingTransform. (MEMORY_LIMIT_EXCEEDED) (version 24.10.1.2812 (official build))\"}\n\nAxiosError: Request failed with status code 500\n at settle (/app/node_modules/axios/dist/node/axios.cjs:2019:12)\n at RedirectableRequest.handleResponse (/app/node_modules/axios/dist/node/axios.cjs:3083:9)\n at RedirectableRequest.emit (node:events:519:28)\n at RedirectableRequest._processResponse (/app/node_modules/follow-redirects/index.js:399:10)\n at RedirectableRequest._onNativeResponse (/app/node_modules/follow-redirects/index.js:92:12)\n at Object.onceWrapper (node:events:634:26)\n at ClientRequest.emit (node:events:519:28)\n at HTTPParser.parserOnIncomingClient (node:_http_client:709:27)\n at HTTPParser.parserOnHeadersComplete (node:_http_common:119:17)\n at Socket.socketOnData (node:_http_client:551:22)\n at Axios.request (/app/node_modules/axios/dist/node/axios.cjs:4287:41)\n at process.processTicksAndRejections (node:internal/process/task_queues:95:5)\n at async queryFingerprintsScan (/app/lib/db/clickhouse.js:387:19)\n at async Object.handler (/app/lib/handlers/query_range.js:41:22)","msg":"Request failed with status code 500"} {"level":30,"time":1731409384463,"pid":19,"hostname":"c94776d321b0","name":"qryn","reqId":"req-1zjn","res":{"statusCode":500},"responseTime":303.9276930093765,"msg":"request completed"} {"level":50,"time":1731409384464,"pid":19,"hostname":"c94776d321b0","name":"qryn","reqId":"req-1zjn","err":"Error: Request failed with status code 500\nResponse: [500] {\"exception\": \"Code: 241. DB::Exception: Memory limit (total) exceeded: would use 1.16 GiB (attempt to allocate chunk of 4210220 bytes), current RSS 541.49 MiB, maximum: 921.60 MiB. OvercommitTracker decision: Memory overcommit has freed not enough memory.: (while reading column string): (while reading from part \\/var\\/lib\\/clickhouse\\/store\\/356\\/35618169-82fe-4b8e-a56f-d46cd91df2b4\\/1731369600_188721_188721_0\\/ in table qryn.samples_v3 (35618169-82fe-4b8e-a56f-d46cd91df2b4) located on disk default of type local, from mark 0 with max_rows_to_read = 296): While executing MergeTreeSelect(pool: ReadPoolInOrder, algorithm: InReverseOrder). (MEMORY_LIMIT_EXCEEDED) (version 24.10.1.2812 (official build))\"}\n{\"exception\": \"Code: 241. DB::Exception: Memory limit (total) exceeded: would use 1.16 GiB (attempt to allocate chunk of 4210220 bytes), current RSS 541.49 MiB, maximum: 921.60 MiB. OvercommitTracker decision: Memory overcommit has freed not enough memory.: (while reading column string): (while reading from part \\/var\\/lib\\/clickhouse\\/store\\/356\\/35618169-82fe-4b8e-a56f-d46cd91df2b4\\/1731369600_188721_188721_0\\/ in table qryn.samples_v3 (35618169-82fe-4b8e-a56f-d46cd91df2b4) located on disk default of type local, from mark 0 with max_rows_to_read = 296): While executing MergeTreeSelect(pool: ReadPoolInOrder, algorithm: InReverseOrder). (MEMORY_LIMIT_EXCEEDED) (version 24.10.1.2812 (official build))\"}\n\nAxiosError: Request failed with status code 500\n at settle (/app/node_modules/axios/dist/node/axios.cjs:2019:12)\n at RedirectableRequest.handleResponse (/app/node_modules/axios/dist/node/axios.cjs:3083:9)\n at RedirectableRequest.emit (node:events:519:28)\n at RedirectableRequest._processResponse (/app/node_modules/follow-redirects/index.js:399:10)\n at RedirectableRequest._onNativeResponse (/app/node_modules/follow-redirects/index.js:92:12)\n at Object.onceWrapper (node:events:634:26)\n at ClientRequest.emit (node:events:519:28)\n at HTTPParser.parserOnIncomingClient (node:_http_client:709:27)\n at HTTPParser.parserOnHeadersComplete (node:_http_common:119:17)\n at Socket.socketOnData (node:_http_client:551:22)\n at Axios.request (/app/node_modules/axios/dist/node/axios.cjs:4287:41)\n at process.processTicksAndRejections (node:internal/process/task_queues:95:5)\n at async queryFingerprintsScan (/app/lib/db/clickhouse.js:387:19)\n at async Object.handler (/app/lib/handlers/query_range.js:41:22)","msg":"Request failed with status code 500"} {"level":50,"time":1731409384464,"pid":19,"hostname":"c94776d321b0","name":"qryn","reqId":"req-1zjn","err":"Error: Request failed with status code 500\nResponse: [500] {\"exception\": \"Code: 241. DB::Exception: Memory limit (total) exceeded: would use 1.16 GiB (attempt to allocate chunk of 4210220 bytes), current RSS 541.49 MiB, maximum: 921.60 MiB. OvercommitTracker decision: Memory overcommit has freed not enough memory.: (while reading column string): (while reading from part \\/var\\/lib\\/clickhouse\\/store\\/356\\/35618169-82fe-4b8e-a56f-d46cd91df2b4\\/1731369600_188721_188721_0\\/ in table qryn.samples_v3 (35618169-82fe-4b8e-a56f-d46cd91df2b4) located on disk default of type local, from mark 0 with max_rows_to_read = 296): While executing MergeTreeSelect(pool: ReadPoolInOrder, algorithm: InReverseOrder). (MEMORY_LIMIT_EXCEEDED) (version 24.10.1.2812 (official build))\"}\n{\"exception\": \"Code: 241. DB::Exception: Memory limit (total) exceeded: would use 1.16 GiB (attempt to allocate chunk of 4210220 bytes), current RSS 541.49 MiB, maximum: 921.60 MiB. OvercommitTracker decision: Memory overcommit has freed not enough memory.: (while reading column string): (while reading from part \\/var\\/lib\\/clickhouse\\/store\\/356\\/35618169-82fe-4b8e-a56f-d46cd91df2b4\\/1731369600_188721_188721_0\\/ in table qryn.samples_v3 (35618169-82fe-4b8e-a56f-d46cd91df2b4) located on disk default of type local, from mark 0 with max_rows_to_read = 296): While executing MergeTreeSelect(pool: ReadPoolInOrder, algorithm: InReverseOrder). (MEMORY_LIMIT_EXCEEDED) (version 24.10.1.2812 (official build))\"}\n\nAxiosError: Request failed with status code 500\n at settle (/app/node_modules/axios/dist/node/axios.cjs:2019:12)\n at RedirectableRequest.handleResponse (/app/node_modules/axios/dist/node/axios.cjs:3083:9)\n at RedirectableRequest.emit (node:events:519:28)\n at RedirectableRequest._processResponse (/app/node_modules/follow-redirects/index.js:399:10)\n at RedirectableRequest._onNativeResponse (/app/node_modules/follow-redirects/index.js:92:12)\n at Object.onceWrapper (node:events:634:26)\n at ClientRequest.emit (node:events:519:28)\n at HTTPParser.parserOnIncomingClient (node:_http_client:709:27)\n at HTTPParser.parserOnHeadersComplete (node:_http_common:119:17)\n at Socket.socketOnData (node:_http_client:551:22)\n at Axios.request (/app/node_modules/axios/dist/node/axios.cjs:4287:41)\n at process.processTicksAndRejections (node:internal/process/task_queues:95:5)\n at async queryFingerprintsScan (/app/lib/db/clickhouse.js:387:19)\n at async Object.handler (/app/lib/handlers/query_range.js:41:22)","msg":"Request failed with status code 500"}