From 8b448dbd0c9488545732560a83a22e472137143f Mon Sep 17 00:00:00 2001 From: Blue Mouse Date: Mon, 3 Jul 2023 19:20:42 +0100 Subject: [PATCH 1/4] fixed custom events filter functionality --- .../src/analytics/analytics.controller.ts | 24 ++++++------------- .../src/analytics/analytics.service.ts | 23 +++++++++++------- .../src/analytics/interfaces/index.ts | 4 +++- 3 files changed, 24 insertions(+), 27 deletions(-) diff --git a/apps/production/src/analytics/analytics.controller.ts b/apps/production/src/analytics/analytics.controller.ts index 07aeaa9fd..ba5048eb6 100644 --- a/apps/production/src/analytics/analytics.controller.ts +++ b/apps/production/src/analytics/analytics.controller.ts @@ -290,7 +290,7 @@ export class AnalyticsController { } this.analyticsService.validateTimebucket(timeBucket) - const [filtersQuery, filtersParams, appliedFilters] = + const [filtersQuery, filtersParams, appliedFilters, customEVFilterApplied] = this.analyticsService.getFiltersQuery( filters, isCaptcha ? DataType.CAPTCHA : DataType.ANALYTICS, @@ -315,17 +315,11 @@ export class AnalyticsController { let subQuery = `FROM ${ isCaptcha ? 'captcha' : 'analytics' } WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}` - let customEVFilterApplied = false - if (filtersParams?.ev && !isCaptcha) { - customEVFilterApplied = true - queryCustoms = `SELECT ev, count() FROM customEV WHERE ${ - filtersParams.ev_exclusive ? 'NOT' : '' - } ev = {ev:String} AND pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY ev` + if (customEVFilterApplied && !isCaptcha) { + queryCustoms = `SELECT ev, count() FROM customEV WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY ev` - subQuery = `FROM customEV WHERE ${ - filtersParams.ev_exclusive ? 'NOT' : '' - } ev = {ev:String} AND pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}` + subQuery = `FROM customEV WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}` } const paramsData = { @@ -424,7 +418,7 @@ export class AnalyticsController { } this.analyticsService.validateTimebucket(timeBucket) - const [filtersQuery, filtersParams, appliedFilters] = + const [filtersQuery, filtersParams, appliedFilters, customEVFilterApplied] = this.analyticsService.getFiltersQuery(filters, DataType.ANALYTICS) const safeTimezone = this.analyticsService.getSafeTimezone(timezone) @@ -442,13 +436,9 @@ export class AnalyticsController { ) let subQuery = `FROM analytics WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}` - let customEVFilterApplied = false - if (filtersParams?.ev) { - customEVFilterApplied = true - subQuery = `FROM customEV WHERE ${ - filtersParams.ev_exclusive ? 'NOT' : '' - } ev = {ev:String} AND pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}` + if (customEVFilterApplied) { + subQuery = `FROM customEV WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}` } const paramsData = { diff --git a/apps/production/src/analytics/analytics.service.ts b/apps/production/src/analytics/analytics.service.ts index fc4f3cfc5..156b03f2d 100644 --- a/apps/production/src/analytics/analytics.service.ts +++ b/apps/production/src/analytics/analytics.service.ts @@ -656,20 +656,21 @@ export class AnalyticsService { const params = {} let parsed = [] let query = '' + let customEVFilterApplied = false if (_isEmpty(filters)) { - return [query, params, parsed] + return [query, params, parsed, customEVFilterApplied] } try { parsed = JSON.parse(filters) } catch (e) { console.error(`Cannot parse the filters array: ${filters}`) - return [query, params, parsed] + return [query, params, parsed, customEVFilterApplied] } if (_isEmpty(parsed)) { - return [query, params, parsed] + return [query, params, parsed, customEVFilterApplied] } if (!_isArray(parsed)) { @@ -688,7 +689,9 @@ export class AnalyticsService { (prev, curr) => { const { column, filter, isExclusive = false } = curr - if (!_includes(SUPPORTED_COLUMNS, column)) { + if (column === 'ev') { + customEVFilterApplied = true + } else if (!_includes(SUPPORTED_COLUMNS, column)) { throw new UnprocessableEntityException( `The provided filter (${column}) is not supported`, ) @@ -738,7 +741,12 @@ export class AnalyticsService { query += ')' } - return [query, params, this.postProcessParsedFilters(parsed)] + return [ + query, + params, + this.postProcessParsedFilters(parsed), + customEVFilterApplied, + ] } validateTimebucket(tb: TimeBucketType): void { @@ -1162,7 +1170,6 @@ export class AnalyticsService { generateCustomEventsAggregationQuery( timeBucket: TimeBucketType, filtersQuery: string, - paramsData: any, safeTimezone: string, ): string { const timeBucketFunc = timeBucketConversion[timeBucket] @@ -1178,8 +1185,7 @@ export class AnalyticsService { SELECT *, ${timeBucketFunc}(toTimeZone(created, '${safeTimezone}')) as tz_created FROM customEV - WHERE ${paramsData.params.ev_exclusive ? 'NOT' : ''} ev = {ev:String} - AND pid = {pid:FixedString(12)} + WHERE pid = {pid:FixedString(12)} AND created BETWEEN ${tzFromDate} AND ${tzToDate} ${filtersQuery} ) as subquery @@ -1348,7 +1354,6 @@ export class AnalyticsService { const query = this.generateCustomEventsAggregationQuery( timeBucket, filtersQuery, - paramsData, safeTimezone, ) diff --git a/apps/production/src/analytics/interfaces/index.ts b/apps/production/src/analytics/interfaces/index.ts index 0844bcaf6..fda538f0c 100644 --- a/apps/production/src/analytics/interfaces/index.ts +++ b/apps/production/src/analytics/interfaces/index.ts @@ -52,13 +52,15 @@ export interface IGetGroupFromTo { groupToUTC: string } -export interface GetFiltersQuery extends Array { +export interface GetFiltersQuery extends Array { // SQL query 0: string // an object that has structure like { cf_pg: '/signup', ev_exclusive: false } 1: { [key: string]: string | boolean } // an array of objects like [{ "column":"pg", "filter":"/signup", "isExclusive":true }] 2: Array<{ [key: string]: string }> | [] + // flag that indicates if there is an 'ev' filter for custom events + 3: boolean } export interface IUserFlowNode { From 9d86d3d6801e37e0b72a8285c63d633a78be3dc0 Mon Sep 17 00:00:00 2001 From: Blue Mouse Date: Mon, 3 Jul 2023 19:24:29 +0100 Subject: [PATCH 2/4] fixed captcha query --- apps/production/src/analytics/analytics.service.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/production/src/analytics/analytics.service.ts b/apps/production/src/analytics/analytics.service.ts index 156b03f2d..155072f6d 100644 --- a/apps/production/src/analytics/analytics.service.ts +++ b/apps/production/src/analytics/analytics.service.ts @@ -202,7 +202,7 @@ const generateParamsQuery = ( } if (isCaptcha) { - return `SELECT ${col}, count(*) as count ${subQuery} AND ${col} IS NOT NULL GROUP BY ${col}` + return `SELECT ${columnsQuery}, count(*) as count ${subQuery} AND ${col} IS NOT NULL GROUP BY ${col}` } if (customEVFilterApplied) { From ede129468b73361934b01ab11ebcf47ac664b8b0 Mon Sep 17 00:00:00 2001 From: Blue Mouse Date: Mon, 3 Jul 2023 19:31:02 +0100 Subject: [PATCH 3/4] fixed custom events filter functionality for selfhosted --- .../src/analytics/analytics.controller.ts | 24 ++++++------------- .../src/analytics/analytics.service.ts | 23 +++++++++++------- .../src/analytics/interfaces/index.ts | 4 +++- 3 files changed, 24 insertions(+), 27 deletions(-) diff --git a/apps/selfhosted/src/analytics/analytics.controller.ts b/apps/selfhosted/src/analytics/analytics.controller.ts index 5220adddf..6013acb74 100644 --- a/apps/selfhosted/src/analytics/analytics.controller.ts +++ b/apps/selfhosted/src/analytics/analytics.controller.ts @@ -273,7 +273,7 @@ export class AnalyticsController { } this.analyticsService.validateTimebucket(timeBucket) - const [filtersQuery, filtersParams, parsedFilters] = + const [filtersQuery, filtersParams, parsedFilters, customEVFilterApplied] = this.analyticsService.getFiltersQuery( filters, isCaptcha ? DataType.CAPTCHA : DataType.ANALYTICS, @@ -294,17 +294,11 @@ export class AnalyticsController { let subQuery = `FROM ${ isCaptcha ? 'captcha' : 'analytics' } WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}` - let customEVFilterApplied = false - if (filtersParams?.ev && !isCaptcha) { - customEVFilterApplied = true - queryCustoms = `SELECT ev, count() FROM customEV WHERE ${ - filtersParams.ev_exclusive ? 'NOT' : '' - } ev = {ev:String} AND pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY ev` + if (customEVFilterApplied && !isCaptcha) { + queryCustoms = `SELECT ev, count() FROM customEV WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY ev` - subQuery = `FROM customEV WHERE ${ - filtersParams.ev_exclusive ? 'NOT' : '' - } ev = {ev:String} AND pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}` + subQuery = `FROM customEV WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}` } const paramsData = { @@ -383,7 +377,7 @@ export class AnalyticsController { } this.analyticsService.validateTimebucket(timeBucket) - const [filtersQuery, filtersParams, parsedFilters] = + const [filtersQuery, filtersParams, parsedFilters, customEVFilterApplied] = this.analyticsService.getFiltersQuery(filters, DataType.ANALYTICS) const safeTimezone = this.analyticsService.getSafeTimezone(timezone) @@ -397,13 +391,9 @@ export class AnalyticsController { await this.analyticsService.checkProjectAccess(pid, uid) let subQuery = `FROM analytics WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}` - let customEVFilterApplied = false - if (filtersParams?.ev) { - customEVFilterApplied = true - subQuery = `FROM customEV WHERE ${ - filtersParams.ev_exclusive ? 'NOT' : '' - } ev = {ev:String} AND pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}` + if (customEVFilterApplied) { + subQuery = `FROM customEV WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}` } const paramsData = { diff --git a/apps/selfhosted/src/analytics/analytics.service.ts b/apps/selfhosted/src/analytics/analytics.service.ts index ed197a905..e6edfc6d4 100644 --- a/apps/selfhosted/src/analytics/analytics.service.ts +++ b/apps/selfhosted/src/analytics/analytics.service.ts @@ -615,20 +615,21 @@ export class AnalyticsService { const params = {} let parsed = [] let query = '' + let customEVFilterApplied = false if (_isEmpty(filters)) { - return [query, params, parsed] + return [query, params, parsed, customEVFilterApplied] } try { parsed = JSON.parse(filters) } catch (e) { console.error(`Cannot parse the filters array: ${filters}`) - return [query, params, parsed] + return [query, params, parsed, customEVFilterApplied] } if (_isEmpty(parsed)) { - return [query, params, parsed] + return [query, params, parsed, customEVFilterApplied] } if (!_isArray(parsed)) { @@ -647,7 +648,9 @@ export class AnalyticsService { (prev, curr) => { const { column, filter, isExclusive = false } = curr - if (!_includes(SUPPORTED_COLUMNS, column)) { + if (column === 'ev') { + customEVFilterApplied = true + } else if (!_includes(SUPPORTED_COLUMNS, column)) { throw new UnprocessableEntityException( `The provided filter (${column}) is not supported`, ) @@ -697,7 +700,12 @@ export class AnalyticsService { query += ')' } - return [query, params, this.postProcessParsedFilters(parsed)] + return [ + query, + params, + this.postProcessParsedFilters(parsed), + customEVFilterApplied, + ] } validateTimebucket(tb: TimeBucketType): void { @@ -1096,7 +1104,6 @@ export class AnalyticsService { generateCustomEventsAggregationQuery( timeBucket: TimeBucketType, filtersQuery: string, - paramsData: any, safeTimezone: string, ): string { const timeBucketFunc = timeBucketConversion[timeBucket] @@ -1112,8 +1119,7 @@ export class AnalyticsService { SELECT *, ${timeBucketFunc}(toTimeZone(created, '${safeTimezone}')) as tz_created FROM customEV - WHERE ${paramsData.params.ev_exclusive ? 'NOT' : ''} ev = {ev:String} - AND pid = {pid:FixedString(12)} + WHERE pid = {pid:FixedString(12)} AND created BETWEEN ${tzFromDate} AND ${tzToDate} ${filtersQuery} ) as subquery @@ -1282,7 +1288,6 @@ export class AnalyticsService { const query = this.generateCustomEventsAggregationQuery( timeBucket, filtersQuery, - paramsData, safeTimezone, ) diff --git a/apps/selfhosted/src/analytics/interfaces/index.ts b/apps/selfhosted/src/analytics/interfaces/index.ts index 0844bcaf6..fda538f0c 100644 --- a/apps/selfhosted/src/analytics/interfaces/index.ts +++ b/apps/selfhosted/src/analytics/interfaces/index.ts @@ -52,13 +52,15 @@ export interface IGetGroupFromTo { groupToUTC: string } -export interface GetFiltersQuery extends Array { +export interface GetFiltersQuery extends Array { // SQL query 0: string // an object that has structure like { cf_pg: '/signup', ev_exclusive: false } 1: { [key: string]: string | boolean } // an array of objects like [{ "column":"pg", "filter":"/signup", "isExclusive":true }] 2: Array<{ [key: string]: string }> | [] + // flag that indicates if there is an 'ev' filter for custom events + 3: boolean } export interface IUserFlowNode { From 7666703a8378ef844136ae5a7b7d8ffcb2c84838 Mon Sep 17 00:00:00 2001 From: Blue Mouse Date: Mon, 3 Jul 2023 19:36:59 +0100 Subject: [PATCH 4/4] captcha clean-up --- .../src/analytics/analytics.controller.ts | 84 ++------- .../src/analytics/analytics.service.ts | 172 +++--------------- 2 files changed, 37 insertions(+), 219 deletions(-) diff --git a/apps/selfhosted/src/analytics/analytics.controller.ts b/apps/selfhosted/src/analytics/analytics.controller.ts index 6013acb74..1d2d8798f 100644 --- a/apps/selfhosted/src/analytics/analytics.controller.ts +++ b/apps/selfhosted/src/analytics/analytics.controller.ts @@ -255,7 +255,6 @@ export class AnalyticsController { async getData( @Query() data: AnalyticsGET_DTO, @CurrentUserId() uid: string, - isCaptcha = false, ): Promise { const { pid, @@ -274,10 +273,7 @@ export class AnalyticsController { this.analyticsService.validateTimebucket(timeBucket) const [filtersQuery, filtersParams, parsedFilters, customEVFilterApplied] = - this.analyticsService.getFiltersQuery( - filters, - isCaptcha ? DataType.CAPTCHA : DataType.ANALYTICS, - ) + this.analyticsService.getFiltersQuery(filters, DataType.ANALYTICS) const safeTimezone = this.analyticsService.getSafeTimezone(timezone) const { groupFrom, groupTo, groupFromUTC, groupToUTC } = @@ -291,11 +287,9 @@ export class AnalyticsController { await this.analyticsService.checkProjectAccess(pid, uid) let queryCustoms = `SELECT ev, count() FROM customEV WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY ev` - let subQuery = `FROM ${ - isCaptcha ? 'captcha' : 'analytics' - } WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}` + let subQuery = `FROM analytics WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}` - if (customEVFilterApplied && !isCaptcha) { + if (customEVFilterApplied) { queryCustoms = `SELECT ev, count() FROM customEV WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY ev` subQuery = `FROM customEV WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}` @@ -310,38 +304,17 @@ export class AnalyticsController { }, } - let result: object | void - - if (isCaptcha) { - result = await this.analyticsService.groupCaptchaByTimeBucket( - timeBucket, - groupFrom, - groupTo, - subQuery, - filtersQuery, - paramsData, - safeTimezone, - ) - } else { - result = await this.analyticsService.groupByTimeBucket( - timeBucket, - groupFrom, - groupTo, - subQuery, - filtersQuery, - paramsData, - safeTimezone, - customEVFilterApplied, - parsedFilters, - ) - } - - if (isCaptcha) { - return { - ...result, - appliedFilters: parsedFilters, - } - } + const result = await this.analyticsService.groupByTimeBucket( + timeBucket, + groupFrom, + groupTo, + subQuery, + filtersQuery, + paramsData, + safeTimezone, + customEVFilterApplied, + parsedFilters, + ) const customs = await this.analyticsService.processCustomEV( queryCustoms, @@ -543,15 +516,6 @@ export class AnalyticsController { } } - @Get('captcha') - @Auth([], true, true) - async getCaptchaData( - @Query() data: AnalyticsGET_DTO, - @CurrentUserId() uid: string, - ): Promise { - return this.getData(data, uid, true) - } - @Get('user-flow') @Auth([], true, true) async getUserFlow( @@ -614,26 +578,6 @@ export class AnalyticsController { return this.analyticsService.getSummary(pidsArray, 'w') } - @Get('captcha/birdseye') - @Auth([], true, true) - // returns overall short statistics per CAPTCHA project - async getCaptchaOverallStats( - @Query() data, - @CurrentUserId() uid: string, - ): Promise { - const { pids, pid } = data - const pidsArray = getPIDsArray(pids, pid) - - const validationPromises = _map(pidsArray, async currentPID => { - this.analyticsService.validatePID(currentPID) - await this.analyticsService.checkProjectAccess(currentPID, uid) - }) - - await Promise.all(validationPromises) - - return this.analyticsService.getCaptchaSummary(pidsArray, 'w') - } - @Get('hb') async getHeartBeatStats( @Query() data, diff --git a/apps/selfhosted/src/analytics/analytics.service.ts b/apps/selfhosted/src/analytics/analytics.service.ts index e6edfc6d4..d5cc74225 100644 --- a/apps/selfhosted/src/analytics/analytics.service.ts +++ b/apps/selfhosted/src/analytics/analytics.service.ts @@ -84,7 +84,6 @@ const GMT_0_TIMEZONES = [ const cols = ['cc', 'pg', 'lc', 'br', 'os', 'dv', 'ref', 'so', 'me', 'ca'] -const captchaColumns = ['cc', 'br', 'os', 'dv'] const perfColumns = ['cc', 'pg', 'dv', 'br'] const validPeriods = [ @@ -179,32 +178,36 @@ const generateParamsQuery = ( subQuery: string, customEVFilterApplied: boolean, isPageInclusiveFilterSet: boolean, - isCaptcha?: boolean, isPerformance?: boolean, ): string => { - if (isPerformance) { - return `SELECT ${col}, avg(pageLoad) ${subQuery} AND ${col} IS NOT NULL GROUP BY ${col}` + let columns = [`${col} as name`] + + // For regions and cities we'll return an array of objects, that will also include the country code + // We need the conutry code to display the flag next to the region/city name + if (col === 'rg' || col === 'ct') { + columns = [...columns, 'cc'] } - if (isCaptcha) { - return `SELECT ${col}, count(*) ${subQuery} AND ${col} IS NOT NULL GROUP BY ${col}` + const columnsQuery = columns.join(', ') + + if (isPerformance) { + return `SELECT ${columnsQuery}, round(divide(avg(pageLoad), 1000), 2) as count ${subQuery} AND ${col} IS NOT NULL GROUP BY ${columnsQuery}` } if (customEVFilterApplied) { - return `SELECT ${col}, count(*) ${subQuery} AND ${col} IS NOT NULL GROUP BY ${col}` + return `SELECT ${columnsQuery}, count(*) as count ${subQuery} AND ${col} IS NOT NULL GROUP BY ${columnsQuery}` } if (col === 'pg' || isPageInclusiveFilterSet) { - return `SELECT ${col}, count(*) ${subQuery} AND ${col} IS NOT NULL GROUP BY ${col}` + return `SELECT ${columnsQuery}, count(*) as count ${subQuery} AND ${col} IS NOT NULL GROUP BY ${columnsQuery}` } - return `SELECT ${col}, count(*) ${subQuery} AND ${col} IS NOT NULL AND unique='1' GROUP BY ${col}` + return `SELECT ${columnsQuery}, count(*) as count ${subQuery} AND ${col} IS NOT NULL AND unique='1' GROUP BY ${columnsQuery}` } export enum DataType { ANALYTICS = 'analytics', PERFORMANCE = 'performance', - CAPTCHA = 'captcha', } const isValidOrigin = (origins: string[], origin: string) => { @@ -353,11 +356,7 @@ export class AnalyticsService { return cols } - if (dataType === DataType.PERFORMANCE) { - return perfColumns - } - - return captchaColumns + return perfColumns } getGroupFromTo( @@ -730,13 +729,9 @@ export class AnalyticsService { return this.getSummaryStats(pids, 'analytics', period, amountToSubtract) } - async getCaptchaSummary(pids: string[], period: 'w' | 'M' = 'w') { - return this.getSummaryStats(pids, 'captcha', period) - } - async getSummaryStats( pids: string[], - tableName: 'analytics' | 'captcha', + tableName: 'analytics', period: 'w' | 'M' = 'w', amountToSubtract = 1, ) { @@ -834,28 +829,22 @@ export class AnalyticsService { subQuery: string, customEVFilterApplied: boolean, paramsData: any, - isCaptcha: boolean, isPerformance: boolean, ): Promise { const params = {} // We need this to display all the pageview related data (e.g. country, browser) when user applies an inclusive filter on the Page column - const isPageInclusiveFilterSet = - isCaptcha || isPerformance - ? false - : !_isEmpty( - _find( - parsedFilters, - filter => filter.column === 'pg' && !filter.isExclusive, - ), - ) + const isPageInclusiveFilterSet = isPerformance + ? false + : !_isEmpty( + _find( + parsedFilters, + filter => filter.column === 'pg' && !filter.isExclusive, + ), + ) let columns = cols - if (isCaptcha) { - columns = captchaColumns - } - if (isPerformance) { columns = perfColumns } @@ -866,7 +855,6 @@ export class AnalyticsService { subQuery, customEVFilterApplied, isPageInclusiveFilterSet, - isCaptcha, isPerformance, ) const res = await clickhouse.query(query, paramsData).toPromise() @@ -1162,34 +1150,6 @@ export class AnalyticsService { ` } - generateCaptchaAggregationQuery( - timeBucket: TimeBucketType, - filtersQuery: string, - safeTimezone: string, - ): string { - const timeBucketFunc = timeBucketConversion[timeBucket] - const [selector, groupBy] = this.getGroupSubquery(timeBucket) - const tzFromDate = `toTimeZone(parseDateTimeBestEffort({groupFrom:String}), '${safeTimezone}')` - const tzToDate = `toTimeZone(parseDateTimeBestEffort({groupTo:String}), '${safeTimezone}')` - - return ` - SELECT - ${selector}, - count() as count - FROM ( - SELECT *, - ${timeBucketFunc}(toTimeZone(created, '${safeTimezone}')) as tz_created - FROM captcha - WHERE - pid = {pid:FixedString(12)} - AND created BETWEEN ${tzFromDate} AND ${tzToDate} - ${filtersQuery} - ) as subquery - GROUP BY ${groupBy} - ORDER BY ${groupBy} - ` - } - async groupByTimeBucket( timeBucket: TimeBucketType, from: string, @@ -1264,7 +1224,6 @@ export class AnalyticsService { customEVFilterApplied, paramsData, false, - false, ) } @@ -1334,90 +1293,6 @@ export class AnalyticsService { }) } - extractCaptchaChartData(result, x: string[]): any { - const count = Array(x.length).fill(0) - - for (let row = 0; row < _size(result); ++row) { - const dateString = this.generateDateString(result[row]) - - const index = x.indexOf(dateString) - - if (index !== -1) { - count[index] = result[row].count - } - } - - return { - count, - } - } - - async groupCaptchaByTimeBucket( - timeBucket: TimeBucketType, - from: string, - to: string, - subQuery: string, - filtersQuery: string, - paramsData: object, - safeTimezone: string, - ): Promise { - let params: unknown = {} - let chart: unknown = {} - - const promises = [ - // Getting params - (async () => { - params = await this.generateParams( - null, - subQuery, - false, - paramsData, - true, - false, - ) - - if (!_some(_values(params), val => !_isEmpty(val))) { - throw new BadRequestException( - 'The are no parameters for the specified time frames', - ) - } - })(), - - // Getting CAPTCHA chart data - (async () => { - const { xShifted } = this.generateXAxis( - timeBucket, - from, - to, - safeTimezone, - ) - - const query = this.generateCaptchaAggregationQuery( - timeBucket, - filtersQuery, - safeTimezone, - ) - - const result = >( - await clickhouse.query(query, paramsData).toPromise() - ) - const { count } = this.extractCaptchaChartData(result, xShifted) - - chart = { - x: xShifted, - results: count, - } - })(), - ] - - await Promise.all(promises) - - return Promise.resolve({ - params, - chart, - }) - } - extractPerformanceChartData(result, x: string[]): any { const dns = Array(x.length).fill(0) const tls = Array(x.length).fill(0) @@ -1500,7 +1375,6 @@ export class AnalyticsService { subQuery, false, paramsData, - false, true, )