Merge pull request #164 from Swetrix/fix/custom-ev-filter

(fix) Custom Event filtering
Swetrix · Jul 3, 2023 · cfd2e58 · cfd2e58
2 parents a18a2b7 + 7666703
commit cfd2e58
Show file tree

Hide file tree

Showing 6 changed files with 87 additions and 275 deletions.
diff --git a/apps/production/src/analytics/analytics.controller.ts b/apps/production/src/analytics/analytics.controller.ts
@@ -290,7 +290,7 @@ export class AnalyticsController {
     }
 
     this.analyticsService.validateTimebucket(timeBucket)
-    const [filtersQuery, filtersParams, appliedFilters] =
+    const [filtersQuery, filtersParams, appliedFilters, customEVFilterApplied] =
       this.analyticsService.getFiltersQuery(
         filters,
         isCaptcha ? DataType.CAPTCHA : DataType.ANALYTICS,
@@ -315,17 +315,11 @@ export class AnalyticsController {
     let subQuery = `FROM ${
       isCaptcha ? 'captcha' : 'analytics'
     } WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`
-    let customEVFilterApplied = false
 
-    if (filtersParams?.ev && !isCaptcha) {
-      customEVFilterApplied = true
-      queryCustoms = `SELECT ev, count() FROM customEV WHERE ${
-        filtersParams.ev_exclusive ? 'NOT' : ''
-      } ev = {ev:String} AND pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY ev`
+    if (customEVFilterApplied && !isCaptcha) {
+      queryCustoms = `SELECT ev, count() FROM customEV WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY ev`
 
-      subQuery = `FROM customEV WHERE ${
-        filtersParams.ev_exclusive ? 'NOT' : ''
-      } ev = {ev:String} AND pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`
+      subQuery = `FROM customEV WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`
     }
 
     const paramsData = {
@@ -424,7 +418,7 @@ export class AnalyticsController {
     }
 
     this.analyticsService.validateTimebucket(timeBucket)
-    const [filtersQuery, filtersParams, appliedFilters] =
+    const [filtersQuery, filtersParams, appliedFilters, customEVFilterApplied] =
       this.analyticsService.getFiltersQuery(filters, DataType.ANALYTICS)
 
     const safeTimezone = this.analyticsService.getSafeTimezone(timezone)
@@ -442,13 +436,9 @@ export class AnalyticsController {
     )
 
     let subQuery = `FROM analytics WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`
-    let customEVFilterApplied = false
 
-    if (filtersParams?.ev) {
-      customEVFilterApplied = true
-      subQuery = `FROM customEV WHERE ${
-        filtersParams.ev_exclusive ? 'NOT' : ''
-      } ev = {ev:String} AND pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`
+    if (customEVFilterApplied) {
+      subQuery = `FROM customEV WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`
     }
 
     const paramsData = {

diff --git a/apps/production/src/analytics/analytics.service.ts b/apps/production/src/analytics/analytics.service.ts
@@ -202,7 +202,7 @@ const generateParamsQuery = (
   }
 
   if (isCaptcha) {
-    return `SELECT ${col}, count(*) as count ${subQuery} AND ${col} IS NOT NULL GROUP BY ${col}`
+    return `SELECT ${columnsQuery}, count(*) as count ${subQuery} AND ${col} IS NOT NULL GROUP BY ${col}`
   }
 
   if (customEVFilterApplied) {
@@ -656,20 +656,21 @@ export class AnalyticsService {
     const params = {}
     let parsed = []
     let query = ''
+    let customEVFilterApplied = false
 
     if (_isEmpty(filters)) {
-      return [query, params, parsed]
+      return [query, params, parsed, customEVFilterApplied]
     }
 
     try {
       parsed = JSON.parse(filters)
     } catch (e) {
       console.error(`Cannot parse the filters array: ${filters}`)
-      return [query, params, parsed]
+      return [query, params, parsed, customEVFilterApplied]
     }
 
     if (_isEmpty(parsed)) {
-      return [query, params, parsed]
+      return [query, params, parsed, customEVFilterApplied]
     }
 
     if (!_isArray(parsed)) {
@@ -688,7 +689,9 @@ export class AnalyticsService {
       (prev, curr) => {
         const { column, filter, isExclusive = false } = curr
 
-        if (!_includes(SUPPORTED_COLUMNS, column)) {
+        if (column === 'ev') {
+          customEVFilterApplied = true
+        } else if (!_includes(SUPPORTED_COLUMNS, column)) {
           throw new UnprocessableEntityException(
             `The provided filter (${column}) is not supported`,
           )
@@ -738,7 +741,12 @@ export class AnalyticsService {
       query += ')'
     }
 
-    return [query, params, this.postProcessParsedFilters(parsed)]
+    return [
+      query,
+      params,
+      this.postProcessParsedFilters(parsed),
+      customEVFilterApplied,
+    ]
   }
 
   validateTimebucket(tb: TimeBucketType): void {
@@ -1162,7 +1170,6 @@ export class AnalyticsService {
   generateCustomEventsAggregationQuery(
     timeBucket: TimeBucketType,
     filtersQuery: string,
-    paramsData: any,
     safeTimezone: string,
   ): string {
     const timeBucketFunc = timeBucketConversion[timeBucket]
@@ -1178,8 +1185,7 @@ export class AnalyticsService {
         SELECT *,
         ${timeBucketFunc}(toTimeZone(created, '${safeTimezone}')) as tz_created
         FROM customEV
-        WHERE ${paramsData.params.ev_exclusive ? 'NOT' : ''} ev = {ev:String}
-          AND pid = {pid:FixedString(12)}
+        WHERE pid = {pid:FixedString(12)}
           AND created BETWEEN ${tzFromDate} AND ${tzToDate}
           ${filtersQuery}
       ) as subquery
@@ -1348,7 +1354,6 @@ export class AnalyticsService {
       const query = this.generateCustomEventsAggregationQuery(
         timeBucket,
         filtersQuery,
-        paramsData,
         safeTimezone,
       )
 

diff --git a/apps/production/src/analytics/interfaces/index.ts b/apps/production/src/analytics/interfaces/index.ts
@@ -52,13 +52,15 @@ export interface IGetGroupFromTo {
   groupToUTC: string
 }
 
-export interface GetFiltersQuery extends Array<string | object> {
+export interface GetFiltersQuery extends Array<string | object | boolean> {
   // SQL query
   0: string
   // an object that has structure like { cf_pg: '/signup', ev_exclusive: false }
   1: { [key: string]: string | boolean }
   // an array of objects like [{ "column":"pg", "filter":"/signup", "isExclusive":true }]
   2: Array<{ [key: string]: string }> | []
+  // flag that indicates if there is an 'ev' filter for custom events
+  3: boolean
 }
 
 export interface IUserFlowNode {

diff --git a/apps/selfhosted/src/analytics/analytics.controller.ts b/apps/selfhosted/src/analytics/analytics.controller.ts
@@ -255,7 +255,6 @@ export class AnalyticsController {
   async getData(
     @Query() data: AnalyticsGET_DTO,
     @CurrentUserId() uid: string,
-    isCaptcha = false,
   ): Promise<any> {
     const {
       pid,
@@ -273,11 +272,8 @@ export class AnalyticsController {
     }
 
     this.analyticsService.validateTimebucket(timeBucket)
-    const [filtersQuery, filtersParams, parsedFilters] =
-      this.analyticsService.getFiltersQuery(
-        filters,
-        isCaptcha ? DataType.CAPTCHA : DataType.ANALYTICS,
-      )
+    const [filtersQuery, filtersParams, parsedFilters, customEVFilterApplied] =
+      this.analyticsService.getFiltersQuery(filters, DataType.ANALYTICS)
 
     const safeTimezone = this.analyticsService.getSafeTimezone(timezone)
     const { groupFrom, groupTo, groupFromUTC, groupToUTC } =
@@ -291,20 +287,12 @@ export class AnalyticsController {
     await this.analyticsService.checkProjectAccess(pid, uid)
 
     let queryCustoms = `SELECT ev, count() FROM customEV WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY ev`
-    let subQuery = `FROM ${
-      isCaptcha ? 'captcha' : 'analytics'
-    } WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`
-    let customEVFilterApplied = false
-
-    if (filtersParams?.ev && !isCaptcha) {
-      customEVFilterApplied = true
-      queryCustoms = `SELECT ev, count() FROM customEV WHERE ${
-        filtersParams.ev_exclusive ? 'NOT' : ''
-      } ev = {ev:String} AND pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY ev`
-
-      subQuery = `FROM customEV WHERE ${
-        filtersParams.ev_exclusive ? 'NOT' : ''
-      } ev = {ev:String} AND pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`
+    let subQuery = `FROM analytics WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`
+
+    if (customEVFilterApplied) {
+      queryCustoms = `SELECT ev, count() FROM customEV WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY ev`
+
+      subQuery = `FROM customEV WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`
     }
 
     const paramsData = {
@@ -316,38 +304,17 @@ export class AnalyticsController {
       },
     }
 
-    let result: object | void
-
-    if (isCaptcha) {
-      result = await this.analyticsService.groupCaptchaByTimeBucket(
-        timeBucket,
-        groupFrom,
-        groupTo,
-        subQuery,
-        filtersQuery,
-        paramsData,
-        safeTimezone,
-      )
-    } else {
-      result = await this.analyticsService.groupByTimeBucket(
-        timeBucket,
-        groupFrom,
-        groupTo,
-        subQuery,
-        filtersQuery,
-        paramsData,
-        safeTimezone,
-        customEVFilterApplied,
-        parsedFilters,
-      )
-    }
-
-    if (isCaptcha) {
-      return {
-        ...result,
-        appliedFilters: parsedFilters,
-      }
-    }
+    const result = await this.analyticsService.groupByTimeBucket(
+      timeBucket,
+      groupFrom,
+      groupTo,
+      subQuery,
+      filtersQuery,
+      paramsData,
+      safeTimezone,
+      customEVFilterApplied,
+      parsedFilters,
+    )
 
     const customs = await this.analyticsService.processCustomEV(
       queryCustoms,
@@ -383,7 +350,7 @@ export class AnalyticsController {
     }
 
     this.analyticsService.validateTimebucket(timeBucket)
-    const [filtersQuery, filtersParams, parsedFilters] =
+    const [filtersQuery, filtersParams, parsedFilters, customEVFilterApplied] =
       this.analyticsService.getFiltersQuery(filters, DataType.ANALYTICS)
 
     const safeTimezone = this.analyticsService.getSafeTimezone(timezone)
@@ -397,13 +364,9 @@ export class AnalyticsController {
     await this.analyticsService.checkProjectAccess(pid, uid)
 
     let subQuery = `FROM analytics WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`
-    let customEVFilterApplied = false
 
-    if (filtersParams?.ev) {
-      customEVFilterApplied = true
-      subQuery = `FROM customEV WHERE ${
-        filtersParams.ev_exclusive ? 'NOT' : ''
-      } ev = {ev:String} AND pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`
+    if (customEVFilterApplied) {
+      subQuery = `FROM customEV WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`
     }
 
     const paramsData = {
@@ -553,15 +516,6 @@ export class AnalyticsController {
     }
   }
 
-  @Get('captcha')
-  @Auth([], true, true)
-  async getCaptchaData(
-    @Query() data: AnalyticsGET_DTO,
-    @CurrentUserId() uid: string,
-  ): Promise<any> {
-    return this.getData(data, uid, true)
-  }
-
   @Get('user-flow')
   @Auth([], true, true)
   async getUserFlow(
@@ -624,26 +578,6 @@ export class AnalyticsController {
     return this.analyticsService.getSummary(pidsArray, 'w')
   }
 
-  @Get('captcha/birdseye')
-  @Auth([], true, true)
-  // returns overall short statistics per CAPTCHA project
-  async getCaptchaOverallStats(
-    @Query() data,
-    @CurrentUserId() uid: string,
-  ): Promise<any> {
-    const { pids, pid } = data
-    const pidsArray = getPIDsArray(pids, pid)
-
-    const validationPromises = _map(pidsArray, async currentPID => {
-      this.analyticsService.validatePID(currentPID)
-      await this.analyticsService.checkProjectAccess(currentPID, uid)
-    })
-
-    await Promise.all(validationPromises)
-
-    return this.analyticsService.getCaptchaSummary(pidsArray, 'w')
-  }
-
   @Get('hb')
   async getHeartBeatStats(
     @Query() data,