Merge pull request #161 from Swetrix/improvement/filters

(improvement) Advanced filters + standardise filters API resonse
Swetrix · Jun 27, 2023 · a18a2b7 · a18a2b7
2 parents b9e9fb2 + 12044b3
commit a18a2b7
Show file tree

Hide file tree

Showing 2 changed files with 158 additions and 68 deletions.
diff --git a/apps/production/src/analytics/analytics.service.ts b/apps/production/src/analytics/analytics.service.ts
@@ -6,6 +6,8 @@ import * as _map from 'lodash/map'
 import * as _toUpper from 'lodash/toUpper'
 import * as _join from 'lodash/join'
 import * as _isArray from 'lodash/isArray'
+import * as _reduce from 'lodash/reduce'
+import * as _keys from 'lodash/keys'
 import * as _last from 'lodash/last'
 import * as _some from 'lodash/some'
 import * as _find from 'lodash/find'
@@ -628,11 +630,32 @@ export class AnalyticsService {
     }
   }
 
+  postProcessParsedFilters(parsedFilters: any[]): any[] {
+    return _reduce(
+      parsedFilters,
+      (prev, curr) => {
+        const { column, filter, isExclusive } = curr
+
+        if (_isArray(filter)) {
+          const filterArray = _map(filter, f => ({
+            column,
+            filter: f,
+            isExclusive,
+          }))
+          return [...prev, ...filterArray]
+        }
+
+        return [...prev, curr]
+      },
+      [],
+    )
+  }
+
   // returns SQL filters query in a format like 'AND col=value AND ...'
   getFiltersQuery(filters: string, dataType: DataType): GetFiltersQuery {
+    const params = {}
     let parsed = []
     let query = ''
-    let params = {}
 
     if (_isEmpty(filters)) {
       return [query, params, parsed]
@@ -649,64 +672,73 @@ export class AnalyticsService {
       return [query, params, parsed]
     }
 
-    const columns = this.getDataTypeColumns(dataType)
+    if (!_isArray(parsed)) {
+      throw new UnprocessableEntityException(
+        'The provided filters are not in a valid format',
+      )
+    }
+
+    const SUPPORTED_COLUMNS = this.getDataTypeColumns(dataType)
 
-    for (let i = 0; i < _size(parsed); ++i) {
-      const { column, filter, isExclusive } = parsed[i]
+    // Converting something like [{"column":"cc","filter":"BG", "isExclusive":false},{"column":"cc","filter":"PL", "isExclusive":false},{"column":"pg","filter":"/hello", "isExclusive":false}]
+    // to
+    // {cc: [{filter: 'BG', isExclusive: false}, {filter: 'PL', isExclusive: false}], pg: [{filter: '/hello', isExclusive: false}]}
+    const converted = _reduce(
+      parsed,
+      (prev, curr) => {
+        const { column, filter, isExclusive = false } = curr
 
-      // Currently Custom events do not support multiple filters at once
-      if (column === 'ev') {
-        params = {
-          ...params,
-          [column]: filter,
-          [`${column}_exclusive`]: isExclusive,
+        if (!_includes(SUPPORTED_COLUMNS, column)) {
+          throw new UnprocessableEntityException(
+            `The provided filter (${column}) is not supported`,
+          )
         }
 
-        continue
-      }
+        const res = []
 
-      if (!_includes(columns, column)) {
-        throw new UnprocessableEntityException(
-          `The provided filter (${column}) is not supported`,
-        )
-      }
+        if (_isArray(filter)) {
+          for (const f of filter) {
+            res.push({ filter: f, isExclusive })
+          }
+        } else {
+          res.push({ filter, isExclusive })
+        }
+
+        if (prev[column]) {
+          prev[column].push(...res)
+        } else {
+          prev[column] = res
+        }
 
-      // TODO: Currently for multiple filters, NOT will be applied to all of them instead of each one individually
-      // Implement filtering with an ability to apply exclusivity to each filter (some might be exclusive, some not)
-      const selector = isExclusive ? 'AND NOT' : 'AND'
+        return prev
+      },
+      {},
+    )
 
-      if (_isArray(filter)) {
-        query += ` ${selector} (`
+    const columns = _keys(converted)
 
-        // TODO: Should we limit the number of filters?
-        for (let filterIndex = 0; filterIndex < _size(filter); ++filterIndex) {
-          if (filterIndex > 0) {
-            query += ' OR '
-          }
+    for (let col = 0; col < _size(columns); ++col) {
+      const column = columns[col]
+      query += ' AND ('
 
-          const key = `cf_${column}_${filterIndex}`
-          query += `${column}={${key}:String}`
-          params = {
-            ...params,
-            [key]: filter[filterIndex],
-          }
+      for (let f = 0; f < _size(converted[column]); ++f) {
+        if (f > 0) {
+          query += ' OR '
         }
 
-        query += ')'
-        continue
-      }
+        const { filter, isExclusive } = converted[column][f]
+
+        const param = `qf_${col}_${f}`
 
-      // working only on 1 filter per 1 column
-      const colFilter = `cf_${column}`
+        query += `${isExclusive ? 'NOT ' : ''}${column} = {${param}:String}`
 
-      params = {
-        ...params,
-        [colFilter]: filter,
+        params[param] = filter
       }
-      query += ` ${selector} ${column}={${colFilter}:String}`
+
+      query += ')'
     }
 
-    return [query, params, parsed]
+    return [query, params, this.postProcessParsedFilters(parsed)]
   }
 
   validateTimebucket(tb: TimeBucketType): void {

diff --git a/apps/selfhosted/src/analytics/analytics.service.ts b/apps/selfhosted/src/analytics/analytics.service.ts
@@ -8,6 +8,9 @@ import * as _join from 'lodash/join'
 import * as _last from 'lodash/last'
 import * as _some from 'lodash/some'
 import * as _find from 'lodash/find'
+import * as _isArray from 'lodash/isArray'
+import * as _reduce from 'lodash/reduce'
+import * as _keys from 'lodash/keys'
 import * as _now from 'lodash/now'
 import * as _values from 'lodash/values'
 import * as _round from 'lodash/round'
@@ -586,11 +589,32 @@ export class AnalyticsService {
     }
   }
 
+  postProcessParsedFilters(parsedFilters: any[]): any[] {
+    return _reduce(
+      parsedFilters,
+      (prev, curr) => {
+        const { column, filter, isExclusive } = curr
+
+        if (_isArray(filter)) {
+          const filterArray = _map(filter, f => ({
+            column,
+            filter: f,
+            isExclusive,
+          }))
+          return [...prev, ...filterArray]
+        }
+
+        return [...prev, curr]
+      },
+      [],
+    )
+  }
+
   // returns SQL filters query in a format like 'AND col=value AND ...'
   getFiltersQuery(filters: string, dataType: DataType): GetFiltersQuery {
+    const params = {}
     let parsed = []
     let query = ''
-    let params = {}
 
     if (_isEmpty(filters)) {
       return [query, params, parsed]
@@ -607,39 +631,73 @@ export class AnalyticsService {
       return [query, params, parsed]
     }
 
-    const columns = this.getDataTypeColumns(dataType)
+    if (!_isArray(parsed)) {
+      throw new UnprocessableEntityException(
+        'The provided filters are not in a valid format',
+      )
+    }
 
-    for (let i = 0; i < _size(parsed); ++i) {
-      const { column, filter, isExclusive } = parsed[i]
+    const SUPPORTED_COLUMNS = this.getDataTypeColumns(dataType)
 
-      if (column === 'ev') {
-        params = {
-          ...params,
-          [column]: filter,
-          [`${column}_exclusive`]: isExclusive,
+    // Converting something like [{"column":"cc","filter":"BG", "isExclusive":false},{"column":"cc","filter":"PL", "isExclusive":false},{"column":"pg","filter":"/hello", "isExclusive":false}]
+    // to
+    // {cc: [{filter: 'BG', isExclusive: false}, {filter: 'PL', isExclusive: false}], pg: [{filter: '/hello', isExclusive: false}]}
+    const converted = _reduce(
+      parsed,
+      (prev, curr) => {
+        const { column, filter, isExclusive = false } = curr
+
+        if (!_includes(SUPPORTED_COLUMNS, column)) {
+          throw new UnprocessableEntityException(
+            `The provided filter (${column}) is not supported`,
+          )
         }
 
-        continue
-      }
+        const res = []
 
-      if (!_includes(columns, column)) {
-        throw new UnprocessableEntityException(
-          `The provided filter (${column}) is not supported`,
-        )
-      }
-      // working only on 1 filter per 1 column
-      const colFilter = `cf_${column}`
+        if (_isArray(filter)) {
+          for (const f of filter) {
+            res.push({ filter: f, isExclusive })
+          }
+        } else {
+          res.push({ filter, isExclusive })
+        }
+
+        if (prev[column]) {
+          prev[column].push(...res)
+        } else {
+          prev[column] = res
+        }
+
+        return prev
+      },
+      {},
+    )
 
-      params = {
-        ...params,
-        [colFilter]: filter,
+    const columns = _keys(converted)
+
+    for (let col = 0; col < _size(columns); ++col) {
+      const column = columns[col]
+      query += ' AND ('
+
+      for (let f = 0; f < _size(converted[column]); ++f) {
+        if (f > 0) {
+          query += ' OR '
+        }
+
+        const { filter, isExclusive } = converted[column][f]
+
+        const param = `qf_${col}_${f}`
+
+        query += `${isExclusive ? 'NOT ' : ''}${column} = {${param}:String}`
+
+        params[param] = filter
       }
-      query += ` ${
-        isExclusive ? 'AND NOT' : 'AND'
-      } ${column}={${colFilter}:String}`
+
+      query += ')'
     }
 
-    return [query, params, parsed]
+    return [query, params, this.postProcessParsedFilters(parsed)]
   }
 
   validateTimebucket(tb: TimeBucketType): void {