Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

(fix) Custom Event filtering #164

Merged
merged 4 commits into from
Jul 3, 2023
Merged

(fix) Custom Event filtering #164

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
8b448db
fixed custom events filter functionality
Blaumaus Jul 3, 2023
9d86d3d
fixed captcha query
Blaumaus Jul 3, 2023
ede1294
fixed custom events filter functionality for selfhosted
Blaumaus Jul 3, 2023
7666703
captcha clean-up
Blaumaus Jul 3, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 7 additions & 17 deletions apps/production/src/analytics/analytics.controller.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -290,7 +290,7 @@ export class AnalyticsController {
}

this.analyticsService.validateTimebucket(timeBucket)
const [filtersQuery, filtersParams, appliedFilters] =
const [filtersQuery, filtersParams, appliedFilters, customEVFilterApplied] =
this.analyticsService.getFiltersQuery(
filters,
isCaptcha ? DataType.CAPTCHA : DataType.ANALYTICS,
Expand All @@ -315,17 +315,11 @@ export class AnalyticsController {
let subQuery = `FROM ${
isCaptcha ? 'captcha' : 'analytics'
} WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`
let customEVFilterApplied = false

if (filtersParams?.ev && !isCaptcha) {
customEVFilterApplied = true
queryCustoms = `SELECT ev, count() FROM customEV WHERE ${
filtersParams.ev_exclusive ? 'NOT' : ''
} ev = {ev:String} AND pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY ev`
if (customEVFilterApplied && !isCaptcha) {
queryCustoms = `SELECT ev, count() FROM customEV WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY ev`

subQuery = `FROM customEV WHERE ${
filtersParams.ev_exclusive ? 'NOT' : ''
} ev = {ev:String} AND pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`
subQuery = `FROM customEV WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`
}

const paramsData = {
Expand Down Expand Up @@ -424,7 +418,7 @@ export class AnalyticsController {
}

this.analyticsService.validateTimebucket(timeBucket)
const [filtersQuery, filtersParams, appliedFilters] =
const [filtersQuery, filtersParams, appliedFilters, customEVFilterApplied] =
this.analyticsService.getFiltersQuery(filters, DataType.ANALYTICS)

const safeTimezone = this.analyticsService.getSafeTimezone(timezone)
Expand All @@ -442,13 +436,9 @@ export class AnalyticsController {
)

let subQuery = `FROM analytics WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`
let customEVFilterApplied = false

if (filtersParams?.ev) {
customEVFilterApplied = true
subQuery = `FROM customEV WHERE ${
filtersParams.ev_exclusive ? 'NOT' : ''
} ev = {ev:String} AND pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`
if (customEVFilterApplied) {
subQuery = `FROM customEV WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`
}

const paramsData = {
Expand Down
25 changes: 15 additions & 10 deletions apps/production/src/analytics/analytics.service.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -202,7 +202,7 @@ const generateParamsQuery = (
}

if (isCaptcha) {
return `SELECT ${col}, count(*) as count ${subQuery} AND ${col} IS NOT NULL GROUP BY ${col}`
return `SELECT ${columnsQuery}, count(*) as count ${subQuery} AND ${col} IS NOT NULL GROUP BY ${col}`
}

if (customEVFilterApplied) {
Expand Down Expand Up @@ -656,20 +656,21 @@ export class AnalyticsService {
const params = {}
let parsed = []
let query = ''
let customEVFilterApplied = false

if (_isEmpty(filters)) {
return [query, params, parsed]
return [query, params, parsed, customEVFilterApplied]
}

try {
parsed = JSON.parse(filters)
} catch (e) {
console.error(`Cannot parse the filters array: ${filters}`)
return [query, params, parsed]
return [query, params, parsed, customEVFilterApplied]
}

if (_isEmpty(parsed)) {
return [query, params, parsed]
return [query, params, parsed, customEVFilterApplied]
}

if (!_isArray(parsed)) {
Expand All @@ -688,7 +689,9 @@ export class AnalyticsService {
(prev, curr) => {
const { column, filter, isExclusive = false } = curr

if (!_includes(SUPPORTED_COLUMNS, column)) {
if (column === 'ev') {
customEVFilterApplied = true
} else if (!_includes(SUPPORTED_COLUMNS, column)) {
throw new UnprocessableEntityException(
`The provided filter (${column}) is not supported`,
)
Expand Down Expand Up @@ -738,7 +741,12 @@ export class AnalyticsService {
query += ')'
}

return [query, params, this.postProcessParsedFilters(parsed)]
return [
query,
params,
this.postProcessParsedFilters(parsed),
customEVFilterApplied,
]
}

validateTimebucket(tb: TimeBucketType): void {
Expand Down Expand Up @@ -1162,7 +1170,6 @@ export class AnalyticsService {
generateCustomEventsAggregationQuery(
timeBucket: TimeBucketType,
filtersQuery: string,
paramsData: any,
safeTimezone: string,
): string {
const timeBucketFunc = timeBucketConversion[timeBucket]
Expand All @@ -1178,8 +1185,7 @@ export class AnalyticsService {
SELECT *,
${timeBucketFunc}(toTimeZone(created, '${safeTimezone}')) as tz_created
FROM customEV
WHERE ${paramsData.params.ev_exclusive ? 'NOT' : ''} ev = {ev:String}
AND pid = {pid:FixedString(12)}
WHERE pid = {pid:FixedString(12)}
AND created BETWEEN ${tzFromDate} AND ${tzToDate}
${filtersQuery}
) as subquery
Expand Down Expand Up @@ -1348,7 +1354,6 @@ export class AnalyticsService {
const query = this.generateCustomEventsAggregationQuery(
timeBucket,
filtersQuery,
paramsData,
safeTimezone,
)

Expand Down
4 changes: 3 additions & 1 deletion apps/production/src/analytics/interfaces/index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,13 +52,15 @@ export interface IGetGroupFromTo {
groupToUTC: string
}

export interface GetFiltersQuery extends Array<string | object> {
export interface GetFiltersQuery extends Array<string | object | boolean> {
// SQL query
0: string
// an object that has structure like { cf_pg: '/signup', ev_exclusive: false }
1: { [key: string]: string | boolean }
// an array of objects like [{ "column":"pg", "filter":"/signup", "isExclusive":true }]
2: Array<{ [key: string]: string }> | []
// flag that indicates if there is an 'ev' filter for custom events
3: boolean
}

export interface IUserFlowNode {
Expand Down
110 changes: 22 additions & 88 deletions apps/selfhosted/src/analytics/analytics.controller.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -255,7 +255,6 @@ export class AnalyticsController {
async getData(
@Query() data: AnalyticsGET_DTO,
@CurrentUserId() uid: string,
isCaptcha = false,
): Promise<any> {
const {
pid,
Expand All @@ -273,11 +272,8 @@ export class AnalyticsController {
}

this.analyticsService.validateTimebucket(timeBucket)
const [filtersQuery, filtersParams, parsedFilters] =
this.analyticsService.getFiltersQuery(
filters,
isCaptcha ? DataType.CAPTCHA : DataType.ANALYTICS,
)
const [filtersQuery, filtersParams, parsedFilters, customEVFilterApplied] =
this.analyticsService.getFiltersQuery(filters, DataType.ANALYTICS)

const safeTimezone = this.analyticsService.getSafeTimezone(timezone)
const { groupFrom, groupTo, groupFromUTC, groupToUTC } =
Expand All @@ -291,20 +287,12 @@ export class AnalyticsController {
await this.analyticsService.checkProjectAccess(pid, uid)

let queryCustoms = `SELECT ev, count() FROM customEV WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY ev`
let subQuery = `FROM ${
isCaptcha ? 'captcha' : 'analytics'
} WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`
let customEVFilterApplied = false

if (filtersParams?.ev && !isCaptcha) {
customEVFilterApplied = true
queryCustoms = `SELECT ev, count() FROM customEV WHERE ${
filtersParams.ev_exclusive ? 'NOT' : ''
} ev = {ev:String} AND pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY ev`

subQuery = `FROM customEV WHERE ${
filtersParams.ev_exclusive ? 'NOT' : ''
} ev = {ev:String} AND pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`
let subQuery = `FROM analytics WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`

if (customEVFilterApplied) {
queryCustoms = `SELECT ev, count() FROM customEV WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY ev`

subQuery = `FROM customEV WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`
}

const paramsData = {
Expand All @@ -316,38 +304,17 @@ export class AnalyticsController {
},
}

let result: object | void

if (isCaptcha) {
result = await this.analyticsService.groupCaptchaByTimeBucket(
timeBucket,
groupFrom,
groupTo,
subQuery,
filtersQuery,
paramsData,
safeTimezone,
)
} else {
result = await this.analyticsService.groupByTimeBucket(
timeBucket,
groupFrom,
groupTo,
subQuery,
filtersQuery,
paramsData,
safeTimezone,
customEVFilterApplied,
parsedFilters,
)
}

if (isCaptcha) {
return {
...result,
appliedFilters: parsedFilters,
}
}
const result = await this.analyticsService.groupByTimeBucket(
timeBucket,
groupFrom,
groupTo,
subQuery,
filtersQuery,
paramsData,
safeTimezone,
customEVFilterApplied,
parsedFilters,
)

const customs = await this.analyticsService.processCustomEV(
queryCustoms,
Expand Down Expand Up @@ -383,7 +350,7 @@ export class AnalyticsController {
}

this.analyticsService.validateTimebucket(timeBucket)
const [filtersQuery, filtersParams, parsedFilters] =
const [filtersQuery, filtersParams, parsedFilters, customEVFilterApplied] =
this.analyticsService.getFiltersQuery(filters, DataType.ANALYTICS)

const safeTimezone = this.analyticsService.getSafeTimezone(timezone)
Expand All @@ -397,13 +364,9 @@ export class AnalyticsController {
await this.analyticsService.checkProjectAccess(pid, uid)

let subQuery = `FROM analytics WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`
let customEVFilterApplied = false

if (filtersParams?.ev) {
customEVFilterApplied = true
subQuery = `FROM customEV WHERE ${
filtersParams.ev_exclusive ? 'NOT' : ''
} ev = {ev:String} AND pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`
if (customEVFilterApplied) {
subQuery = `FROM customEV WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`
}

const paramsData = {
Expand Down Expand Up @@ -553,15 +516,6 @@ export class AnalyticsController {
}
}

@Get('captcha')
@Auth([], true, true)
async getCaptchaData(
@Query() data: AnalyticsGET_DTO,
@CurrentUserId() uid: string,
): Promise<any> {
return this.getData(data, uid, true)
}

@Get('user-flow')
@Auth([], true, true)
async getUserFlow(
Expand Down Expand Up @@ -624,26 +578,6 @@ export class AnalyticsController {
return this.analyticsService.getSummary(pidsArray, 'w')
}

@Get('captcha/birdseye')
@Auth([], true, true)
// returns overall short statistics per CAPTCHA project
async getCaptchaOverallStats(
@Query() data,
@CurrentUserId() uid: string,
): Promise<any> {
const { pids, pid } = data
const pidsArray = getPIDsArray(pids, pid)

const validationPromises = _map(pidsArray, async currentPID => {
this.analyticsService.validatePID(currentPID)
await this.analyticsService.checkProjectAccess(currentPID, uid)
})

await Promise.all(validationPromises)

return this.analyticsService.getCaptchaSummary(pidsArray, 'w')
}

@Get('hb')
async getHeartBeatStats(
@Query() data,
Expand Down
Loading