Skip to content

Commit

Permalink
Merge pull request #172 from kruzhambus/feature/new-period
Browse files Browse the repository at this point in the history
  • Loading branch information
@kruzhambus
kruzhambus authored Jul 20, 2023
2 parents 9d83f76 + 338e4c0 commit 46fd189
Show file tree
Hide file tree
Showing 3 changed files with 156 additions and 11 deletions.
96 changes: 86 additions & 10 deletions apps/production/src/analytics/analytics.controller.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import * as _isEmpty from 'lodash/isEmpty'
import * as _isArray from 'lodash/isArray'
import * as _toNumber from 'lodash/toNumber'
import * as _pick from 'lodash/pick'
import * as _includes from 'lodash/includes'
import * as _map from 'lodash/map'
import * as _uniqBy from 'lodash/uniqBy'
import * as _round from 'lodash/round'
Expand Down Expand Up @@ -35,6 +36,7 @@ import {
getSessionKey,
getHeartbeatKey,
DataType,
validPeriods,
} from './analytics.service'
import { TaskManagerService } from '../task-manager/task-manager.service'
import { CurrentUserId } from '../auth/decorators/current-user-id.decorator'
Expand Down Expand Up @@ -289,7 +291,26 @@ export class AnalyticsController {
this.analyticsService.validatePeriod(period)
}

this.analyticsService.validateTimebucket(timeBucket)
let newTimebucket = timeBucket
let allowedTumebucketForPeriodAll

let diff

if (period === 'all') {
const res = await this.analyticsService.getTimeBucketForAllTime(
pid,
period,
)

diff = res.diff
// eslint-disable-next-line prefer-destructuring
newTimebucket = _includes(res.timeBucket, timeBucket)
? timeBucket
: res.timeBucket[0]
allowedTumebucketForPeriodAll = res.timeBucket
}

this.analyticsService.validateTimebucket(newTimebucket)
const [filtersQuery, filtersParams, appliedFilters, customEVFilterApplied] =
this.analyticsService.getFiltersQuery(
filters,
Expand All @@ -301,9 +322,10 @@ export class AnalyticsController {
this.analyticsService.getGroupFromTo(
from,
to,
timeBucket,
newTimebucket,
period,
safeTimezone,
diff,
)
await this.analyticsService.checkProjectAccess(
pid,
Expand Down Expand Up @@ -335,7 +357,7 @@ export class AnalyticsController {

if (isCaptcha) {
result = await this.analyticsService.groupCaptchaByTimeBucket(
timeBucket,
newTimebucket,
groupFrom,
groupTo,
subQuery,
Expand All @@ -345,7 +367,7 @@ export class AnalyticsController {
)
} else {
result = await this.analyticsService.groupByTimeBucket(
timeBucket,
newTimebucket,
groupFrom,
groupTo,
subQuery,
Expand All @@ -361,6 +383,7 @@ export class AnalyticsController {
return {
...result,
appliedFilters,
timeBucket: allowedTumebucketForPeriodAll,
}
}

Expand All @@ -373,6 +396,7 @@ export class AnalyticsController {
...result,
customs,
appliedFilters,
timeBucket: allowedTumebucketForPeriodAll,
}
}

Expand Down Expand Up @@ -489,17 +513,36 @@ export class AnalyticsController {
this.analyticsService.validatePeriod(period)
}

this.analyticsService.validateTimebucket(timeBucket)
let newTimeBucket = timeBucket
let allowedTumebucketForPeriodAll
let diff

if (period === 'all') {
const res = await this.analyticsService.getTimeBucketForAllTime(
pid,
period,
)

diff = res.diff
// eslint-disable-next-line prefer-destructuring
newTimeBucket = _includes(res.timeBucket, timeBucket)
? timeBucket
: res.timeBucket[0]
allowedTumebucketForPeriodAll = res.timeBucket
}

this.analyticsService.validateTimebucket(newTimeBucket)
const [filtersQuery, filtersParams, appliedFilters] =
this.analyticsService.getFiltersQuery(filters, DataType.PERFORMANCE)

const safeTimezone = this.analyticsService.getSafeTimezone(timezone)
const { groupFrom, groupTo } = this.analyticsService.getGroupFromTo(
from,
to,
timeBucket,
newTimeBucket,
period,
safeTimezone,
diff,
)
await this.analyticsService.checkProjectAccess(
pid,
Expand All @@ -519,7 +562,7 @@ export class AnalyticsController {
}

const result = await this.analyticsService.groupPerfByTimeBucket(
timeBucket,
newTimeBucket,
groupFrom,
groupTo,
subQuery,
Expand All @@ -531,6 +574,7 @@ export class AnalyticsController {
return {
...result,
appliedFilters,
timeBucket: allowedTumebucketForPeriodAll,
}
}

Expand Down Expand Up @@ -621,6 +665,17 @@ export class AnalyticsController {
this.analyticsService.validatePeriod(period)
}

let diff

if (period === 'all') {
const res = await this.analyticsService.getTimeBucketForAllTime(
pid,
period,
)

diff = res.diff
}

await this.analyticsService.checkProjectAccess(
pid,
uid,
Expand All @@ -634,6 +689,7 @@ export class AnalyticsController {
null,
period,
safeTimezone,
diff,
)

const [filtersQuery, filtersParams, appliedFilters] =
Expand Down Expand Up @@ -1123,7 +1179,25 @@ export class AnalyticsController {
this.analyticsService.validatePeriod(period)
}

this.analyticsService.validateTimebucket(timeBucket)
let newTimeBucket = timeBucket
let diff
let timeBucketForAllTime

if (period === validPeriods[validPeriods.length - 1]) {
const res = await this.analyticsService.getTimeBucketForAllTime(
pid,
period,
)

// eslint-disable-next-line prefer-destructuring
newTimeBucket = _includes(res.timeBucket, timeBucket)
? timeBucket
: res.timeBucket[0]
diff = res.diff
timeBucketForAllTime = res.timeBucket
}

this.analyticsService.validateTimebucket(newTimeBucket)
const [filtersQuery, filtersParams, appliedFilters] =
this.analyticsService.getFiltersQuery(filters, DataType.ANALYTICS)
await this.analyticsService.checkProjectAccess(
Expand All @@ -1136,9 +1210,10 @@ export class AnalyticsController {
const { groupFrom, groupTo } = this.analyticsService.getGroupFromTo(
from,
to,
timeBucket,
newTimeBucket,
period,
safeTimezone,
diff,
)

const paramsData = {
Expand All @@ -1151,7 +1226,7 @@ export class AnalyticsController {
}

const result: any = await this.analyticsService.groupCustomEVByTimeBucket(
timeBucket,
newTimeBucket,
groupFrom,
groupTo,
filtersQuery,
Expand Down Expand Up @@ -1181,6 +1256,7 @@ export class AnalyticsController {
return {
...result,
appliedFilters,
timeBucket: timeBucketForAllTime,
}
}
}
70 changes: 69 additions & 1 deletion apps/production/src/analytics/analytics.service.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,8 @@ const GMT_0_TIMEZONES = [
// 'Africa/Casablanca',
]

const validPeriods = [
export const validPeriods = [
'thishour',
'today',
'yesterday',
'1d',
Expand All @@ -101,12 +102,14 @@ const validPeriods = [
'3M',
'12M',
'24M',
'all',
]

const validTimebuckets = [
TimeBucketType.HOUR,
TimeBucketType.DAY,
TimeBucketType.MONTH,
TimeBucketType.YEAR,
]

// mapping of allowed timebuckets per difference between days
Expand All @@ -122,6 +125,8 @@ const timeBucketToDays = [
}, // 4 weeks
{ lt: 366, tb: [TimeBucketType.MONTH] }, // 12 months
{ lt: 732, tb: [TimeBucketType.MONTH] }, // 24 months
{ lt: 1464, tb: [TimeBucketType.MONTH, TimeBucketType.YEAR] }, // 48 months
{ lt: 99999, tb: [TimeBucketType.YEAR] },
]

// Smaller than 64 characters, must start with an English letter and contain only letters (a-z A-Z), numbers (0-9), underscores (_) and dots (.)
Expand All @@ -132,6 +137,7 @@ const timeBucketConversion = {
hour: 'toStartOfHour',
day: 'toStartOfDay',
month: 'toStartOfMonth',
year: 'toStartOfYear',
}

const isValidTimezone = (timezone: string): boolean => {
Expand Down Expand Up @@ -407,6 +413,7 @@ export class AnalyticsService {
timeBucket: TimeBucketType | null,
period: string,
safeTimezone: string,
diff?: number,
): IGetGroupFromTo {
let groupFrom: dayjs.Dayjs
let groupTo: dayjs.Dayjs
Expand Down Expand Up @@ -473,9 +480,15 @@ export class AnalyticsService {
if (period === 'today') {
groupFrom = djsNow.startOf('d')
groupTo = djsNow
} else if (period === 'thishour') {
groupFrom = djsNow.subtract(1, 'hour').startOf('h')
groupTo = djsNow
} else if (period === 'yesterday') {
groupFrom = djsNow.subtract(1, 'day').startOf('d')
groupTo = djsNow.subtract(1, 'day').endOf('d')
} else if (period === 'all' && (diff === 0 || diff === 1)) {
groupFrom = djsNow.subtract(1, 'day').startOf('d')
groupTo = djsNow
} else {
if (period === '1d') {
groupFrom = djsNow.subtract(parseInt(period, 10), _last(period))
Expand Down Expand Up @@ -630,6 +643,53 @@ export class AnalyticsService {
}
}

async getTimeBucketForAllTime(
pid: string,
period: string,
): Promise<{
timeBucket: TimeBucketType[]
diff: number
}> {
if (period !== 'all') {
return null
}

const from: any = await clickhouse
.query(
'SELECT created as from FROM analytics where pid = {pid:FixedString(12)} ORDER BY created ASC LIMIT 1',
{ params: { pid } },
)
.toPromise()
const to: any = await clickhouse
.query(
'SELECT created as to FROM analytics where pid = {pid:FixedString(12)} ORDER BY created DESC LIMIT 1',
{ params: { pid } },
)
.toPromise()

let newTimeBucket = [TimeBucketType.MONTH]
let diff = null

if (from && to) {
diff = dayjs(to[0].to).diff(dayjs(from[0].from), 'days')

const tbMap = _find(timeBucketToDays, ({ lt }) => diff <= lt)

if (_isEmpty(tbMap)) {
throw new PreconditionFailedException(
"The difference between 'from' and 'to' is greater than allowed",
)
}

newTimeBucket = tbMap.tb
}

return {
timeBucket: newTimeBucket,
diff,
}
}

postProcessParsedFilters(parsedFilters: any[]): any[] {
return _reduce(
parsedFilters,
Expand Down Expand Up @@ -1002,6 +1062,10 @@ export class AnalyticsService {
format = 'YYYY-MM'
break

case TimeBucketType.YEAR:
format = 'YYYY'
break

default:
throw new BadRequestException(
`The provided time bucket (${timeBucket}) is incorrect`,
Expand Down Expand Up @@ -1129,6 +1193,10 @@ export class AnalyticsService {
]
}

if (timeBucket === TimeBucketType.YEAR) {
return [`toYear(tz_created) as year`, 'year']
}

return [
`toYear(tz_created) as year,
toMonth(tz_created) as month,
Expand Down
1 change: 1 addition & 0 deletions apps/production/src/analytics/dto/getData.dto.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ export enum TimeBucketType {
HOUR = 'hour',
DAY = 'day',
MONTH = 'month',
YEAR = 'year',
}

// eslint-disable-next-line @typescript-eslint/naming-convention
Expand Down

0 comments on commit 46fd189

Please sign in to comment.