Feat/pln metrics (#60)

* create nlp stats entity repository and impl

* dailyTotalProcessingTime and timeByPipelineStage

* create filter dto to performance metrics

* feat: pln metrics finish integration with mongodb

src/app.module.ts

@@ -7,6 +7,7 @@ import { AuthModule } from './modules/auth/auth.module';
 import { AuthGuard } from './modules/auth/auth.guard';
 import { CommentsModule } from './modules/comments/comments.module';
 import { DashboardModule } from './modules/dashboard/dashboard.module';
+import { PerformanceMetricsModule } from './modules/performance-metrics/performance-metrics.module';
 
 @Module({
 	imports: [
@@ -15,6 +16,7 @@ import { DashboardModule } from './modules/dashboard/dashboard.module';
 		AuthModule,
 		CommentsModule,
 		DashboardModule,
+		PerformanceMetricsModule,
 	],
 	providers: [
 		{
@@ -23,4 +25,4 @@ import { DashboardModule } from './modules/dashboard/dashboard.module';
 		},
 	],
 })
-export class AppModule {}
+export class AppModule { }

src/domain/abstractions/data-services.abstract.ts

@@ -1,8 +1,9 @@
-import { CommentEntity, UserEntity } from '../../domain/entities';
+import { CommentEntity, NlpStatsEntity, UserEntity } from '../../domain/entities';
 
 import { IGenericRepository } from './generic-repository.abstract';
 
 export abstract class IDataServices {
 	abstract users: IGenericRepository<UserEntity>;
 	abstract comments: IGenericRepository<CommentEntity>;
+	abstract nlpStats: IGenericRepository<NlpStatsEntity>;
 }

src/domain/dtos/index.ts

@@ -6,12 +6,14 @@ import { CreateCommentDto } from './comment/create.comment.dto';
 import { FilterCommentDto } from './comment/filter.comment.dto';
 
 import { FilterDashboardDto } from './dashboard/filter.dashboard.dto';
+import { FilterNlpStatsDto } from './nlp-stats/filter.nlp-stats.dto';
 
 export {
 	CreateUserDto,
 	UpdateUserDto,
 	UpdateUserRoleDto,
 	CreateCommentDto,
 	FilterCommentDto,
-	FilterDashboardDto
+	FilterDashboardDto,
+	FilterNlpStatsDto
 };

src/domain/dtos/nlp-stats/filter.nlp-stats.dto.ts

@@ -0,0 +1,30 @@
+import { IsDate, IsOptional } from 'class-validator';
+import { PartialType } from '@nestjs/swagger';
+
+import { Transform } from 'class-transformer';
+
+class FilterNlpStats {
+	@IsOptional()
+	@Transform(({ value }) => {
+		const date = new Date(value)
+		if (Date.prototype.toString.call(date) == 'Invalid Date') {
+			return value
+		}
+		return date
+	})
+	@IsDate()
+	dateStart: Date;
+
+	@IsOptional()
+	@Transform(({ value }) => {
+		const date = new Date(value)
+		if (Date.prototype.toString.call(date) == 'Invalid Date') {
+			return value
+		}
+		return date
+	})
+	@IsDate()
+	dateEnd: Date;
+}
+
+export class FilterNlpStatsDto extends PartialType(FilterNlpStats) { }

src/domain/entities/index.ts

@@ -7,6 +7,8 @@ import {
 	CommentMongoEntity,
 } from './comment.entity';
 
+import { NlpStatsEntity, NlpStatsMongoEntity } from './nlp_stats.entity';
+
 export {
 	UserEntity,
 	UserMongoEntity,
@@ -15,4 +17,6 @@ export {
 	CommentSentimentEnum,
 	CommentTopicEnum,
 	CommentMongoEntity,
+	NlpStatsEntity,
+	NlpStatsMongoEntity,
 };

src/domain/entities/nlp_stats.entity.ts

@@ -0,0 +1,106 @@
+import { ObjectId } from 'mongodb';
+
+interface IPipelineStageMetric {
+	stage: string;
+	time: string;
+	day: Date;
+}
+
+interface ITotalOfDataMetric {
+	total_of_data: number;
+}
+
+interface IModelMetric {
+	model_accuracy: number;
+	model_precision: number;
+}
+
+type MetricEntry = IPipelineStageMetric | ITotalOfDataMetric | IModelMetric;
+
+interface IError {
+	type: string;
+	value: number;
+	day?: Date;
+}
+
+export class NlpStatsMongoEntity {
+	_id?: ObjectId;
+	createdAt: Date;
+	modelAccuracy?: number;
+	modelPrecision?: number;
+	totalOfData?: number;
+
+	metrics: MetricEntry[] = [];
+	erros: IError[] = [];
+	errors: IError[] = [];
+
+	stages: IPipelineStageMetric[] = []
+
+	constructor(
+		_id?: ObjectId,
+		metrics: MetricEntry[] = [],
+		erros: IError[] = []
+	) {
+		this._id = _id as unknown as ObjectId;
+		this.metrics = metrics;
+		this.errors = erros.flatMap((error) => error);
+
+		this.parseMetrics();
+		this.handleCreatedAt();
+	}
+
+	private parseMetrics() {
+		for (const metric of this.metrics) {
+			if ('stage' in metric) {
+				const stageMetric = metric as IPipelineStageMetric;
+				this.stages.push(stageMetric)
+
+				if (!this.createdAt || Boolean(stageMetric.day) && this.createdAt > stageMetric.day) {
+					this.createdAt = stageMetric.day
+				}
+
+			} else if ('total_of_data' in metric) {
+				const totalOfDataMetric = metric as ITotalOfDataMetric;
+				this.totalOfData = totalOfDataMetric.total_of_data
+			} else if ('model_accuracy' in metric) {
+				const modelMetric = metric as IModelMetric;
+				this.modelAccuracy = modelMetric.model_accuracy
+				this.modelPrecision = modelMetric.model_precision
+			}
+		}
+	}
+
+	private handleCreatedAt() {
+		this.errors.forEach(error => {
+
+			if (!this.createdAt || Boolean(error.day) && this.createdAt > error.day) {
+				this.createdAt = error.day
+			}
+		})
+	}
+}
+
+
+export class NlpStatsEntity {
+	id: string;
+	createdAt: Date
+	modelAccuracy: number
+	modelPrecision: number
+	totalOfData: number
+
+	stages: IPipelineStageMetric[]
+	errors: IError[]
+
+
+	constructor(data: Partial<NlpStatsMongoEntity>) {
+		this.id = data._id.toString();
+		this.createdAt = data.createdAt;
+		this.modelAccuracy = data.modelAccuracy;
+		this.modelPrecision = data.modelPrecision;
+		this.totalOfData = data.totalOfData;
+
+		this.errors = data.errors;
+		this.stages = data.stages;
+	}
+}
+

src/domain/repositories/index.ts

@@ -1,4 +1,5 @@
 import { UserRepository } from './user.repository';
 import { CommentRepository } from './comment.repository';
+import { NlpStatsRepository } from './nlp_stats.repository';
 
-export { UserRepository, CommentRepository };
+export { UserRepository, CommentRepository, NlpStatsRepository };

src/domain/repositories/nlp_stats.repository.ts

@@ -0,0 +1,10 @@
+import { IGenericRepository } from '../../domain/abstractions/generic-repository.abstract';
+import { FilterNlpStatsDto } from '../dtos';
+import { NlpStatsEntity } from '../entities';
+
+
+export interface NlpStatsRepository extends IGenericRepository<NlpStatsEntity> {
+	processingTime(
+		filters: FilterNlpStatsDto
+	): Promise<any>;
+}

src/modules/dashboard/dashboard.controller.ts

@@ -1,10 +1,12 @@
 import { Controller, Get, Query } from '@nestjs/common';
+import { ApiTags } from '@nestjs/swagger';
 import { DashboardService } from './dashboard.service';
 import { FilterDashboardDto } from '../../domain/dtos/dashboard/filter.dashboard.dto';
 
 @Controller('dashboard')
+@ApiTags('Dashboard')
 export class DashboardController {
-	constructor(private readonly dashboardService: DashboardService) {}
+	constructor(private readonly dashboardService: DashboardService) { }
 
 	@Get()
 	findAll(

src/modules/performance-metrics/performance-metrics.controller.ts

@@ -0,0 +1,17 @@
+import { Controller, Get, Query } from '@nestjs/common';
+import { ApiTags } from '@nestjs/swagger';
+import { PerformanceMetricsService } from './performance-metrics.service';
+import { FilterNlpStatsDto } from '../../domain/dtos/nlp-stats/filter.nlp-stats.dto';
+
+@Controller('performance-metrics')
+@ApiTags('Performance Metrics')
+export class PerformanceMetricsController {
+	constructor(private readonly performanceMetricsService: PerformanceMetricsService) { }
+
+	@Get()
+	findAll(
+		@Query() filters: FilterNlpStatsDto,
+	) {
+		return this.performanceMetricsService.findAll(filters);
+	}
+}

src/modules/performance-metrics/performance-metrics.module.ts

@@ -0,0 +1,10 @@
+import { Module } from '@nestjs/common';
+import { PerformanceMetricsService } from './performance-metrics.service';
+import { PerformanceMetricsController } from './performance-metrics.controller';
+import { DatabaseService } from '../../shared/database/services/database.service';
+
+@Module({
+	controllers: [PerformanceMetricsController],
+	providers: [PerformanceMetricsService, DatabaseService],
+})
+export class PerformanceMetricsModule { }

src/modules/performance-metrics/performance-metrics.service.ts

@@ -0,0 +1,28 @@
+import { Injectable } from '@nestjs/common';
+import { DatabaseService } from '../../shared/database/services/database.service';
+import { FilterNlpStatsDto } from '../../domain/dtos/nlp-stats/filter.nlp-stats.dto';
+
+@Injectable()
+export class PerformanceMetricsService {
+	constructor(private readonly databaseService: DatabaseService) { }
+
+	async findAll(filters: FilterNlpStatsDto) {
+		const {
+			totalDocumentsProcessed,
+			dailyTotalProcessingTime,
+			timeByPipelineStage,
+			errorRate,
+			dailyTotalErrors,
+			errorsByType
+		} = await this.databaseService.nlpStats.processingTime(filters)
+
+		return {
+			totalDocumentsProcessed,
+			errorRate,
+			dailyTotalProcessingTime,
+			dailyTotalErrors,
+			errorsByType,
+			timeByPipelineStage
+		}
+	}
+}

src/shared/database/implementations/index.ts

@@ -1,4 +1,5 @@
 import { UserRepositoryImpl } from './user.repository.impl';
 import { CommentRepositoryImpl } from './comment.repository.impl';
+import { NlpStatsRepositoryImpl } from './nlp_stats.repository.impl';
 
-export { UserRepositoryImpl, CommentRepositoryImpl };
+export { UserRepositoryImpl, CommentRepositoryImpl, NlpStatsRepositoryImpl };