From bf0a41142b3d492afde99a80cdddf111a70d1adf Mon Sep 17 00:00:00 2001 From: mhmadalaa Date: Thu, 11 Jul 2024 09:43:13 +0300 Subject: [PATCH 1/4] feat: add pages text summarization --- .../textSummarizationController.js | 136 ++++++++++++++++++ src/routers/aiRoute.js | 10 +- 2 files changed, 142 insertions(+), 4 deletions(-) create mode 100644 src/controllers/textSummarizationController.js diff --git a/src/controllers/textSummarizationController.js b/src/controllers/textSummarizationController.js new file mode 100644 index 0000000..977a6c2 --- /dev/null +++ b/src/controllers/textSummarizationController.js @@ -0,0 +1,136 @@ +const axios = require('axios'); +const catchAsync = require('../utils/catchAsync'); +const Question = require('./../models/questionModel'); +const AppError = require('../utils/appError'); +const Book = require('../models/BookModel'); +const Document = require('./../models/documentModel'); +const Chat = require('./../models/chatModel'); +const { pipeline } = require('node:stream/promises'); +const { BufferListStream } = require('bl'); +const AI_API = process.env.AI_API; + +exports.summarizeQuestion = catchAsync(async (req, res) => { + const file = await chatAndFileId(req); + + req.chat_id = file.chat._id; + + const dataToSend = { + start_page: parseInt(req.query.start_page), + end_page: parseInt(req.query.end_page), + }; + + const bufferStream = new BufferListStream(); + + // @app.get("/summarize_pages/{doc_id}") + // async def summarize_pages(doc_id: str, start_page: int, end_page: int): + axios + .get(`${AI_API}/summarize_pages/${file.file_id.toString()}`, { + data: dataToSend, + responseType: 'stream', + }) + .then(async (response) => { + // accumulate the response data to a buffer list + response.data.on('data', (chunk) => { + // Append each chunk to the buffer + bufferStream.append(Buffer.from(chunk)); + }); + + // pipe the response stream to client + await pipeline(response.data, res); + + try { + const chat_answer = bufferStream.toString(); + + // save the question data to database + await Question.create({ + question: req.body.question, + answer: chat_answer, + chat_id: req.chat_id, + user: req.user._id, + createdAt: Date.now(), + }); + + // Increment the book recommendation factor by 0.1 + await Book.findByIdAndUpdate( + req.params.id, + { $inc: { recommendation: 0.1 } }, + { new: true }, + ); + } catch (error) { + console.error( + '✗ can not save [page text summarization] in chat answer to database', + error, + ); + } + }) + .catch(function (error) { + res.status(500).json({ + message: + 'can not connect to ai-api to answer [page text summarization] in chat question', + }); + }); +}); + +exports.enforceQueryParams = (req, res, next) => { + if ( + req.query && + (req.query.type === 'book' || req.query.type === 'document') && + req.query.start_page && + req.query.end_page + ) { + return next(); + } + + return next( + new AppError( + 'Missing required query parameters, it should be in the form ?type=book or ?type=document and &req.qurey.start_page &req.query.end_page', + 400, + ), + ); +}; + +const chatAndFileId = async (req) => { + let file_id, chat; + if (req.query.type === 'book') { + const book = await Book.findById(req.params.id); + file_id = book.file_id; + + chat = await Chat.findOne({ + book: req.params.id, + user: req.user._id, + }); + + // chat = null meaning that it's the first question in the chat + // and there is no chat with these data + if (chat === null) { + chat = await Chat.create({ + chat_summary: 'none', + book: book._id, + user: req.user._id, + createdAt: Date.now(), + }); + } + } + if (req.query.type === 'document') { + const document = await Document.findById(req.params.id); + file_id = document.file_id; + + chat = await Chat.findOne({ + document: req.params.id, + user: req.user._id, + }); + + // chat = null meaning that it's the first question in the chat + // and there is no chat with these data + if (chat === null) { + chat = await Chat.create({ + chat_summary: 'none', + document: document._id, + user: req.user._id, + createdAt: Date.now(), + }); + } + } + + return { file_id: file_id, chat: chat }; +}; diff --git a/src/routers/aiRoute.js b/src/routers/aiRoute.js index 382c7b8..14ea665 100644 --- a/src/routers/aiRoute.js +++ b/src/routers/aiRoute.js @@ -3,6 +3,7 @@ const questionController = require('../controllers/questionController'); const textToSpeechController = require('../controllers/textToSpeechController'); const chapterSummaryController = require('../controllers/chSummaryController'); const authController = require('./../controllers/authControllers'); +const textSummarizationController = require('../controllers/textSummarizationController'); const router = express.Router(); @@ -25,10 +26,11 @@ router router .route('/tts') - .get( - authController.isLogin, - textToSpeechController.textToSpeech, - ); + .get(authController.isLogin, textToSpeechController.textToSpeech); + +router + .route('/text-summarization/:id') + .get(authController.isLogin, textSummarizationController.summarizeQuestion); /* TODO: From 5d879af286689c0b2f4de22f7ff3cbcb391cd7b4 Mon Sep 17 00:00:00 2001 From: mhmadalaa Date: Thu, 11 Jul 2024 09:48:03 +0300 Subject: [PATCH 2/4] remove ai-api interface document --- src/controllers/textSummarizationController.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/controllers/textSummarizationController.js b/src/controllers/textSummarizationController.js index 977a6c2..ac46037 100644 --- a/src/controllers/textSummarizationController.js +++ b/src/controllers/textSummarizationController.js @@ -21,8 +21,6 @@ exports.summarizeQuestion = catchAsync(async (req, res) => { const bufferStream = new BufferListStream(); - // @app.get("/summarize_pages/{doc_id}") - // async def summarize_pages(doc_id: str, start_page: int, end_page: int): axios .get(`${AI_API}/summarize_pages/${file.file_id.toString()}`, { data: dataToSend, From 97a370b76187e7253a0ff5381bc626a0f47eb454 Mon Sep 17 00:00:00 2001 From: mhmadalaa Date: Thu, 11 Jul 2024 10:02:30 +0300 Subject: [PATCH 3/4] fix: let user query if he need web retrieval or not --- src/controllers/questionController.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/controllers/questionController.js b/src/controllers/questionController.js index bf21ab7..06ad043 100644 --- a/src/controllers/questionController.js +++ b/src/controllers/questionController.js @@ -12,7 +12,7 @@ const AI_API = process.env.AI_API; exports.askQuestion = catchAsync(async (req, res) => { // ai-request query parameters const queryParams = { - enable_web_retrieval: true, + enable_web_retrieval: req.query.enable_web_retrieval === 'true' || true, }; const chat = await chatAndFileId(req); From 73708761078b82a7f89b0391a89302a8785995fb Mon Sep 17 00:00:00 2001 From: mhmadalaa Date: Thu, 11 Jul 2024 10:09:12 +0300 Subject: [PATCH 4/4] fix: enforce query parameters validation --- src/routers/aiRoute.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/routers/aiRoute.js b/src/routers/aiRoute.js index 14ea665..d871e68 100644 --- a/src/routers/aiRoute.js +++ b/src/routers/aiRoute.js @@ -30,7 +30,11 @@ router router .route('/text-summarization/:id') - .get(authController.isLogin, textSummarizationController.summarizeQuestion); + .get( + authController.isLogin, + textSummarizationController.enforceQueryParams, + textSummarizationController.summarizeQuestion, + ); /* TODO: