From bce483282bc90ba7770a2d070affe59ab6842617 Mon Sep 17 00:00:00 2001 From: Alperen Elhan Date: Fri, 14 Oct 2022 04:02:01 +0300 Subject: [PATCH] feat: add scheduling --- .gitignore | 4 +- .../migration.sql | 5 +- prisma/schema.prisma | 5 +- src/components/addLibrary.tsx | 5 +- src/components/addManga/form.tsx | 10 +- src/components/addManga/mangaSearchResult.tsx | 1 - .../addManga/steps/downloadStep.tsx | 8 +- src/components/addManga/steps/reviewStep.tsx | 5 +- src/components/addManga/steps/searchStep.tsx | 3 - src/components/mangaCard.tsx | 48 ++++- src/pages/index.tsx | 44 +++- src/server/downloader/bullboard.ts | 12 -- src/server/downloader/config.ts | 196 ------------------ src/server/downloader/downloader.ts | 71 ------- src/server/downloader/library.ts | 44 ---- src/server/downloader/queue.ts | 23 -- src/server/downloader/scheduler.ts | 100 --------- src/server/downloader/utils.ts | 8 - src/server/index.ts | 24 ++- src/server/queue/checkChapters.ts | 101 +++++++++ src/server/queue/download.ts | 47 +++++ src/server/trpc/router/library.ts | 8 +- src/server/trpc/router/manga.ts | 65 ++++-- src/{ => server}/utils/mangal.ts | 63 ++++-- .../{downloader => utils}/notification.ts | 0 src/utils/sanitize.ts | 6 + 26 files changed, 368 insertions(+), 538 deletions(-) rename prisma/migrations/{20221006210146_init => 20221014002316_init}/migration.sql (77%) delete mode 100644 src/server/downloader/bullboard.ts delete mode 100644 src/server/downloader/config.ts delete mode 100644 src/server/downloader/downloader.ts delete mode 100644 src/server/downloader/library.ts delete mode 100644 src/server/downloader/queue.ts delete mode 100644 src/server/downloader/scheduler.ts delete mode 100644 src/server/downloader/utils.ts create mode 100644 src/server/queue/checkChapters.ts create mode 100644 src/server/queue/download.ts rename src/{ => server}/utils/mangal.ts (62%) rename src/server/{downloader => utils}/notification.ts (100%) create mode 100644 src/utils/sanitize.ts diff --git a/.gitignore b/.gitignore index e338717..9c2fe90 100644 --- a/.gitignore +++ b/.gitignore @@ -379,4 +379,6 @@ $RECYCLE.BIN/ # and uncomment the following lines # .pnp.* -# End of https://www.toptal.com/developers/gitignore/api/linux,windows,macos,node,visualstudiocode,intellij+all,nextjs,yarn \ No newline at end of file +# End of https://www.toptal.com/developers/gitignore/api/linux,windows,macos,node,visualstudiocode,intellij+all,nextjs,yarn + +mangas/ \ No newline at end of file diff --git a/prisma/migrations/20221006210146_init/migration.sql b/prisma/migrations/20221014002316_init/migration.sql similarity index 77% rename from prisma/migrations/20221006210146_init/migration.sql rename to prisma/migrations/20221014002316_init/migration.sql index 95a4644..545de16 100644 --- a/prisma/migrations/20221006210146_init/migration.sql +++ b/prisma/migrations/20221014002316_init/migration.sql @@ -11,7 +11,6 @@ CREATE TABLE "Manga" ( "cover" TEXT NOT NULL, "interval" TEXT NOT NULL, "source" TEXT NOT NULL, - "query" TEXT NOT NULL, - "libraryId" INTEGER, - CONSTRAINT "Manga_libraryId_fkey" FOREIGN KEY ("libraryId") REFERENCES "Library" ("id") ON DELETE SET NULL ON UPDATE CASCADE + "libraryId" INTEGER NOT NULL, + CONSTRAINT "Manga_libraryId_fkey" FOREIGN KEY ("libraryId") REFERENCES "Library" ("id") ON DELETE RESTRICT ON UPDATE CASCADE ); diff --git a/prisma/schema.prisma b/prisma/schema.prisma index bbbfcc8..7b04c78 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -23,7 +23,6 @@ model Manga { cover String interval String source String - query String - Library Library? @relation(fields: [libraryId], references: [id]) - libraryId Int? + Library Library @relation(fields: [libraryId], references: [id]) + libraryId Int } diff --git a/src/components/addLibrary.tsx b/src/components/addLibrary.tsx index b685a13..45ce012 100644 --- a/src/components/addLibrary.tsx +++ b/src/components/addLibrary.tsx @@ -29,8 +29,9 @@ function Form({ onClose }: { onClose: () => void }) {
{ setVisible((v) => !v); + let library = null; try { - await libraryMutation.mutateAsync({ + library = await libraryMutation.mutateAsync({ path: values.library.path, }); } catch (err) { @@ -61,7 +62,7 @@ function Form({ onClose }: { onClose: () => void }) { title: 'Library', message: ( - Library is created at {values.library.path} + Library is created at {library.path} ), }); diff --git a/src/components/addManga/form.tsx b/src/components/addManga/form.tsx index 92a3c9e..1e3845e 100644 --- a/src/components/addManga/form.tsx +++ b/src/components/addManga/form.tsx @@ -27,7 +27,6 @@ const useStyles = createStyles((theme) => ({ const schema = z.object({ source: z.string().min(1, { message: 'You must select a source' }), query: z.string().min(1, { message: 'Cannot be empty' }), - mangaOrder: z.number().gte(0, { message: 'Please select a manga' }), mangaTitle: z.string().min(1, { message: 'Please select a manga' }), interval: z.string().min(1, { message: 'Please select an interval' }), }); @@ -46,7 +45,6 @@ export function AddMangaForm({ onClose }: { onClose: () => void }) { initialValues: { source: '', query: '', - mangaOrder: -1, mangaTitle: '', interval: '', }, @@ -62,8 +60,7 @@ export function AddMangaForm({ onClose }: { onClose: () => void }) { } if (active === 1) { form.validateField('mangaTitle'); - form.validateField('mangaOrder'); - if (!form.isValid('mangaOrder') || !form.isValid('mangaTitle')) { + if (!form.isValid('mangaTitle')) { return; } } @@ -89,7 +86,6 @@ export function AddMangaForm({ onClose }: { onClose: () => void }) { } if (active === 2) { form.setFieldValue('query', ''); - form.setFieldValue('mangaOrder', -1); form.setFieldValue('mangaTitle', ''); form.setFieldValue('interval', ''); } @@ -101,11 +97,9 @@ export function AddMangaForm({ onClose }: { onClose: () => void }) { const onSubmit = form.onSubmit(async (values) => { setVisible((v) => !v); - const { mangaOrder, mangaTitle, query, source, interval } = values; + const { mangaTitle, source, interval } = values; try { await mutation.mutateAsync({ - keyword: query, - order: mangaOrder, title: mangaTitle, interval, source, diff --git a/src/components/addManga/mangaSearchResult.tsx b/src/components/addManga/mangaSearchResult.tsx index 96f69ce..0d222e8 100644 --- a/src/components/addManga/mangaSearchResult.tsx +++ b/src/components/addManga/mangaSearchResult.tsx @@ -98,7 +98,6 @@ ImageCheckbox.defaultProps = { type IMangaSearchResult = { status: string; title: string; - order: number; cover: string; }; diff --git a/src/components/addManga/steps/downloadStep.tsx b/src/components/addManga/steps/downloadStep.tsx index a25644a..c1fa00f 100644 --- a/src/components/addManga/steps/downloadStep.tsx +++ b/src/components/addManga/steps/downloadStep.tsx @@ -1,6 +1,7 @@ import { Box, LoadingOverlay, Select, Stack, TextInput } from '@mantine/core'; import { UseFormReturnType } from '@mantine/form'; import { IconFolderPlus } from '@tabler/icons'; +import { sanitizer } from '../../../utils/sanitize'; import { trpc } from '../../../utils/trpc'; import type { FormType } from '../form'; @@ -17,12 +18,7 @@ export function DownloadStep({ form }: { form: UseFormReturnType }) { return ; } - const sanitizeMangaName = form.values.mangaTitle - .replaceAll(/[\\/<>:;"'|?!*{}#%&^+,~\s]/g, '_') - .replaceAll(/__+/g, '_') - .replaceAll(/^[_\-.]+|[_\-.]+$/g, '_'); - - const downloadPath = `${libraryPath}/${sanitizeMangaName}`; + const downloadPath = `${libraryPath}/${sanitizer(form.values.mangaTitle)}`; return ( diff --git a/src/components/addManga/steps/reviewStep.tsx b/src/components/addManga/steps/reviewStep.tsx index e7dea29..57cde94 100644 --- a/src/components/addManga/steps/reviewStep.tsx +++ b/src/components/addManga/steps/reviewStep.tsx @@ -13,13 +13,12 @@ const useStyles = createStyles((_theme) => ({ export function ReviewStep({ form }: { form: UseFormReturnType }) { const query = trpc.manga.detail.useQuery( { - keyword: form.values.query, source: form.values.source, - order: form.values.mangaOrder, + title: form.values.mangaTitle, }, { staleTime: Infinity, - enabled: !!form.values.query && !!form.values.source && !!form.values.mangaTitle && form.values.mangaOrder > -1, + enabled: !!form.values.source && !!form.values.mangaTitle, }, ); diff --git a/src/components/addManga/steps/searchStep.tsx b/src/components/addManga/steps/searchStep.tsx index 2f591d4..854bc41 100644 --- a/src/components/addManga/steps/searchStep.tsx +++ b/src/components/addManga/steps/searchStep.tsx @@ -19,7 +19,6 @@ export function SearchStep({ form }: { form: UseFormReturnType }) { if (!form.isValid('query')) { return; } - form.setFieldValue('mangaOrder', -1); form.setFieldValue('mangaTitle', ''); setLoading(true); const result = await ctx.manga.search.fetch({ @@ -59,10 +58,8 @@ export function SearchStep({ form }: { form: UseFormReturnType }) { items={searchResult} onSelect={(selected) => { if (selected) { - form.setFieldValue('mangaOrder', selected.order); form.setFieldValue('mangaTitle', selected.title); } else { - form.setFieldValue('mangaOrder', -1); form.setFieldValue('mangaTitle', ''); } }} diff --git a/src/components/mangaCard.tsx b/src/components/mangaCard.tsx index 160cca7..f0da48c 100644 --- a/src/components/mangaCard.tsx +++ b/src/components/mangaCard.tsx @@ -1,8 +1,10 @@ -import { Badge, Button, createStyles, Paper, Title } from '@mantine/core'; -import { IconExternalLink } from '@tabler/icons'; +import { ActionIcon, Badge, Button, Code, createStyles, Paper, Text, Title } from '@mantine/core'; +import { openConfirmModal } from '@mantine/modals'; +import { IconExternalLink, IconX } from '@tabler/icons'; -const useStyles = createStyles((theme) => ({ +const useStyles = createStyles((theme, _params, getRef) => ({ card: { + position: 'relative', height: 350, width: 210, display: 'flex', @@ -18,8 +20,17 @@ const useStyles = createStyles((theme) => ({ transform: 'scale(1.01)', boxShadow: theme.shadows.md, }, + [`&:hover .${getRef('removeButton')}`]: { + display: 'flex', + }, + }, + removeButton: { + ref: getRef('removeButton'), + position: 'absolute', + right: -5, + top: -5, + display: 'none', }, - title: { fontFamily: `${theme.fontFamily}`, fontWeight: 900, @@ -37,10 +48,34 @@ interface ArticleCardImageProps { image: string; title: string; category?: string; + onRemove: () => void; } -export function MangaCard({ image, title, category }: ArticleCardImageProps) { +const createRemoveModal = (title: string, onRemove: () => void) => { + const openDeleteModal = () => + openConfirmModal({ + title: `Delete ${title}?`, + centered: true, + children: ( + + Are you sure you want to delete + + {title} + + ? This action is destructive and all downloaded files will be removed + + ), + labels: { confirm: 'Delete', cancel: 'Cancel' }, + confirmProps: { color: 'red' }, + onConfirm: onRemove, + }); + + return openDeleteModal; +}; + +export function MangaCard({ image, title, category, onRemove }: ArticleCardImageProps) { const { classes } = useStyles(); + const removeModal = createRemoveModal(title, onRemove); return ( + + +
{category && ( diff --git a/src/pages/index.tsx b/src/pages/index.tsx index 7875bd6..4079c31 100644 --- a/src/pages/index.tsx +++ b/src/pages/index.tsx @@ -1,4 +1,6 @@ -import { Grid, LoadingOverlay } from '@mantine/core'; +import { Code, Grid, LoadingOverlay, Text } from '@mantine/core'; +import { showNotification } from '@mantine/notifications'; +import { IconCheck, IconX } from '@tabler/icons'; import { AddManga } from '../components/addManga'; import { EmptyPrompt } from '../components/emptyPrompt'; @@ -7,6 +9,7 @@ import { trpc } from '../utils/trpc'; export default function IndexPage() { const libraryQuery = trpc.library.query.useQuery(); + const mangaDelete = trpc.manga.remove.useMutation(); const libraryId = libraryQuery.data?.id; @@ -34,6 +37,38 @@ export default function IndexPage() { ); } + const handleDelete = async (id: number, title: string) => { + try { + await mangaDelete.mutateAsync({ + id, + }); + showNotification({ + icon: , + color: 'teal', + autoClose: true, + title: 'Manga', + message: ( + + {title} is removed from library + + ), + }); + } catch (err) { + showNotification({ + icon: , + color: 'red', + autoClose: true, + title: 'Manga', + message: ( + + {`${err}`} + + ), + }); + } + mangaQuery.refetch(); + }; + return ( @@ -47,7 +82,12 @@ export default function IndexPage() { mangaQuery.data.map((manga) => { return ( - + handleDelete(manga.id, manga.title)} + /> ); })} diff --git a/src/server/downloader/bullboard.ts b/src/server/downloader/bullboard.ts deleted file mode 100644 index 1ffc16a..0000000 --- a/src/server/downloader/bullboard.ts +++ /dev/null @@ -1,12 +0,0 @@ -import { createBullBoard } from '@bull-board/api'; -import { BullAdapter } from '@bull-board/api/bullAdapter'; -import { ExpressAdapter } from '@bull-board/express'; -import { checkChaptersQue, downloadQue } from './queue'; - -export const serverAdapter = new ExpressAdapter(); -serverAdapter.setBasePath('/admin/queues'); - -createBullBoard({ - queues: [new BullAdapter(downloadQue), new BullAdapter(checkChaptersQue)], - serverAdapter, -}); diff --git a/src/server/downloader/config.ts b/src/server/downloader/config.ts deleted file mode 100644 index 06692db..0000000 --- a/src/server/downloader/config.ts +++ /dev/null @@ -1,196 +0,0 @@ -import fs from 'fs/promises'; -import { diffString } from 'json-diff'; -import path from 'path'; -import { parse, stringify } from 'yaml'; -import { logger } from '../../utils/logging'; -import { IFlags } from './utils'; - -export interface IComic { - name: string; - download?: { - checkForUpdate: 'daily' | 'hourly' | 'never' | 'weekly' | 'minutely'; - source: string; - query: string; - index: number; - }; - metadata?: { - write: boolean; - match: { - anilist?: string; - kitsu?: string; - myanimelist?: string; - komga?: string; - }; - }; - synonyms: string[]; -} - -interface ICollection { - path: string; - titles: { - [key: string]: IComic; - }; -} - -export interface IConfig { - collection: ICollection; -} - -export const getDefaultConfig = (): IConfig => ({ - collection: { - path: '', - titles: {}, - }, -}); - -export const readConfig = async (configPath: string): Promise => { - const ymlPath = path.resolve(process.cwd(), path.relative(process.cwd(), configPath)); - try { - const configFile = await fs.readFile(ymlPath, 'utf-8'); - if (configFile.trim().length === 0) { - logger.error(`Config file at ${ymlPath} is empty`); - return getDefaultConfig(); - } - return parse(configFile); - } catch (err) { - logger.error(`Failed to parse file ${ymlPath}. err: ${err}`); - return getDefaultConfig(); - } -}; - -const getIndex = async (libraryPath: string, title: string): Promise => { - const titlePath = path.resolve(libraryPath, title); - await fs.mkdir(titlePath, { recursive: true }); - const titleFiles = await fs.readdir(titlePath); - - const chapters = titleFiles - .filter((chapter) => chapter.endsWith('cbz')) - .sort() - .reverse(); - - if (chapters.length === 0 || !chapters[0]) { - return 1; - } - const indexRegexp = /.*?\[(\d+)\].*/; - const match = indexRegexp.exec(chapters[0]); - if (!match || match.length < 2 || !match[1]) { - return 1; - } - return parseInt(match[1], 10); -}; - -export const updateConfig = async (flags: IFlags) => { - const configPath = path.resolve(process.cwd(), path.relative(process.cwd(), flags.config)); - let configFile; - try { - configFile = await fs.open(configPath, 'r+'); - } catch (err) { - logger.error(`Cannot open the config at ${configPath}. err: ${err}`); - process.exit(1); - } - - const configFileContent = await configFile.readFile('utf-8'); - if (configFileContent.trim().length === 0) { - logger.error(`Config file at "${configPath}" is empty`); - await configFile.close(); - process.exit(1); - } - - let config: IConfig; - try { - config = parse(configFileContent); - } catch (err) { - logger.error(`Failed to parse config file at ${configPath}`); - await configFile.close(); - process.exit(1); - } - - await fs.mkdir(config.collection.path, { recursive: true }); - const library = [ - ...new Set([...(await fs.readdir(config.collection.path)), ...(Object.keys(config.collection.titles) || [])]), - ]; - - const titles = await Promise.all( - library.map(async (title) => { - const sanitizedTitle = title.replaceAll('_', ' '); - const download = config.collection.titles[title]?.download; - const synonyms = config.collection.titles[title]?.synonyms; - return { - name: title, - download: { - index: await getIndex(config.collection.path, title), - checkForUpdate: download?.checkForUpdate || 'hourly', - query: download?.query || sanitizedTitle, - source: download?.source || '', - }, - synonyms: synonyms || [sanitizedTitle], - }; - }), - ); - - const newTitles = titles.sort().reduce((acc, current) => { - return { - ...acc, - [current.name]: { name: current.name, download: current.download, synonyms: current.synonyms }, - }; - }, {}); - - const diff = diffString(config.collection.titles, newTitles); - logger.debug(diff || 'No diff'); - - if (diff) { - config.collection.titles = newTitles; - await configFile.write(stringify(config), 0); - - logger.info(`Successfully updated config at "${configPath}" for the library "${config.collection.path}"`); - } - await configFile.close(); -}; - -export const generateConfig = async (flags: IFlags) => { - const configPath = path.resolve(process.cwd(), path.relative(process.cwd(), flags.config)); - const libraryPath = path.resolve(process.cwd(), path.relative(process.cwd(), flags.library)); - const configFile = await fs.open(configPath, 'a+'); - await fs.mkdir(libraryPath, { recursive: true }); - const configFileContent = await configFile.readFile('utf-8'); - if (configFileContent.trim().length > 0) { - logger.error(`Config file already exists at "${configPath}"`); - await configFile.close(); - process.exit(1); - } - - const config: IConfig = getDefaultConfig(); - config.collection.path = libraryPath; - - const library = await fs.readdir(libraryPath); - - const titles = await Promise.all( - library.map(async (title) => { - const sanitizedTitle = title.replaceAll('_', ' '); - logger.info(`Setting up config for "${sanitizedTitle}"`); - - return { - name: title, - download: { - index: await getIndex(libraryPath, title), - checkForUpdate: 'hourly', - query: sanitizedTitle, - source: '', - }, - synonyms: [sanitizedTitle], - }; - }), - ); - - config.collection.titles = titles.reduce((acc, current) => { - return { - ...acc, - [current.name]: { name: current.name, download: current.download, synonyms: current.synonyms }, - }; - }, {}); - - await configFile.write(stringify(config), 0); - await configFile.close(); - - logger.info(`Successfully generated config at "${configPath}" for the library "${libraryPath}"`); -}; diff --git a/src/server/downloader/downloader.ts b/src/server/downloader/downloader.ts deleted file mode 100644 index a29c6be..0000000 --- a/src/server/downloader/downloader.ts +++ /dev/null @@ -1,71 +0,0 @@ -import { Job, Worker } from 'bullmq'; -import fs from 'fs/promises'; -import path from 'path'; -import { logger } from '../../utils/logging'; -import { downloadChapter, getAvailableSources, getChapters } from '../../utils/mangal'; -import { IComic } from './config'; -import { sendNotification } from './notification'; - -interface IDownloadWorkerData { - chapterIndex: number; - source: string; - query: string; - name: string; - libraryPath: string; -} - -export const findMissingChapters = async (title: IComic, libraryPath: string) => { - if (!title.download) { - logger.error(`Download option is not configured for ${title.name}`); - throw new Error(); - } - if (!title.download.source) { - logger.error(`Download source is not configured for ${title.name}`); - throw new Error(); - } - const sources = await getAvailableSources(); - if (sources.indexOf(title.download.source) < 0) { - logger.error(`Specified source: ${title.download.source} for ${title.name} is not installed.`); - throw new Error(); - } - const titlePath = path.resolve(libraryPath, title.name); - await fs.mkdir(titlePath, { recursive: true }); - const titleFiles = await fs.readdir(titlePath); - - const localChapters = titleFiles - .filter((chapter) => chapter.endsWith('cbz')) - .map((chapter) => { - const indexRegexp = /.*?\[(\d+)\].*/; - const match = indexRegexp.exec(chapter); - if (!match || match.length < 2 || !match[1]) { - return 1; - } - return parseInt(match[1], 10); - }) - .filter((index) => index !== -1); - - const remoteChapters = await getChapters(title.download?.source, title.download?.query, 'first'); - return remoteChapters.filter((c) => !localChapters.includes(c)); -}; - -export const downloadWorker = new Worker( - 'downloadQue', - async (job: Job) => { - const { chapterIndex, libraryPath, name, query, source }: IDownloadWorkerData = job.data; - try { - downloadChapter(name, source, query, chapterIndex, 'first', libraryPath); - } catch (err) { - await job.log(`${err}`); - throw err; - } - await sendNotification(`Downloaded a new chapter #${chapterIndex + 1} for ${name} from ${source}`); - await job.updateProgress(100); - }, - { - concurrency: 5, - connection: { - host: 'localhost', - port: 6379, - }, - }, -); diff --git a/src/server/downloader/library.ts b/src/server/downloader/library.ts deleted file mode 100644 index a7d3c9a..0000000 --- a/src/server/downloader/library.ts +++ /dev/null @@ -1,44 +0,0 @@ -import chokidar from 'chokidar'; -import fs from 'fs/promises'; -import { diffString } from 'json-diff'; -import path from 'path'; -import { readConfig, updateConfig } from './config'; -import { logger } from '../../utils/logging'; -import { scheduleDownload } from './scheduler'; -import { IFlags } from './utils'; - -export const watchLibrary = async (flags: IFlags) => { - const ymlPath = path.resolve(process.cwd(), path.relative(process.cwd(), flags.config)); - let config = await readConfig(ymlPath); - - await fs.mkdir(config.collection.path, { recursive: true }); - let library = Object.keys(config.collection.titles); - await Promise.all( - library.map(async (dir) => { - const title = config.collection.titles[dir]; - if (title) { - scheduleDownload(title, config); - } - }), - ); - - chokidar.watch(ymlPath, { persistent: false }).on('change', async () => { - const newConfig = await readConfig(ymlPath); - const diff = diffString(config, newConfig); - config = newConfig; - if (diff) { - await updateConfig(flags); - library = Object.keys(config.collection.titles); - await Promise.all( - library.map(async (dir) => { - const title = config.collection.titles[dir]; - if (title) { - scheduleDownload(title, config); - } - }), - ); - } - - logger.debug(diff || 'No diff'); - }); -}; diff --git a/src/server/downloader/queue.ts b/src/server/downloader/queue.ts deleted file mode 100644 index 76ea4b1..0000000 --- a/src/server/downloader/queue.ts +++ /dev/null @@ -1,23 +0,0 @@ -import { Queue } from 'bullmq'; - -// Create a new connection in every instance -export const downloadQue = new Queue('downloadQue', { - connection: { - host: 'localhost', - port: 6379, - }, - defaultJobOptions: { - attempts: 20, - backoff: { - type: 'fixed', - delay: 1000 * 60 * 2, - }, - }, -}); - -export const checkChaptersQue = new Queue('checkChaptersQue', { - connection: { - host: 'localhost', - port: 6379, - }, -}); diff --git a/src/server/downloader/scheduler.ts b/src/server/downloader/scheduler.ts deleted file mode 100644 index fcabcff..0000000 --- a/src/server/downloader/scheduler.ts +++ /dev/null @@ -1,100 +0,0 @@ -import { Job, Worker } from 'bullmq'; -import { IComic, IConfig } from './config'; -import { findMissingChapters } from './downloader'; -import { logger } from '../../utils/logging'; -import { checkChaptersQue, downloadQue } from './queue'; - -const CRON_MAP = { - daily: '0 0 * * *', - hourly: '0 * * * *', - minutely: '* * * * *', - weekly: '0 * * * 7', -}; - -const checkChapters = async (title: IComic, libraryPath: string) => { - logger.info(`Checking for new chapters: ${title.name}`); - const missingChapters = await findMissingChapters(title, libraryPath); - - if (missingChapters.length === 0) { - logger.info(`There are no missing chapters for ${title.name}`); - } else { - logger.info(`There are ${missingChapters.length} new chapters for ${title.name}`); - } - - await Promise.all( - missingChapters.map(async (chapterIndex) => { - const job = await downloadQue.getJob(`${title.name}_${chapterIndex - 1}_download`); - if (job) { - await job.remove(); - } - }), - ); - - await downloadQue.addBulk( - missingChapters.map((chapterIndex) => ({ - opts: { - jobId: `${title.name}_${chapterIndex - 1}_download`, - }, - name: `${title.name}_${chapterIndex - 1}_download`, - data: { - chapterIndex: chapterIndex - 1, - source: title.download?.source, - query: title.download?.query, - name: title.name, - libraryPath, - }, - })), - ); -}; - -export const scheduleDownload = async (title: IComic, config: IConfig) => { - if (!title.download || title.download.checkForUpdate === 'never') { - return; - } - const jobs = await checkChaptersQue.getJobs('delayed'); - await Promise.all( - jobs.map(async (job) => { - if (job.id) { - return checkChaptersQue.remove(job.id); - } - return null; - }), - ); - await checkChaptersQue.add( - `check_${title.name}_chapters`, - { - title, - libraryPath: config.collection.path, - }, - { - jobId: `check_${title.name}_chapters`, - repeatJobKey: `check_${title.name}_chapters`, - repeat: { - pattern: CRON_MAP[title.download.checkForUpdate], - }, - }, - ); - - await checkChapters(title, config.collection.path); -}; - -interface ICheckChaptersWorker { - title: IComic; - libraryPath: string; -} - -export const checkChaptersWorker = new Worker( - 'checkChaptersQue', - async (job: Job) => { - const { title, libraryPath }: ICheckChaptersWorker = job.data; - await checkChapters(title, libraryPath); - await job.updateProgress(100); - }, - { - concurrency: 5, - connection: { - host: 'localhost', - port: 6379, - }, - }, -); diff --git a/src/server/downloader/utils.ts b/src/server/downloader/utils.ts deleted file mode 100644 index 9ab97a6..0000000 --- a/src/server/downloader/utils.ts +++ /dev/null @@ -1,8 +0,0 @@ -export interface IFlags { - config: string; - library: string; - help: boolean; - generate: boolean; - update: boolean; - watch: boolean; -} diff --git a/src/server/index.ts b/src/server/index.ts index 2098c8f..1cfce2a 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -1,14 +1,25 @@ import express, { Request, Response } from 'express'; import next from 'next'; -import { serverAdapter } from './downloader/bullboard'; -import { watchLibrary } from './downloader/library'; +import { ExpressAdapter } from '@bull-board/express'; +import { createBullBoard } from '@bull-board/api'; +import { BullAdapter } from '@bull-board/api/bullAdapter'; import { logger } from '../utils/logging'; +import { downloadQueue } from './queue/download'; +import { checkChaptersQueue } from './queue/checkChapters'; const dev = process.env.NODE_ENV !== 'production'; const app = next({ dev }); const handle = app.getRequestHandler(); const port = process.env.PORT || 3000; +const serverAdapter = new ExpressAdapter(); +serverAdapter.setBasePath('/admin/queues'); + +createBullBoard({ + queues: [new BullAdapter(downloadQueue), new BullAdapter(checkChaptersQueue)], + serverAdapter, +}); + (async () => { try { await app.prepare(); @@ -17,15 +28,6 @@ const port = process.env.PORT || 3000; return handle(req, res); }); - await watchLibrary({ - config: 'manup.yml', - generate: false, - help: false, - update: false, - library: '', - watch: true, - }); - server.listen(port, () => { logger.info(`> Ready on http://localhost:${port} - env ${process.env.NODE_ENV}`); }); diff --git a/src/server/queue/checkChapters.ts b/src/server/queue/checkChapters.ts new file mode 100644 index 0000000..c7374f8 --- /dev/null +++ b/src/server/queue/checkChapters.ts @@ -0,0 +1,101 @@ +import { Prisma } from '@prisma/client'; +import { Job, Queue, Worker } from 'bullmq'; +import path from 'path'; +import { logger } from '../../utils/logging'; +import { sanitizer } from '../../utils/sanitize'; +import { findMissingChapters } from '../utils/mangal'; +import { downloadQueue } from './download'; + +const cronMap = { + daily: '0 0 * * *', + hourly: '0 * * * *', + minutely: '* * * * *', + weekly: '0 * * * 7', +}; + +const mangaWithLibrary = Prisma.validator()({ + include: { Library: true }, +}); + +export type MangaWithLibrary = Prisma.MangaGetPayload; + +const checkChapters = async (manga: MangaWithLibrary) => { + logger.info(`Checking for new chapters: ${manga.title}`); + const mangaDir = path.resolve(manga.Library.path, sanitizer(manga.title)); + const missingChapters = await findMissingChapters(mangaDir, manga.source, manga.title); + + if (missingChapters.length === 0) { + logger.info(`There are no missing chapters for ${manga.title}`); + } else { + logger.info(`There are ${missingChapters.length} new chapters for ${manga.title}`); + } + + await Promise.all( + missingChapters.map(async (chapterIndex) => { + const job = await downloadQueue.getJob(`${sanitizer(manga.title)}_${chapterIndex - 1}_download`); + if (job) { + await job.remove(); + } + }), + ); + + await downloadQueue.addBulk( + missingChapters.map((chapterIndex) => ({ + opts: { + jobId: `${sanitizer(manga.title)}_${chapterIndex - 1}_download`, + }, + name: `${sanitizer(manga.title)}_${chapterIndex - 1}_download`, + data: { + chapterIndex: chapterIndex - 1, + source: manga.source, + title: manga.title, + libraryPath: manga.Library.path, + }, + })), + ); +}; + +export const checkChaptersQueue = new Queue('checkChaptersQueue', { + connection: { + host: 'localhost', + port: 6379, + }, +}); + +export const checkChaptersWorker = new Worker( + 'checkChaptersQueue', + async (job: Job) => { + const { manga }: { manga: MangaWithLibrary } = job.data; + await checkChapters(manga); + await job.updateProgress(100); + }, + { + concurrency: 5, + connection: { + host: 'localhost', + port: 6379, + }, + }, +); + +export const schedule = async (manga: MangaWithLibrary) => { + if (manga.interval === 'never') { + return; + } + + await checkChaptersQueue.add( + `check_${manga.title}_chapters`, + { + manga, + }, + { + jobId: `check_${manga.libraryId}_${manga.id}_chapters`, + repeatJobKey: `check_${manga.libraryId}_${manga.id}_chapters`, + repeat: { + pattern: cronMap[manga.interval as keyof typeof cronMap], + }, + }, + ); + + await checkChapters(manga); +}; diff --git a/src/server/queue/download.ts b/src/server/queue/download.ts new file mode 100644 index 0000000..21c5be3 --- /dev/null +++ b/src/server/queue/download.ts @@ -0,0 +1,47 @@ +import { Job, Queue, Worker } from 'bullmq'; +import { downloadChapter } from '../utils/mangal'; +import { sendNotification } from '../utils/notification'; + +interface IDownloadWorkerData { + chapterIndex: number; + source: string; + query: string; + title: string; + libraryPath: string; +} + +export const downloadWorker = new Worker( + 'downloadQueue', + async (job: Job) => { + const { chapterIndex, libraryPath, title, source }: IDownloadWorkerData = job.data; + try { + downloadChapter(title, source, chapterIndex, libraryPath); + } catch (err) { + await job.log(`${err}`); + throw err; + } + await sendNotification(`Downloaded a new chapter #${chapterIndex + 1} for ${title} from ${source}`); + await job.updateProgress(100); + }, + { + concurrency: 5, + connection: { + host: 'localhost', + port: 6379, + }, + }, +); + +export const downloadQueue = new Queue('downloadQueue', { + connection: { + host: 'localhost', + port: 6379, + }, + defaultJobOptions: { + attempts: 20, + backoff: { + type: 'fixed', + delay: 1000 * 60 * 2, + }, + }, +}); diff --git a/src/server/trpc/router/library.ts b/src/server/trpc/router/library.ts index 57b9e93..f7b9db2 100644 --- a/src/server/trpc/router/library.ts +++ b/src/server/trpc/router/library.ts @@ -1,5 +1,6 @@ +import path from 'path'; import { z } from 'zod'; -import { logger } from '../../../utils/logging'; +import { createLibrary } from '../../utils/mangal'; import { t } from '../trpc'; export const libraryRouter = t.router({ @@ -13,12 +14,13 @@ export const libraryRouter = t.router({ }), ) .mutation(async ({ input, ctx }) => { - logger.info(`input: ${JSON.stringify(input, null, 2)}`); + const libraryPath = path.resolve(process.cwd(), path.relative(process.cwd(), input.path)); const library = await ctx.prisma.library.create({ data: { - path: input.path, + path: libraryPath, }, }); + await createLibrary(libraryPath); return library; }), }); diff --git a/src/server/trpc/router/manga.ts b/src/server/trpc/router/manga.ts index 703d572..9aceb3e 100644 --- a/src/server/trpc/router/manga.ts +++ b/src/server/trpc/router/manga.ts @@ -1,6 +1,9 @@ import { TRPCError } from '@trpc/server'; +import path from 'path'; import { z } from 'zod'; -import { getAvailableSources, getMangaDetail, search } from '../../../utils/mangal'; +import { sanitizer } from '../../../utils/sanitize'; +import { schedule } from '../../queue/checkChapters'; +import { getAvailableSources, getMangaDetail, removeManga, search } from '../../utils/mangal'; import { t } from '../trpc'; export const mangaRouter = t.router({ @@ -23,14 +26,13 @@ export const mangaRouter = t.router({ detail: t.procedure .input( z.object({ - keyword: z.string().trim().min(1), source: z.string().trim().min(1), - order: z.number().gte(0), + title: z.string().trim().min(1), }), ) .query(async ({ input }) => { - const { keyword, source, order } = input; - return getMangaDetail(source, keyword, order.toString()); + const { title, source } = input; + return getMangaDetail(source, title); }), search: t.procedure .input( @@ -42,29 +44,49 @@ export const mangaRouter = t.router({ .query(async ({ input }) => { const { keyword, source } = input; const result = await search(source, keyword); - return result.Manga.map((m, i) => ({ + return result.Manga.map((m) => ({ status: m.Metadata.Status, title: m.Name, - order: i, cover: m.Metadata.Cover, })); }), + remove: t.procedure + .input( + z.object({ + id: z.number(), + }), + ) + .mutation(async ({ input, ctx }) => { + const { id } = input; + const removed = await ctx.prisma.manga.delete({ + include: { + Library: true, + }, + where: { + id, + }, + }); + const mangaPath = path.resolve(removed.Library.path, sanitizer(removed.title)); + await removeManga(mangaPath); + // TODO: remove jobs also + }), add: t.procedure .input( z.object({ - keyword: z.string().trim().min(1), source: z.string().trim().min(1), title: z.string().trim().min(1), interval: z.string().trim().min(1), - order: z.number().gte(0), }), ) .mutation(async ({ input, ctx }) => { - const { keyword, source, order, title, interval } = input; - const detail = await getMangaDetail(source, keyword, order.toString()); + const { source, title, interval } = input; + const detail = await getMangaDetail(source, title); const library = await ctx.prisma.library.findFirst(); if (!detail || !library) { - return undefined; + throw new TRPCError({ + code: 'NOT_FOUND', + message: `Cannot find the ${title}.`, + }); } const result = await ctx.prisma.manga.findFirst({ where: { @@ -78,15 +100,28 @@ export const mangaRouter = t.router({ }); } - return ctx.prisma.manga.create({ + if (detail.Name !== title) { + throw new TRPCError({ + code: 'NOT_FOUND', + message: `${title} does not match the found manga.`, + }); + } + + const manga = await ctx.prisma.manga.create({ + include: { + Library: true, + }, data: { cover: detail.Metadata.Cover, - query: keyword, source, title: detail.Name, - libraryId: library?.id, + libraryId: library.id, interval, }, }); + + schedule(manga); + + return manga; }), }); diff --git a/src/utils/mangal.ts b/src/server/utils/mangal.ts similarity index 62% rename from src/utils/mangal.ts rename to src/server/utils/mangal.ts index 1299cc2..829ef81 100644 --- a/src/utils/mangal.ts +++ b/src/server/utils/mangal.ts @@ -1,5 +1,6 @@ import execa from 'execa'; -import { logger } from './logging'; +import fs from 'fs/promises'; +import { logger } from '../../utils/logging'; export interface IOutput { Manga: Manga[]; @@ -54,9 +55,9 @@ export const getAvailableSources = async () => { return []; }; -export const search = async (source: string, query: string): Promise => { +export const search = async (source: string, keyword: string): Promise => { try { - const { stdout, command } = await execa('mangal', ['inline', '--source', source, '--query', query, '-j']); + const { stdout, command } = await execa('mangal', ['inline', '--source', source, '--query', keyword, '-j']); logger.info(`Search manga with following command: ${command}`); return JSON.parse(stdout); } catch (err) { @@ -68,16 +69,16 @@ export const search = async (source: string, query: string): Promise => }; }; -export const getChapters = async (source: string, query: string, order: string): Promise => { +export const getChapters = async (source: string, title: string): Promise => { try { const { stdout, command } = await execa('mangal', [ 'inline', '--source', source, '--query', - query, + title, '--manga', - order, + 'first', '--chapters', 'all', '-j', @@ -94,16 +95,16 @@ export const getChapters = async (source: string, query: string, order: string): return []; }; -export const getMangaDetail = async (source: string, query: string, order: string) => { +export const getMangaDetail = async (source: string, title: string) => { try { const { stdout, command } = await execa('mangal', [ 'inline', '--source', source, '--query', - query, + title, '--manga', - order, + 'first', '-j', ]); logger.info(`Get manga detail with following command: ${command}`); @@ -118,19 +119,12 @@ export const getMangaDetail = async (source: string, query: string, order: strin return undefined; }; -export const downloadChapter = async ( - title: string, - source: string, - query: string, - chapterIndex: number, - order: string, - libraryPath: string, -) => { +export const downloadChapter = async (title: string, source: string, chapterIndex: number, libraryPath: string) => { try { logger.info(`Downloading chapter #${chapterIndex} for ${title} from ${source}`); const { stdout, stderr, command } = await execa( 'mangal', - ['inline', '--source', source, '--query', query, '--manga', order, '--chapters', `${chapterIndex}`, '-d'], + ['inline', '--source', source, '--query', title, '--manga', 'first', '--chapters', `${chapterIndex}`, '-d'], { cwd: libraryPath, }, @@ -149,3 +143,36 @@ export const downloadChapter = async ( throw err; } }; + +export const findMissingChapters = async (mangaDir: string, source: string, title: string) => { + const sources = await getAvailableSources(); + if (sources.indexOf(source) < 0) { + logger.error(`Specified source: ${source} is not installed.`); + throw new Error(); + } + await fs.mkdir(mangaDir, { recursive: true }); + const titleFiles = await fs.readdir(mangaDir); + + const localChapters = titleFiles + .filter((chapter) => chapter.endsWith('cbz')) + .map((chapter) => { + const indexRegexp = /.*?\[(\d+)\].*/; + const match = indexRegexp.exec(chapter); + if (!match || match.length < 2 || !match[1]) { + return 1; + } + return parseInt(match[1], 10); + }) + .filter((index) => index !== -1); + + const remoteChapters = await getChapters(source, title); + return remoteChapters.filter((c) => !localChapters.includes(c)); +}; + +export const createLibrary = async (libraryPath: string) => { + await fs.mkdir(libraryPath, { recursive: true }); +}; + +export const removeManga = async (mangaDir: string) => { + await fs.rm(mangaDir, { recursive: true, force: true }); +}; diff --git a/src/server/downloader/notification.ts b/src/server/utils/notification.ts similarity index 100% rename from src/server/downloader/notification.ts rename to src/server/utils/notification.ts diff --git a/src/utils/sanitize.ts b/src/utils/sanitize.ts new file mode 100644 index 0000000..b14dfd4 --- /dev/null +++ b/src/utils/sanitize.ts @@ -0,0 +1,6 @@ +export const sanitizer = (value: string): string => { + return value + .replaceAll(/[\\/<>:;"'|?!*{}#%&^+,~\s]/g, '_') + .replaceAll(/__+/g, '_') + .replaceAll(/^[_\-.]+|[_\-.]+$/g, '_'); +};