feat: sync db with files

next.config.mjs

@@ -1,6 +1,11 @@
+import withBundleAnalyzer from '@next/bundle-analyzer';
 // eslint-disable-next-line @typescript-eslint/no-unused-vars
 import { env } from './src/env/server.mjs';
 
+const bundleAnalyzer = withBundleAnalyzer({
+  enabled: process.env.ANALYZE === 'true',
+});
+
 /**
  * Don't be scared of the generics here.
  * All they do is to give us autocompletion when using this.
@@ -10,7 +15,7 @@ import { env } from './src/env/server.mjs';
  * @constraint {{import('next').NextConfig}}
  */
 function defineNextConfig(config) {
-  return config;
+  return bundleAnalyzer(config);
 }
 
 export default defineNextConfig({

package.json

@@ -9,7 +9,8 @@
     "start:server": "NODE_ENV=production node dist/server/index.js",
     "dev": "next dev",
     "build:next": "next build",
-    "build": "npm run build:next && npm run build:server",
+    "build": "yarn run build:next && yarn run build:server",
+    "build:analyze": "ANALYZE=true yarn run build:next && ANALYZE=false yarn run build:server",
     "start": "next start",
     "lint": "next lint",
     "postinstall": "prisma generate"
@@ -43,6 +44,7 @@
     "@mantine/notifications": "^5.5.4",
     "@mantine/nprogress": "^5.5.4",
     "@mantine/spotlight": "^5.5.4",
+    "@next/bundle-analyzer": "^12.3.1",
     "@prisma/client": "4.4.0",
     "@tabler/icons": "^1.101.0",
     "@tanstack/react-query": "^4.9.0",

prisma/migrations/20221015131156_init/migration.sql

@@ -0,0 +1,32 @@
+-- CreateTable
+CREATE TABLE "Library" (
+    "id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
+    "path" TEXT NOT NULL
+);
+
+-- CreateTable
+CREATE TABLE "Manga" (
+    "id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
+    "title" TEXT NOT NULL,
+    "cover" TEXT NOT NULL,
+    "interval" TEXT NOT NULL,
+    "source" TEXT NOT NULL,
+    "libraryId" INTEGER NOT NULL,
+    CONSTRAINT "Manga_libraryId_fkey" FOREIGN KEY ("libraryId") REFERENCES "Library" ("id") ON DELETE RESTRICT ON UPDATE CASCADE
+);
+
+-- CreateTable
+CREATE TABLE "Chapter" (
+    "id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
+    "index" INTEGER NOT NULL,
+    "fileName" TEXT NOT NULL,
+    "size" INTEGER NOT NULL,
+    "mangaId" INTEGER NOT NULL,
+    CONSTRAINT "Chapter_mangaId_fkey" FOREIGN KEY ("mangaId") REFERENCES "Manga" ("id") ON DELETE RESTRICT ON UPDATE CASCADE
+);
+
+-- CreateIndex
+CREATE UNIQUE INDEX "Library_path_key" ON "Library"("path");
+
+-- CreateIndex
+CREATE UNIQUE INDEX "Manga_title_key" ON "Manga"("title");

prisma/schema.prisma

@@ -13,26 +13,26 @@ datasource db {
 
 model Library {
   id     Int     @id @default(autoincrement())
-  path   String
+  path   String  @unique
   mangas Manga[]
 }
 
 model Manga {
   id        Int       @id @default(autoincrement())
-  title     String
+  title     String    @unique
   cover     String
   interval  String
   source    String
-  Library   Library   @relation(fields: [libraryId], references: [id])
+  library   Library   @relation(fields: [libraryId], references: [id])
   libraryId Int
-  Chapter   Chapter[]
+  chapter   Chapter[]
 }
 
 model Chapter {
   id       Int    @id @default(autoincrement())
   index    Int
   fileName String
-  size     BigInt
+  size     Int
   manga    Manga  @relation(fields: [mangaId], references: [id])
   mangaId  Int
 }

src/pages/manga/[id].tsx

@@ -19,5 +19,9 @@ export default function LibraryPage() {
     return <LoadingOverlay visible overlayBlur={2} />;
   }
 
+  if (mangaQuery.isError) {
+    router.push('/404');
+  }
+
   return <Code>{JSON.stringify(mangaQuery.data, null, 2)}</Code>;
 }

src/server/queue/checkChapters.ts

@@ -1,9 +1,9 @@
-import { Prisma } from '@prisma/client';
+import { Prisma, PrismaClient } from '@prisma/client';
 import { Job, Queue, Worker } from 'bullmq';
 import path from 'path';
 import { logger } from '../../utils/logging';
 import { sanitizer } from '../../utils/sanitize';
-import { findMissingChapters } from '../utils/mangal';
+import { findMissingChapterFiles, getChaptersFromLocal } from '../utils/mangal';
 import { downloadQueue } from './download';
 
 const cronMap = {
@@ -13,43 +13,80 @@ const cronMap = {
   weekly: '0 * * * 7',
 };
 
+const prisma = new PrismaClient();
+
 const mangaWithLibrary = Prisma.validator<Prisma.MangaArgs>()({
-  include: { Library: true },
+  include: { library: true },
 });
 
 export type MangaWithLibrary = Prisma.MangaGetPayload<typeof mangaWithLibrary>;
 
 const checkChapters = async (manga: MangaWithLibrary) => {
   logger.info(`Checking for new chapters: ${manga.title}`);
-  const mangaDir = path.resolve(manga.Library.path, sanitizer(manga.title));
-  const missingChapters = await findMissingChapters(mangaDir, manga.source, manga.title);
+  const mangaDir = path.resolve(manga.library.path, sanitizer(manga.title));
+  const missingChapterFiles = await findMissingChapterFiles(mangaDir, manga.source, manga.title);
+
+  if (missingChapterFiles.length === 0) {
+    logger.info(`There are no missing chapter files for ${manga.title}`);
+
+    const localChapters = await getChaptersFromLocal(mangaDir);
+
+    await prisma.chapter.deleteMany({
+      where: {
+        mangaId: manga.id,
+        index: {
+          notIn: localChapters.map((chapter) => chapter.index),
+        },
+        fileName: {
+          notIn: localChapters.map((chapter) => chapter.fileName),
+        },
+      },
+    });
+
+    const dbChapters = await prisma.chapter.findMany({
+      where: {
+        mangaId: manga.id,
+      },
+    });
+
+    const missingDbChapters = localChapters.filter(
+      (localChapter) => dbChapters.findIndex((dbChapter) => dbChapter.index === localChapter.index) < 0,
+    );
+
+    await Promise.all(
+      missingDbChapters.map(async (chapter) => {
+        return prisma.chapter.create({
+          data: {
+            ...chapter,
+            mangaId: manga.id,
+          },
+        });
+      }),
+    );
 
-  if (missingChapters.length === 0) {
-    logger.info(`There are no missing chapters for ${manga.title}`);
-  } else {
-    logger.info(`There are ${missingChapters.length} new chapters for ${manga.title}`);
+    return;
   }
 
+  logger.info(`There are ${missingChapterFiles.length} new chapters for ${manga.title}`);
+
   await Promise.all(
-    missingChapters.map(async (chapterIndex) => {
-      const job = await downloadQueue.getJob(`${sanitizer(manga.title)}_${chapterIndex - 1}_download`);
+    missingChapterFiles.map(async (chapterIndex) => {
+      const job = await downloadQueue.getJob(`${sanitizer(manga.title)}_${chapterIndex}_download`);
       if (job) {
         await job.remove();
       }
     }),
   );
 
   await downloadQueue.addBulk(
-    missingChapters.map((chapterIndex) => ({
+    missingChapterFiles.map((chapterIndex) => ({
       opts: {
-        jobId: `${sanitizer(manga.title)}_${chapterIndex - 1}_download`,
+        jobId: `${sanitizer(manga.title)}_${chapterIndex}_download`,
       },
-      name: `${sanitizer(manga.title)}_chapter#${chapterIndex - 1}_download`,
+      name: `${sanitizer(manga.title)}_chapter#${chapterIndex}_download`,
       data: {
-        chapterIndex: chapterIndex - 1,
-        source: manga.source,
-        title: manga.title,
-        libraryPath: manga.Library.path,
+        manga,
+        chapterIndex,
       },
     })),
   );

src/server/queue/download.ts

@@ -1,27 +1,43 @@
+import { PrismaClient } from '@prisma/client';
 import { Job, Queue, Worker } from 'bullmq';
-import { downloadChapter } from '../utils/mangal';
+import { downloadChapter, getChapterFromLocal } from '../utils/mangal';
 import { sendNotification } from '../utils/notification';
+import type { MangaWithLibrary } from './checkChapters';
 
 interface IDownloadWorkerData {
+  manga: MangaWithLibrary;
   chapterIndex: number;
-  source: string;
-  query: string;
-  title: string;
-  libraryPath: string;
 }
 
+const prisma = new PrismaClient();
+
 export const downloadWorker = new Worker(
   'downloadQueue',
   async (job: Job) => {
-    const { chapterIndex, libraryPath, title, source }: IDownloadWorkerData = job.data;
+    const { chapterIndex, manga }: IDownloadWorkerData = job.data;
     try {
-      downloadChapter(title, source, chapterIndex, libraryPath);
+      const filePath = await downloadChapter(manga.title, manga.source, chapterIndex, manga.library.path);
+      const chapter = await getChapterFromLocal(filePath);
+
+      await prisma.chapter.deleteMany({
+        where: {
+          mangaId: manga.id,
+          index: chapterIndex,
+        },
+      });
+
+      await prisma.chapter.create({
+        data: {
+          ...chapter,
+          mangaId: manga.id,
+        },
+      });
+      await sendNotification(`Downloaded a new chapter #${chapterIndex + 1} for ${manga.title} from ${manga.source}`);
+      await job.updateProgress(100);
     } catch (err) {
       await job.log(`${err}`);
       throw err;
     }
-    await sendNotification(`Downloaded a new chapter #${chapterIndex + 1} for ${title} from ${source}`);
-    await job.updateProgress(100);
   },
   {
     concurrency: 5,

src/server/trpc/router/manga.ts

@@ -42,7 +42,7 @@ export const mangaRouter = t.router({
     )
     .query(async ({ input, ctx }) => {
       const { id } = input;
-      return ctx.prisma.manga.findFirst({ where: { id } });
+      return ctx.prisma.manga.findUniqueOrThrow({ include: { chapter: true, library: true }, where: { id } });
     }),
   search: t.procedure
     .input(
@@ -70,13 +70,13 @@ export const mangaRouter = t.router({
       const { id } = input;
       const removed = await ctx.prisma.manga.delete({
         include: {
-          Library: true,
+          library: true,
         },
         where: {
           id,
         },
       });
-      const mangaPath = path.resolve(removed.Library.path, sanitizer(removed.title));
+      const mangaPath = path.resolve(removed.library.path, sanitizer(removed.title));
       await removeManga(mangaPath);
       await removeJob(removed.title);
     }),
@@ -119,7 +119,7 @@ export const mangaRouter = t.router({
 
       const manga = await ctx.prisma.manga.create({
         include: {
-          Library: true,
+          library: true,
         },
         data: {
           cover: detail.Metadata.Cover,