Merge pull request #14 from yuki-wtf/save-whitelist-by-chunks

Save whitelist proofs to db by chunks

.gitignore

@@ -137,4 +137,6 @@ dmypy.json
 node_modules/
 
 certificates 
-*.crt
+*.crt
+
+*/whitelist/*.json

indexer/src/entity/whitelist.ts

@@ -6,7 +6,7 @@ export class Whitelist {
     @PrimaryColumn({type: "integer", unique: true})
     generation!: number;
 
-    @Column()
+    @Column({type: 'text'})
     proof!: string;
 
     @CreateDateColumn()

indexer/src/indexer.ts

@@ -276,7 +276,7 @@ const updatePendingMints = async () => {
 
 const wait = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
 export const indexer = async () => {
-  await checkWhitelistProofs();
+  // await checkWhitelistProofs();
   try {
     while (true) {
       await pullEvents();

indexer/src/migrations/1707472656807-migration.ts

@@ -0,0 +1,16 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+export class migration1707472656807 implements MigrationInterface {
+    name = 'migration1707472656807'
+
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`CREATE TABLE "whitelist" ("generation" integer NOT NULL, "proof" text NOT NULL, "createdAt" TIMESTAMP NOT NULL DEFAULT now(), CONSTRAINT "PK_365e5cf79f9365dc5eb3c4e4c25" PRIMARY KEY ("generation"))`);
+        await queryRunner.query(`CREATE INDEX "IDX_365e5cf79f9365dc5eb3c4e4c2" ON "whitelist" ("generation") `);
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`DROP INDEX "public"."IDX_365e5cf79f9365dc5eb3c4e4c2"`);
+        await queryRunner.query(`DROP TABLE "whitelist"`);
+    }
+
+}

indexer/src/utils/checkWhitelistProofs.ts

@@ -1,17 +1,14 @@
-import { AppDataSource } from "./db"
+import fs from 'fs/promises'
 import { saveWhitelistProofsFromFileToDB } from "./saveWhitelistProofsFromFileToDB"
 
 export const checkWhitelistProofs = async () => {
-    const res = await AppDataSource.query(`SELECT EXISTS(SELECT 1 FROM whitelist LIMIT 1) AS isNotEmpty;`)
-    console.log('resres', res)
-    const isNotEmpty = res[0]?.isnotempty
-    if(isNotEmpty) {
-        console.log('Whitelist proofs already exist. Skipping...')
-        return
-    }
-
     try {
-        await saveWhitelistProofsFromFileToDB()
+        const whitelistFiles = await fs.readdir('whitelist')
+        whitelistFiles.forEach(async (filename) => {
+            console.log(`Saving ${filename} ...`)
+            await saveWhitelistProofsFromFileToDB(`whitelist/${filename}`)
+            console.log(`${filename} saved to DB.`)
+        })
     }catch(err) {
         console.error('Error saving whitelist proofs to DB', err)
     }   

indexer/src/utils/saveWhitelistProofsFromFileToDB.ts

@@ -3,11 +3,9 @@ import { Whitelist } from "../entity/whitelist";
 import { AppDataSource } from './db';
 import { exit } from 'process';
 
-const filename = "whitelist.json";
-
-export const saveWhitelistProofsFromFileToDB = async () => {
+export const saveWhitelistProofsFromFileToDB = async (filename:string) => {
   console.log(`Retrieving whitelist proofs from "${filename}" file.`);
-  const dataRaw = await fs.readFile("whitelist.json", "utf8");
+  const dataRaw = await fs.readFile(filename, "utf8");
   const data = JSON.parse(dataRaw);
 
   const whitelist = [];
@@ -24,7 +22,6 @@ export const saveWhitelistProofsFromFileToDB = async () => {
     }
     await AppDataSource.manager.save(whitelist);
     console.log("Data inserted successfully.");
-    exit(0);
   } catch (err) {
     console.error("Error inserting data", err);
     exit(1);