feature/redis (#179)

* add redis for getting transaction details

* add cache for some calculation and revert cache in db

* cache get user in middleware

* cache messages api

* remove unused imports

* fix cache key in messages api

server/.env-example

@@ -24,4 +24,6 @@ MIDTRANS_MERCHANT_ID=
 MIDTRANS_CLIENT_KEY=
 MIDTRANS_SERVER_KEY=
 
+REDIS_URI=
+
 IS_MAINTENANCE=

server/ormconfig.js

@@ -29,5 +29,11 @@ module.exports = {
   cli: {
     'migrationsDir': 'src/migrations'
   },
-  namingStrategy: new SnakeNamingStrategy()
+  namingStrategy: new SnakeNamingStrategy(),
+  // ...process.env.REDIS_URI ? {
+  //   cache: {
+  //     type: 'redis',
+  //     options: process.env.REDIS_URI
+  //   }
+  // } : {}
 }

server/package.json

@@ -31,6 +31,7 @@
     "geoip-lite": "^1.4.2",
     "glob": "^7.1.7",
     "input": "^1.0.1",
+    "ioredis": "^4.28.2",
     "is-uuid": "^1.0.2",
     "json-bigint": "^1.0.0",
     "jsonwebtoken": "^8.5.1",
@@ -41,6 +42,7 @@
     "node-mailjet": "^3.3.4",
     "pg": "^8.7.1",
     "rate-limiter-flexible": "^2.3.1",
+    "redis": "^4.0.1",
     "serialize-error": "^8.1.0",
     "source-map-support": "^0.5.19",
     "typeorm": "^0.2.37",
@@ -62,13 +64,15 @@
     "@types/express-rate-limit": "^5.1.3",
     "@types/geoip-lite": "^1.4.1",
     "@types/glob": "^7.1.4",
+    "@types/ioredis": "^4.28.7",
     "@types/is-uuid": "^1.0.0",
     "@types/jsonwebtoken": "^8.5.5",
     "@types/morgan": "^1.9.3",
     "@types/multer": "^1.4.7",
     "@types/nanoid": "^3.0.0",
     "@types/node": "^16.7.2",
     "@types/pg": "^8.6.1",
+    "@types/redis": "^4.0.11",
     "@types/serialize-error": "^4.0.1",
     "@types/source-map-support": "^0.5.4",
     "@typescript-eslint/eslint-plugin": "^4.29.3",
@@ -79,4 +83,4 @@
     "rimraf": "^3.0.2",
     "typescript": "^4.4.2"
   }
-}
+}

server/src/api/middlewares/Auth.ts

@@ -31,11 +31,11 @@ export async function Auth(req: Request, _: Response, next: NextFunction): Promi
     userAuth = await req.tg.getMe()
   } catch (error) {
     try {
-      await new Promise((resolve) => setTimeout(resolve, 1000))
+      await new Promise((resolve) => setTimeout(resolve, 2000))
       await req.tg.connect()
       userAuth = await req.tg.getMe()
     } catch (error) {
-      await new Promise((resolve) => setTimeout(resolve, 1000))
+      await new Promise((resolve) => setTimeout(resolve, 2000))
       await req.tg.connect()
       userAuth = await req.tg.getMe()
     }
@@ -69,7 +69,21 @@ export async function AuthMaybe(req: Request, _: Response, next: NextFunction):
     }
 
     await req.tg.connect()
-    const userAuth = await req.tg.getMe()
+    let userAuth: any
+    try {
+      await req.tg.connect()
+      userAuth = await req.tg.getMe()
+    } catch (error) {
+      try {
+        await new Promise((resolve) => setTimeout(resolve, 2000))
+        await req.tg.connect()
+        userAuth = await req.tg.getMe()
+      } catch (error) {
+        await new Promise((resolve) => setTimeout(resolve, 2000))
+        await req.tg.connect()
+        userAuth = await req.tg.getMe()
+      }
+    }
 
     const user = await Users.findOne({ tg_id: userAuth['id'].toString() })
     if (!user) {

server/src/api/v1/Dialogs.ts

@@ -1,6 +1,7 @@
 import { Api } from '@mgilangjanuar/telegram'
 import bigInt from 'big-integer'
 import { Request, Response } from 'express'
+import { Redis } from '../../service/Cache'
 import { objectParser } from '../../utils/ObjectParser'
 import { Endpoint } from '../base/Endpoint'
 import { Auth } from '../middlewares/Auth'
@@ -11,12 +12,14 @@ export class Dialogs {
   @Endpoint.GET('/', { middlewares: [Auth] })
   public async find(req: Request, res: Response): Promise<any> {
     const { offset, limit } = req.query
-    const dialogs = await req.tg.getDialogs({
-      limit: Number(limit) || 0,
-      offsetDate: Number(offset) || undefined,
-      ignorePinned: false
-    })
-    return res.send({ dialogs: objectParser(dialogs) })
+    const dialogs = await Redis.connect().getFromCacheFirst(`dialogs:${req.user.id}:${JSON.stringify(req.query)}`, async () => {
+      return objectParser(await req.tg.getDialogs({
+        limit: Number(limit) || 0,
+        offsetDate: Number(offset) || undefined,
+        ignorePinned: false
+      }))
+    }, 2)
+    return res.send({ dialogs })
   }
 
   @Endpoint.GET('/:type/:id', { middlewares: [Auth] })

server/src/api/v1/Github.ts

@@ -1,5 +1,6 @@
 import axios from 'axios'
 import { Request, Response } from 'express'
+import { Redis } from '../../service/Cache'
 import { Endpoint } from '../base/Endpoint'
 
 @Endpoint.API()
@@ -10,12 +11,18 @@ export class Github {
     if (!process.env.GITHUB_TOKEN) {
       throw { status: 400, body: { error: 'Token is unavailable' } }
     }
-    const { data: collaborators } = await axios.get('https://api.github.com/repos/mgilangjanuar/teledrive/collaborators', {
-      headers: { authorization: `Bearer ${process.env.GITHUB_TOKEN}` }
-    })
-    const { data: contributors } = await axios.get('https://api.github.com/repos/mgilangjanuar/teledrive/contributors', {
-      headers: { authorization: `Bearer ${process.env.GITHUB_TOKEN}` }
-    })
+    const { data: collaborators } = await Redis.connect().getFromCacheFirst('github:collaborators', async () => {
+      const resp = await axios.get('https://api.github.com/repos/mgilangjanuar/teledrive/collaborators', {
+        headers: { authorization: `Bearer ${process.env.GITHUB_TOKEN}` }
+      })
+      return { data: resp.data }
+    }, 21600)
+    const { data: contributors } = await Redis.connect().getFromCacheFirst('github:contributors', async () => {
+      const resp = await axios.get('https://api.github.com/repos/mgilangjanuar/teledrive/contributors', {
+        headers: { authorization: `Bearer ${process.env.GITHUB_TOKEN}` }
+      })
+      return { data: resp.data }
+    }, 21600)
     return res.send({ contributors: [
       ...contributors, ...collaborators.filter((col: any) => !contributors.find((con: any) => con.login === col.login))
     ] })

server/src/api/v1/Messages.ts

@@ -1,6 +1,7 @@
 import { Api } from '@mgilangjanuar/telegram'
 import bigInt from 'big-integer'
 import { Request, Response } from 'express'
+import { Redis } from '../../service/Cache'
 import { Endpoint } from '../base/Endpoint'
 import { Auth } from '../middlewares/Auth'
 
@@ -27,13 +28,16 @@ export class Messages {
         accessHash: bigInt(accessHash as string) })
     }
 
-    const messages = await req.tg.invoke(new Api.messages.GetHistory({
-      peer: peer,
-      limit: Number(limit) || 0,
-      offsetId: Number(offset) || 0,
-    }))
-    const result = JSON.parse(JSON.stringify(messages))
-    result.messages = result.messages?.map((msg, i) => ({ ...msg, action: { ...msg.action, className: messages['messages'][i]?.action?.className } }))
+    const result = await Redis.connect().getFromCacheFirst(`history:${req.user.id}:${JSON.stringify(req.params)}:${JSON.stringify(req.query)}`, async () => {
+      const messages = await req.tg.invoke(new Api.messages.GetHistory({
+        peer: peer,
+        limit: Number(limit) || 0,
+        offsetId: Number(offset) || 0,
+      }))
+      const result = JSON.parse(JSON.stringify(messages))
+      result.messages = result.messages?.map((msg, i) => ({ ...msg, action: { ...msg.action, className: messages['messages'][i]?.action?.className } }))
+      return result
+    }, 2)
     return res.send({ messages: result })
   }
 

server/src/api/v1/Users.ts

@@ -5,6 +5,7 @@ import moment from 'moment'
 import { Files } from '../../model/entities/Files'
 import { Usages } from '../../model/entities/Usages'
 import { Users as Model } from '../../model/entities/Users'
+import { Redis } from '../../service/Cache'
 import { Midtrans, TransactionDetails } from '../../service/Midtrans'
 import { PayPal, SubscriptionDetails } from '../../service/PayPal'
 import { buildSort, buildWhereQuery } from '../../utils/FilterQuery'
@@ -64,20 +65,22 @@ export class Users {
 
     if (username === 'me' || username === req.user.username) {
       const username = req.userAuth.username || req.userAuth.phone
-      let paymentDetails: SubscriptionDetails | TransactionDetails = null
+
+      let paymentDetails: SubscriptionDetails = null, midtransPaymentDetails: TransactionDetails = null
+
       if (req.user.subscription_id) {
         try {
-          paymentDetails = await new PayPal().getSubscription(req.user.subscription_id)
+          paymentDetails = await Redis.connect().getFromCacheFirst(`paypal:subscription:${req.user.subscription_id}`, async () => await new PayPal().getSubscription(req.user.subscription_id), 21600)
         } catch (error) {
           // ignore
         }
       }
 
       if (req.user.midtrans_id) {
         try {
-          paymentDetails = await new Midtrans().getTransactionStatus(req.user.midtrans_id)
-          if (!paymentDetails?.transaction_status) {
-            paymentDetails = null
+          midtransPaymentDetails = await Redis.connect().getFromCacheFirst(`midtrans:transaction:${req.user.midtrans_id}`, async () => await new Midtrans().getTransactionStatus(req.user.midtrans_id), 21600)
+          if (!midtransPaymentDetails?.transaction_status) {
+            midtransPaymentDetails = null
           }
         } catch (error) {
           // ignore
@@ -86,17 +89,17 @@ export class Users {
 
       let plan: 'free' | 'premium' = 'free'
 
-      if (paymentDetails) {
-        if (req.user.subscription_id && (paymentDetails as SubscriptionDetails).plan_id === process.env.PAYPAL_PLAN_PREMIUM_ID) {
-          const isExpired = new Date().getTime() - new Date((paymentDetails as SubscriptionDetails).billing_info?.last_payment.time).getTime() > 3.154e+10
-          if ((paymentDetails as SubscriptionDetails).billing_info?.last_payment && !isExpired || ['APPROVED', 'ACTIVE'].includes((paymentDetails as SubscriptionDetails).status)) {
-            plan = 'premium'
-          }
-        } else if (req.user.midtrans_id && ((paymentDetails as TransactionDetails).settlement_time || (paymentDetails as TransactionDetails).transaction_time)) {
-          const isExpired = new Date().getTime() - new Date((paymentDetails as TransactionDetails).settlement_time || (paymentDetails as TransactionDetails).transaction_time).getTime() > 3.154e+10
-          if (['settlement', 'capture'].includes((paymentDetails as TransactionDetails).transaction_status) && !isExpired) {
-            plan = 'premium'
-          }
+      if (paymentDetails && paymentDetails.plan_id === process.env.PAYPAL_PLAN_PREMIUM_ID) {
+        const isExpired = new Date().getTime() - new Date(paymentDetails.billing_info?.last_payment.time).getTime() > 3.154e+10
+        if (paymentDetails.billing_info?.last_payment && !isExpired || ['APPROVED', 'ACTIVE'].includes(paymentDetails.status)) {
+          plan = 'premium'
+        }
+      }
+
+      if (midtransPaymentDetails && (midtransPaymentDetails.settlement_time || midtransPaymentDetails.transaction_time)) {
+        const isExpired = new Date().getTime() - new Date(midtransPaymentDetails.settlement_time || midtransPaymentDetails.transaction_time).getTime() > 3.154e+10
+        if (['settlement', 'capture'].includes(midtransPaymentDetails.transaction_status) && !isExpired) {
+          plan = 'premium'
         }
       }
 

server/src/api/v1/Utils.ts

@@ -3,6 +3,7 @@ import { readFileSync } from 'fs'
 import { lookup } from 'geoip-lite'
 import { Files } from '../../model/entities/Files'
 import { Users } from '../../model/entities/Users'
+import { Redis } from '../../service/Cache'
 import { Endpoint } from '../base/Endpoint'
 
 @Endpoint.API()
@@ -25,12 +26,13 @@ export class Utils {
 
   @Endpoint.GET()
   public async simpleAnalytics(_: Request, res: Response): Promise<any> {
-    return res.send({
-      analytics: {
+    const analytics = await Redis.connect().getFromCacheFirst('simpleAnalytics', async () => {
+      return {
         users: await Users.count(),
         files: await Files.count(),
         premiumUsers: await Users.count({ where: { plan: 'premium' } }),
       }
-    })
+    }, 3600)
+    return res.send({ analytics })
   }
 }

server/src/index.ts

@@ -22,6 +22,7 @@ import * as Sentry from '@sentry/node'
 import * as Tracing from '@sentry/tracing'
 import { API } from './api'
 import { runDB } from './model'
+import { Redis } from './service/Cache'
 
 // import bigInt from 'json-bigint'
 // const parse = JSON.parse
@@ -49,6 +50,7 @@ import { runDB } from './model'
 // }
 
 
+Redis.connect()
 runDB()
 
 const app = express()

server/src/model/index.ts

@@ -55,7 +55,13 @@ export const runDB = async (): Promise<void> => {
     cli: {
       'migrationsDir': 'src/migrations'
     },
-    namingStrategy: new SnakeNamingStrategy()
+    namingStrategy: new SnakeNamingStrategy(),
+    // ...process.env.REDIS_URI ? {
+    //   cache: {
+    //     type: 'redis',
+    //     options: process.env.REDIS_URI
+    //   }
+    // } : {}
   }, BaseModel).build()
 }
 

server/src/service/Cache.ts

@@ -0,0 +1,55 @@
+import IORedis, { Redis as IOredis } from 'ioredis'
+
+export class Redis {
+  private static client: Redis
+  private redis: IOredis
+
+  private constructor() {
+    if (!process.env.REDIS_URI) {
+      throw new Error('REDIS_URI is not defined')
+    }
+    this.redis = new IORedis(process.env.REDIS_URI)
+  }
+
+  public static connect(): Redis {
+    if (!this.client) {
+      this.client = new Redis()
+    }
+    return this.client
+  }
+
+  public async get(key: string): Promise<any> {
+    const result = await this.redis.get(key)
+    if (!result) return null
+    try {
+      return JSON.parse(result)
+    } catch (error) {
+      return result
+    }
+  }
+
+  public async set(key: string, data: unknown, ex?: number): Promise<boolean> {
+    try {
+      if (ex) {
+        return await this.redis.set(key, JSON.stringify(data), 'EX', ex) === 'OK'
+      } else {
+        return await this.redis.set(key, JSON.stringify(data)) === 'OK'
+      }
+    } catch (error) {
+      if (ex) {
+        return await this.redis.set(key, data as any, 'EX', ex) === 'OK'
+      } else {
+        return await this.redis.set(key, data as any) === 'OK'
+      }
+    }
+  }
+
+  public async getFromCacheFirst<T>(key: string, fn: () => T | Promise<T>, ex?: number): Promise<T> {
+    const result = await this.get(key)
+    if (result) return result
+
+    const data = await fn()
+    await this.set(key, data, ex)
+    return data
+  }
+}