From 43476dcee0b7f5f929d86441e9b4f1c830c4ed81 Mon Sep 17 00:00:00 2001 From: dev-737 <73829355+dev-737@users.noreply.github.com> Date: Fri, 24 May 2024 14:58:40 +0530 Subject: [PATCH] feat: delete connections after 24h of inactivity (#70) * feat: delete connections after 24h of inactivity * fix: JS-0327 and JS-0116 * final touchups and testing done * rename files * chore: fix condition for running tasks in production build * hi * chore: update recurring task interval for syncing botlist stats * feat: disconnect idle connections after 24 hours --- prisma/schema.prisma | 14 ++-- src/cluster.ts | 38 ++++++----- src/core/Client.ts | 9 +++ src/index.ts | 4 +- src/managers/EventManager.ts | 23 ++++--- src/scripts/network/components.ts | 4 +- src/scripts/network/storeMessageData.ts | 15 ++-- src/scripts/tasks/deleteOldMessages.ts | 15 ---- src/scripts/tasks/pauseIdleConnections.ts | 68 +++++++++++++++++++ src/services/SchedulerService.ts | 5 ++ src/utils/ConnectedList.ts | 14 +++- src/utils/JSON/emojis.json | 3 +- ...ReactionUpdater.ts => RandomComponents.ts} | 25 ++++++- src/utils/Utils.ts | 2 +- 14 files changed, 175 insertions(+), 64 deletions(-) delete mode 100644 src/scripts/tasks/deleteOldMessages.ts create mode 100644 src/scripts/tasks/pauseIdleConnections.ts rename src/utils/{ReactionUpdater.ts => RandomComponents.ts} (88%) diff --git a/prisma/schema.prisma b/prisma/schema.prisma index c9a82570..0be2a391 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -75,8 +75,8 @@ model blacklistedServers { } model connectedList { - id String @id @default(auto()) @map("_id") @db.ObjectId - channelId String @unique // channel can be thread, or a normal channel + id String @id @default(auto()) @map("_id") @db.ObjectId + channelId String @unique // channel can be thread, or a normal channel parentId String? // ID of the parent channel, if it's a thread @map("parentChannelId") serverId String connected Boolean @@ -85,9 +85,10 @@ model connectedList { profFilter Boolean embedColor String? webhookURL String - date DateTime @default(now()) - hub hubs? @relation(fields: [hubId], references: [id]) - hubId String @db.ObjectId + lastActive DateTime? @default(now()) + date DateTime @default(now()) + hub hubs? @relation(fields: [hubId], references: [id]) + hubId String @db.ObjectId } model hubs { @@ -125,6 +126,7 @@ model originalMessages { serverId String authorId String reactions Json? // eg. {"👎": ["9893820930928", "39283902803982"]} "emoji": userId[] basically + createdAt DateTime broadcastMsgs broadcastedMessages[] // Ids of messages that were broadcasted to other hubs messageReference String? @db.String // id of the original message this message is replying to hub hubs? @relation(fields: [hubId], references: [id]) @@ -134,7 +136,7 @@ model originalMessages { model broadcastedMessages { messageId String @id @map("_id") channelId String - createdAt Int + createdAt DateTime originalMsg originalMessages @relation(fields: [originalMsgId], references: [messageId]) originalMsgId String @db.String } diff --git a/src/cluster.ts b/src/cluster.ts index 435d6699..b9bd479d 100644 --- a/src/cluster.ts +++ b/src/cluster.ts @@ -1,34 +1,38 @@ import db from './utils/Db.js'; +import Logger from './utils/Logger.js'; import Scheduler from './services/SchedulerService.js'; +import syncBotlistStats from './scripts/tasks/syncBotlistStats.js'; +import updateBlacklists from './scripts/tasks/updateBlacklists.js'; +import deleteExpiredInvites from './scripts/tasks/deleteExpiredInvites.js'; +import pauseIdleConnections from './scripts/tasks/pauseIdleConnections.js'; import { startApi } from './api/index.js'; import { isDevBuild } from './utils/Constants.js'; import { VoteManager } from './managers/VoteManager.js'; import { ClusterManager } from 'discord-hybrid-sharding'; -import syncBotlistStats from './scripts/tasks/syncBotlistStats.js'; -import deleteExpiredInvites from './scripts/tasks/deleteExpiredInvites.js'; -import updateBlacklists from './scripts/tasks/updateBlacklists.js'; -import deleteOldMessages from './scripts/tasks/deleteOldMessages.js'; -import 'dotenv/config'; import { getUsername, wait } from './utils/Utils.js'; -import Logger from './utils/Logger.js'; +import 'dotenv/config'; const clusterManager = new ClusterManager('build/index.js', { token: process.env.TOKEN, shardsPerClusters: 2, + totalClusters: 'auto', }); clusterManager.on('clusterCreate', async (cluster) => { // if it is the last cluster if (cluster.id === clusterManager.totalClusters - 1) { const scheduler = new Scheduler(); + // remove expired blacklists or set new timers for them - const serverQuery = { where: { hubs: { some: { expires: { isSet: true } } } } }; - const userQuery = { where: { blacklistedFrom: { some: { expires: { isSet: true } } } } }; - updateBlacklists(await db.blacklistedServers.findMany(serverQuery), scheduler).catch( - Logger.error, - ); + const serverQuery = await db.blacklistedServers.findMany({ + where: { hubs: { some: { expires: { isSet: true } } } }, + }); + const userQuery = await db.userData.findMany({ + where: { blacklistedFrom: { some: { expires: { isSet: true } } } }, + }); - updateBlacklists(await db.userData.findMany(userQuery), scheduler).catch(Logger.error); + updateBlacklists(serverQuery, scheduler).catch(Logger.error); + updateBlacklists(userQuery, scheduler).catch(Logger.error); // code must be in production to run these tasks if (isDevBuild) return; @@ -37,12 +41,14 @@ clusterManager.on('clusterCreate', async (cluster) => { // perform start up tasks syncBotlistStats(clusterManager).catch(Logger.error); - deleteOldMessages().catch(Logger.error); deleteExpiredInvites().catch(Logger.error); + pauseIdleConnections(clusterManager).catch(Logger.error); - scheduler.addRecurringTask('deleteExpiredInvites', 60 * 60 * 1_000, deleteExpiredInvites); - scheduler.addRecurringTask('deleteOldMessages', 60 * 60 * 12_000, deleteOldMessages); - scheduler.addRecurringTask('syncBotlistStats', 60 * 10_000, () => + scheduler.addRecurringTask('deleteExpiredInvites', 60 * 60 * 1000, deleteExpiredInvites); + scheduler.addRecurringTask('pauseIdleConnections', 60 * 60 * 1000, () => + pauseIdleConnections(clusterManager), + ); + scheduler.addRecurringTask('syncBotlistStats', 10 * 60 * 10_000, () => syncBotlistStats(clusterManager), ); } diff --git a/src/core/Client.ts b/src/core/Client.ts index fb3a8299..10ad681c 100644 --- a/src/core/Client.ts +++ b/src/core/Client.ts @@ -20,6 +20,8 @@ import { loadLocales } from '../utils/Locale.js'; import loadCommandFiles from '../utils/LoadCommands.js'; import { connectionCache as _connectionCache, + messageTimestamps, + storeMsgTimestamps, syncConnectionCache, } from '../utils/ConnectedList.js'; import { PROJECT_VERSION } from '../utils/Constants.js'; @@ -108,6 +110,13 @@ export default class SuperClient extends Client { syncConnectionCache, ); + // store network message timestamps to connectedList every minute + this.scheduler.addRecurringTask('storeMsgTimestamps', 60 * 1_000, () => { + storeMsgTimestamps(messageTimestamps); + messageTimestamps.clear(); + }); + + await this.login(process.env.TOKEN); } diff --git a/src/index.ts b/src/index.ts index fb4fc62b..64a9e768 100644 --- a/src/index.ts +++ b/src/index.ts @@ -2,14 +2,14 @@ import './instrument.js'; import Logger from './utils/Logger.js'; import SuperClient from './core/Client.js'; import { eventMethods } from './decorators/GatewayEvent.js'; -import ReactionUpdater from './utils/ReactionUpdater.js'; +import { RandomComponents } from './utils/RandomComponents.js'; import EventManager from './managers/EventManager.js'; const client = new SuperClient(); // dum classes // eslint-disable-next-line @typescript-eslint/no-unused-vars -const _reactionUpdater = ReactionUpdater; +const _randomComponentHandlers = RandomComponents; // eslint-disable-next-line @typescript-eslint/no-unused-vars const _eventManager = EventManager; diff --git a/src/managers/EventManager.ts b/src/managers/EventManager.ts index 99d98154..dc53653c 100644 --- a/src/managers/EventManager.ts +++ b/src/managers/EventManager.ts @@ -14,25 +14,25 @@ import { Interaction, Client, } from 'discord.js'; +import db from '../utils/Db.js'; import Logger from '../utils/Logger.js'; +import SuperClient from '../core/Client.js'; import GatewayEvent from '../decorators/GatewayEvent.js'; -import { stripIndents } from 'common-tags'; +import sendBroadcast from '../scripts/network/sendBroadcast.js'; +import storeMessageData from '../scripts/network/storeMessageData.js'; import getWelcomeTargets from '../scripts/guilds/getWelcomeTarget.js'; +import { t } from '../utils/Locale.js'; +import { check } from '../utils/Profanity.js'; +import { runChecks } from '../scripts/network/runChecks.js'; +import { stripIndents } from 'common-tags'; import { logGuildJoin, logGuildLeave } from '../scripts/guilds/goals.js'; import { channels, emojis, colors, LINKS } from '../utils/Constants.js'; -import { check } from '../utils/Profanity.js'; -import db from '../utils/Db.js'; -import { t } from '../utils/Locale.js'; -import storeMessageData from '../scripts/network/storeMessageData.js'; import { getReferredMsgData, sendWelcomeMsg } from '../scripts/network/helpers.js'; import { HubSettingsBitField } from '../utils/BitFields.js'; import { getAttachmentURL, getUserLocale, handleError, simpleEmbed, wait } from '../utils/Utils.js'; -import { runChecks } from '../scripts/network/runChecks.js'; import { addReaction, updateReactions } from '../scripts/reaction/actions.js'; import { checkBlacklists } from '../scripts/reaction/helpers.js'; import { CustomID } from '../utils/CustomID.js'; -import SuperClient from '../core/Client.js'; -import sendBroadcast from '../scripts/network/sendBroadcast.js'; import { logServerLeave } from '../utils/HubLogger/JoinLeave.js'; import { deleteConnections, modifyConnection } from '../utils/ConnectedList.js'; @@ -235,9 +235,12 @@ export default abstract class EventManager { const { connectionCache, cachePopulated } = message.client; - while (!cachePopulated) { - Logger.debug('[InterChat]: Cache not populated, retrying in 5 seconds...'); + if (!cachePopulated) { + Logger.debug('[InterChat]: Connection cache not populated, 5 secs until retry...'); await wait(5000); + + EventManager.onMessageCreate(message); + return; } const locale = await getUserLocale(message.author.id); diff --git a/src/scripts/network/components.ts b/src/scripts/network/components.ts index 81224e69..3bdc1603 100644 --- a/src/scripts/network/components.ts +++ b/src/scripts/network/components.ts @@ -6,6 +6,8 @@ type extraOpts = { disconnectEmoji?: string; connectEmoji?: string; userId?: string; + /** set custom prefix for customId and handle it urself, eg: `epik_reconnect` */ + customCustomId?: string; }; /** @@ -25,7 +27,7 @@ export const buildConnectionButtons = ( new ButtonBuilder() .setCustomId( new CustomID() - .setIdentifier('connection', 'toggle') + .setIdentifier(opts.customCustomId ?? 'connection', 'toggle') .addArgs(channelId) .addArgs(opts?.userId ?? '') .toString(), diff --git a/src/scripts/network/storeMessageData.ts b/src/scripts/network/storeMessageData.ts index cfcf40ba..fe12abd9 100644 --- a/src/scripts/network/storeMessageData.ts +++ b/src/scripts/network/storeMessageData.ts @@ -1,8 +1,8 @@ +import db from '../../utils/Db.js'; import { originalMessages } from '@prisma/client'; import { APIMessage, Message } from 'discord.js'; -import { parseTimestampFromId } from '../../utils/Utils.js'; -import db from '../../utils/Db.js'; -import { modifyConnections } from '../../utils/ConnectedList.js'; +import { messageTimestamps, modifyConnections } from '../../utils/ConnectedList.js'; +import { handleError, parseTimestampFromId } from '../../utils/Utils.js'; export interface NetworkWebhookSendResult { messageOrError: APIMessage | string; @@ -20,7 +20,7 @@ export default async ( hubId: string, dbReference?: originalMessages | null, ) => { - const messageDataObj: { channelId: string; messageId: string; createdAt: number }[] = []; + const messageDataObj: { channelId: string; messageId: string, createdAt: Date }[] = []; const invalidWebhookURLs: string[] = []; const validErrors = ['Invalid Webhook Token', 'Unknown Webhook', 'Missing Permissions']; @@ -30,7 +30,7 @@ export default async ( messageDataObj.push({ channelId: result.messageOrError.channel_id, messageId: result.messageOrError.id, - createdAt: parseTimestampFromId(result.messageOrError.id), + createdAt: new Date(parseTimestampFromId(result.messageOrError.id)), }); } else if (validErrors.some((e) => (result.messageOrError as string).includes(e))) { @@ -48,13 +48,16 @@ export default async ( authorId: message.author.id, serverId: message.guild.id, messageReference: dbReference?.messageId, + createdAt: message.createdAt, broadcastMsgs: { createMany: { data: messageDataObj } }, hub: { connect: { id: hubId } }, reactions: {}, }, - }); + }).catch(handleError); } + // store message timestamps to push to db later + messageTimestamps.set(message.channel.id, message.createdAt); // disconnect network if, webhook does not exist/bot cannot access webhook if (invalidWebhookURLs.length > 0) { await modifyConnections({ webhookURL: { in: invalidWebhookURLs } }, { connected: false }); diff --git a/src/scripts/tasks/deleteOldMessages.ts b/src/scripts/tasks/deleteOldMessages.ts deleted file mode 100644 index 3e0c00f5..00000000 --- a/src/scripts/tasks/deleteOldMessages.ts +++ /dev/null @@ -1,15 +0,0 @@ -import db from '../../utils/Db.js'; - -// Delete all network messages from db that are older than 24 hours old. -export default async () => { - const olderThan24h = Date.now() - 60 * 60 * 24_000; - - const data = await db.broadcastedMessages.findMany({ - where: { createdAt: { lte: olderThan24h } }, - }); - await db.broadcastedMessages.deleteMany({ where: { createdAt: { lte: olderThan24h } } }); - - await db.originalMessages.deleteMany({ - where: { messageId: { in: data.map((d) => d.originalMsgId) } }, - }); -}; diff --git a/src/scripts/tasks/pauseIdleConnections.ts b/src/scripts/tasks/pauseIdleConnections.ts new file mode 100644 index 00000000..263c9496 --- /dev/null +++ b/src/scripts/tasks/pauseIdleConnections.ts @@ -0,0 +1,68 @@ +import db from '../../utils/Db.js'; +import Logger from '../../utils/Logger.js'; +import { ClusterManager } from 'discord-hybrid-sharding'; +import { modifyConnection } from '../../utils/ConnectedList.js'; +import { APIActionRowComponent, APIButtonComponent, Snowflake } from 'discord.js'; +import { buildConnectionButtons } from '../network/components.js'; +import { simpleEmbed } from '../../utils/Utils.js'; +import { stripIndents } from 'common-tags'; +import { emojis } from '../../utils/Constants.js'; +import 'dotenv/config'; + +export default async (manager: ClusterManager) => { + const connections = await db.connectedList.findMany({ + where: { + connected: true, + lastActive: { not: null, lte: new Date(Date.now() - 24 * 60 * 60 * 1000) }, + }, + }); + + if (!connections) return; + + const reconnectButtonArr: { + channelId: Snowflake; + button: APIActionRowComponent; + }[] = []; + + // Loop through the data + connections.forEach(async ({ channelId, lastActive }) => { + Logger.debug( + `[InterChat]: Channel ${channelId} is older than 24 hours: ${lastActive?.toLocaleString()} - ${new Date().toLocaleString()}`, + ); + + // Create the button + reconnectButtonArr.push({ + channelId, + button: buildConnectionButtons(false, channelId, { + customCustomId: 'inactiveConnect', + }).toJSON(), + }); + + // disconnect the channel + await modifyConnection({ channelId }, { lastActive: null, connected: false }); + }); + + const embed = simpleEmbed( + stripIndents` + ## ${emojis.timeout} Paused Due to Inactivity + Connection to this hub has been stopped. **Click the button** below to resume chatting (or alternatively, \`/connection\`). + `, + ).toJSON(); + + await manager.broadcastEval( + (client, { _connections, _embed, buttons }) => { + _connections.forEach(async (connection) => { + const channel = await client.channels.fetch(connection.channelId).catch(() => null); + const button = buttons.find((b) => b.channelId === connection.channelId)?.button; + + if (!channel?.isTextBased() || !button) return; + + // remove it since we are done with it + _connections.splice(_connections.indexOf(connection), 1); + + await channel.send({ embeds: [_embed], components: [button] }).catch(() => null); + }); + }, + { context: { _connections: connections, _embed: embed, buttons: reconnectButtonArr } }, + ); +}; diff --git a/src/services/SchedulerService.ts b/src/services/SchedulerService.ts index 8d8a62e4..5d8130ad 100644 --- a/src/services/SchedulerService.ts +++ b/src/services/SchedulerService.ts @@ -67,6 +67,11 @@ export default class Scheduler { this.tasks.forEach((_, taskName) => this.stopTask(taskName)); } + + hasTask(taskName: string): boolean { + return this.tasks.has(taskName); + } + /** * Returns an array of currently running task names. */ diff --git a/src/utils/ConnectedList.ts b/src/utils/ConnectedList.ts index b8e91d49..8002705f 100644 --- a/src/utils/ConnectedList.ts +++ b/src/utils/ConnectedList.ts @@ -1,10 +1,12 @@ -import { Prisma, connectedList } from '@prisma/client'; import db from './Db.js'; -import { Collection } from 'discord.js'; import Logger from './Logger.js'; +import { Prisma, connectedList } from '@prisma/client'; +import { Collection } from 'discord.js'; +import { captureException } from '@sentry/node'; /** 📡 Contains all the **connected** channels from all hubs. */ export const connectionCache = new Collection(); +export const messageTimestamps = new Collection(); export const syncConnectionCache = async () => { Logger.debug('[InterChat]: Populating connection cache.'); @@ -40,7 +42,6 @@ export const modifyConnection = async ( data: Prisma.connectedListUpdateInput, ) => { const connection = await db.connectedList.update({ where, data }); - connectionCache.set(connection.channelId, connection); return connection; }; @@ -57,3 +58,10 @@ export const modifyConnections = async ( return connections; }; + +export const storeMsgTimestamps = (data: Collection): void => { + data.forEach( + async (lastActive, channelId) => + await modifyConnection({ channelId }, { lastActive }).catch(captureException), + ); +}; diff --git a/src/utils/JSON/emojis.json b/src/utils/JSON/emojis.json index 0e01bd5c..b3797c34 100644 --- a/src/utils/JSON/emojis.json +++ b/src/utils/JSON/emojis.json @@ -13,7 +13,8 @@ "ID": "<:blurple_id:1032318192741261393>", "loading": "", "bruhcat": "<:bruhcat:976368249858056223>", - "tick": "<:tick:1032318190144995399>", + "cursive_tick": "<:tick:1032318190144995399>", + "tick": "<:tick:1212965104182894622>", "force_enabled": "<:devportal_enabled:994444521888821338>", "chatbot_circle": "<:chatbot_circle:1077200892157243503>", "dotYellow": "<:yellowdot:986617776196190248>", diff --git a/src/utils/ReactionUpdater.ts b/src/utils/RandomComponents.ts similarity index 88% rename from src/utils/ReactionUpdater.ts rename to src/utils/RandomComponents.ts index 6fe84336..2f33e4e5 100644 --- a/src/utils/ReactionUpdater.ts +++ b/src/utils/RandomComponents.ts @@ -9,7 +9,7 @@ import { StringSelectMenuBuilder, time, } from 'discord.js'; -import { getEmojiId, sortReactions } from './Utils.js'; +import { getEmojiId, simpleEmbed, sortReactions } from './Utils.js'; import { HubSettingsBitField } from './BitFields.js'; import { CustomID } from './CustomID.js'; import { RegisterInteractionHandler } from '../decorators/Interaction.js'; @@ -18,11 +18,15 @@ import { stripIndents } from 'common-tags'; import { t } from './Locale.js'; import { removeReaction, addReaction, updateReactions } from '../scripts/reaction/actions.js'; import { checkBlacklists } from '../scripts/reaction/helpers.js'; +import { modifyConnection } from './ConnectedList.js'; -export default abstract class ReactionUpdater { +// skipcq: JS-0327 +export abstract class RandomComponents { /** Listens for a reaction button or select menu interaction and updates the reactions accordingly. */ @RegisterInteractionHandler('reaction_') - static async listenForReactionButton(interaction: ButtonInteraction | AnySelectMenuInteraction): Promise { + static async listenForReactionButton( + interaction: ButtonInteraction | AnySelectMenuInteraction, + ): Promise { await interaction.deferUpdate(); if (!interaction.inCachedGuild()) return; @@ -196,4 +200,19 @@ export default abstract class ReactionUpdater { await updateReactions(messageInDb.originalMsg.broadcastMsgs, dbReactions); } } + + @RegisterInteractionHandler('inactiveConnect', 'toggle') + static async inactiveConnect(interaction: ButtonInteraction): Promise { + const customId = CustomID.parseCustomId(interaction.customId); + const channelId = customId.args[0]; + + await modifyConnection({ channelId }, { connected: true }); + + await interaction.update({ + embeds: [ + simpleEmbed(`## ${emojis.tick} Connection Resumed\nConnection has been resumed. Have fun chatting!`), + ], + components: [], + }); + } } diff --git a/src/utils/Utils.ts b/src/utils/Utils.ts index a16a8a8b..f65943b2 100644 --- a/src/utils/Utils.ts +++ b/src/utils/Utils.ts @@ -229,7 +229,7 @@ export const replaceLinks = (string: string, replaceText = '`[LINK HIDDEN]`') => export const simpleEmbed = ( description: string, - opts?: { color: ColorResolvable; title?: string }, + opts?: { color?: ColorResolvable; title?: string }, ) => { return new EmbedBuilder() .setTitle(opts?.title ?? null)