allow setting separate timeout / max size for imports - fixes misskey…

…-dev#479
HanaMisskey · Aug 13, 2024 · ea8a034 · ea8a034
1 parent ccb2f75
commit ea8a034
Show file tree

Hide file tree

Showing 4 changed files with 37 additions and 10 deletions.
diff --git a/.config/example.yml b/.config/example.yml
@@ -295,5 +295,10 @@ signToActivityPubGet: true
 # Upload or download file size limits (bytes)
 #maxFileSize: 262144000
 
+# timeout and maximum size for imports (e.g. note imports)
+#import:
+#  downloadTimeout: 30
+#  maxFileSize: 262144000
+
 # PID File of master process
 #pidFile: /tmp/misskey.pid
diff --git a/packages/backend/src/config.ts b/packages/backend/src/config.ts
@@ -95,6 +95,12 @@ type Source = {
 	perChannelMaxNoteCacheCount?: number;
 	perUserNotificationsMaxCount?: number;
 	deactivateAntennaThreshold?: number;
+
+	import?: {
+		downloadTimeout: number;
+		maxFileSize: number;
+	};
+
 	pidFile: string;
 };
 
@@ -174,6 +180,12 @@ export type Config = {
 	perChannelMaxNoteCacheCount: number;
 	perUserNotificationsMaxCount: number;
 	deactivateAntennaThreshold: number;
+
+	import: {
+		downloadTimeout: number;
+		maxFileSize: number;
+	} | undefined;
+
 	pidFile: string;
 };
 
@@ -275,6 +287,7 @@ export function loadConfig(): Config {
 		perChannelMaxNoteCacheCount: config.perChannelMaxNoteCacheCount ?? 1000,
 		perUserNotificationsMaxCount: config.perUserNotificationsMaxCount ?? 500,
 		deactivateAntennaThreshold: config.deactivateAntennaThreshold ?? (1000 * 60 * 60 * 24 * 7),
+		import: config.import,
 		pidFile: config.pidFile,
 	};
 }

diff --git a/packages/backend/src/core/DownloadService.ts b/packages/backend/src/core/DownloadService.ts
@@ -35,14 +35,14 @@ export class DownloadService {
 	}
 
 	@bindThis
-	public async downloadUrl(url: string, path: string): Promise<{
+	public async downloadUrl(url: string, path: string, options: { timeout?: number, operationTimeout?: number, maxSize?: number} = {} ): Promise<{
 		filename: string;
 	}> {
 		this.logger.info(`Downloading ${chalk.cyan(url)} to ${chalk.cyanBright(path)} ...`);
 
-		const timeout = 30 * 1000;
-		const operationTimeout = 60 * 1000;
-		const maxSize = this.config.maxFileSize ?? 262144000;
+		const timeout = options.timeout ?? 30 * 1000;
+		const operationTimeout = options.operationTimeout ?? 60 * 1000;
+		const maxSize = options.maxSize ?? this.config.maxFileSize ?? 262144000;
 
 		const urlObj = new URL(url);
 		let filename = urlObj.pathname.split('/').pop() ?? 'untitled';

diff --git a/packages/backend/src/queue/processors/ImportNotesProcessorService.ts b/packages/backend/src/queue/processors/ImportNotesProcessorService.ts
@@ -20,12 +20,16 @@ import { IdService } from '@/core/IdService.js';
 import { QueueLoggerService } from '../QueueLoggerService.js';
 import type * as Bull from 'bullmq';
 import type { DbNoteImportToDbJobData, DbNoteImportJobData, DbNoteWithParentImportToDbJobData } from '../types.js';
+import type { Config } from '@/config.js';
 
 @Injectable()
 export class ImportNotesProcessorService {
 	private logger: Logger;
 
 	constructor(
+		@Inject(DI.config)
+		private config: Config,
+
 		@Inject(DI.usersRepository)
 		private usersRepository: UsersRepository,
 
@@ -74,6 +78,11 @@ export class ImportNotesProcessorService {
 		}
 	}
 
+	@bindThis
+	private downloadUrl(url: string, path:string): Promise<{filename: string}> {
+		return this.downloadService.downloadUrl(url, path, { operationTimeout: this.config.import?.downloadTimeout, maxSize: this.config.import?.maxFileSize });
+	}
+
 	@bindThis
 	private async recreateChain(idFieldPath: string[], replyFieldPath: string[], arr: any[], includeOrphans: boolean): Promise<any[]> {
 		type NotesMap = {
@@ -177,7 +186,7 @@ export class ImportNotesProcessorService {
 
 			try {
 				await fsp.writeFile(destPath, '', 'binary');
-				await this.downloadService.downloadUrl(file.url, destPath);
+				await this.downloadUrl(file.url, destPath);
 			} catch (e) { // TODO: 何度か再試行
 				if (e instanceof Error || typeof e === 'string') {
 					this.logger.error(e);
@@ -207,7 +216,7 @@ export class ImportNotesProcessorService {
 
 			try {
 				await fsp.writeFile(destPath, '', 'binary');
-				await this.downloadService.downloadUrl(file.url, destPath);
+				await this.downloadUrl(file.url, destPath);
 			} catch (e) { // TODO: 何度か再試行
 				if (e instanceof Error || typeof e === 'string') {
 					this.logger.error(e);
@@ -240,7 +249,7 @@ export class ImportNotesProcessorService {
 
 			try {
 				await fsp.writeFile(destPath, '', 'binary');
-				await this.downloadService.downloadUrl(file.url, destPath);
+				await this.downloadUrl(file.url, destPath);
 			} catch (e) { // TODO: 何度か再試行
 				if (e instanceof Error || typeof e === 'string') {
 					this.logger.error(e);
@@ -298,7 +307,7 @@ export class ImportNotesProcessorService {
 
 			try {
 				await fsp.writeFile(path, '', 'utf-8');
-				await this.downloadService.downloadUrl(file.url, path);
+				await this.downloadUrl(file.url, path);
 			} catch (e) { // TODO: 何度か再試行
 				if (e instanceof Error || typeof e === 'string') {
 					this.logger.error(e);
@@ -350,7 +359,7 @@ export class ImportNotesProcessorService {
 
 				if (!exists) {
 					try {
-						await this.downloadService.downloadUrl(file.url, filePath);
+						await this.downloadUrl(file.url, filePath);
 					} catch (e) { // TODO: 何度か再試行
 						this.logger.error(e instanceof Error ? e : new Error(e as string));
 					}
@@ -489,7 +498,7 @@ export class ImportNotesProcessorService {
 
 				if (!exists) {
 					try {
-						await this.downloadService.downloadUrl(file.url, filePath);
+						await this.downloadUrl(file.url, filePath);
 					} catch (e) { // TODO: 何度か再試行
 						this.logger.error(e instanceof Error ? e : new Error(e as string));
 					}