From 0d93433b0836238f518658b5719ce335060963b0 Mon Sep 17 00:00:00 2001 From: caipira113 Date: Mon, 17 Jul 2023 21:32:22 +0900 Subject: [PATCH] feat: Enable storage of remote file cache in a separate object storage --- ...689580926821-ObjectStorageRemoteSetting.js | 36 +++ packages/backend/src/core/DriveService.ts | 58 +++-- packages/backend/src/core/S3Service.ts | 44 ++-- packages/backend/src/models/entities/Meta.ts | 72 ++++++ .../CleanRemoteFilesProcessorService.ts | 2 +- .../DeleteAccountProcessorService.ts | 5 +- .../DeleteDriveFilesProcessorService.ts | 5 +- .../processors/DeleteFileProcessorService.ts | 15 +- .../src/server/api/endpoints/admin/meta.ts | 61 +++++ .../server/api/endpoints/admin/update-meta.ts | 65 +++++ .../src/pages/admin/object-storage.vue | 233 +++++++++++++----- 11 files changed, 494 insertions(+), 102 deletions(-) create mode 100644 packages/backend/migration/1689580926821-ObjectStorageRemoteSetting.js diff --git a/packages/backend/migration/1689580926821-ObjectStorageRemoteSetting.js b/packages/backend/migration/1689580926821-ObjectStorageRemoteSetting.js new file mode 100644 index 000000000000..a27fda3e705b --- /dev/null +++ b/packages/backend/migration/1689580926821-ObjectStorageRemoteSetting.js @@ -0,0 +1,36 @@ +export class ObjectStorageRemoteSetting1689580926821 { + name = 'ObjectStorageRemoteSetting1689580926821' + + async up(queryRunner) { + await queryRunner.query(`ALTER TABLE "meta" ADD "useObjectStorageRemote" boolean NOT NULL DEFAULT false`); + await queryRunner.query(`ALTER TABLE "meta" ADD "objectStorageRemoteBucket" character varying(1024)`); + await queryRunner.query(`ALTER TABLE "meta" ADD "objectStorageRemotePrefix" character varying(1024)`); + await queryRunner.query(`ALTER TABLE "meta" ADD "objectStorageRemoteBaseUrl" character varying(1024)`); + await queryRunner.query(`ALTER TABLE "meta" ADD "objectStorageRemoteEndpoint" character varying(1024)`); + await queryRunner.query(`ALTER TABLE "meta" ADD "objectStorageRemoteRegion" character varying(1024)`); + await queryRunner.query(`ALTER TABLE "meta" ADD "objectStorageRemoteAccessKey" character varying(1024)`); + await queryRunner.query(`ALTER TABLE "meta" ADD "objectStorageRemoteSecretKey" character varying(1024)`); + await queryRunner.query(`ALTER TABLE "meta" ADD "objectStorageRemotePort" integer`); + await queryRunner.query(`ALTER TABLE "meta" ADD "objectStorageRemoteUseSSL" boolean NOT NULL DEFAULT true`); + await queryRunner.query(`ALTER TABLE "meta" ADD "objectStorageRemoteUseProxy" boolean NOT NULL DEFAULT true`, undefined); + await queryRunner.query(`ALTER TABLE "meta" ADD "objectStorageRemoteSetPublicRead" boolean NOT NULL DEFAULT false`); + await queryRunner.query(`ALTER TABLE "meta" ADD "objectStorageRemoteS3ForcePathStyle" boolean NOT NULL DEFAULT true`); + } + + async down(queryRunner) { + await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "objectStorageRemoteS3ForcePathStyle"`); + await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "objectStorageRemoteSetPublicRead"`); + await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "objectStorageRemoteUseProxy"`, undefined); + await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "objectStorageRemoteUseSSL"`); + await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "objectStorageRemotePort"`); + await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "objectStorageRemoteSecretKey"`); + await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "objectStorageRemoteAccessKey"`); + await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "objectStorageRemoteRegion"`); + await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "objectStorageRemoteEndpoint"`); + await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "objectStorageRemoteBaseUrl"`); + await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "objectStorageRemotePrefix"`); + await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "objectStorageRemoteBucket"`); + await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "useObjectStorageRemote"`); + } + +} diff --git a/packages/backend/src/core/DriveService.ts b/packages/backend/src/core/DriveService.ts index 6c0b62434675..f9cdbd352dd0 100644 --- a/packages/backend/src/core/DriveService.ts +++ b/packages/backend/src/core/DriveService.ts @@ -131,9 +131,10 @@ export class DriveService { * @param type Content-Type for original * @param hash Hash for original * @param size Size for original + * @param isRemote If true, file is remote file */ @bindThis - private async save(file: DriveFile, path: string, name: string, type: string, hash: string, size: number): Promise { + private async save(file: DriveFile, path: string, name: string, type: string, hash: string, size: number, isRemote: boolean): Promise { // thunbnail, webpublic を必要なら生成 const alts = await this.generateAlts(path, type, !file.uri); @@ -158,11 +159,19 @@ export class DriveService { ext = ''; } - const baseUrl = meta.objectStorageBaseUrl - ?? `${ meta.objectStorageUseSSL ? 'https' : 'http' }://${ meta.objectStorageEndpoint }${ meta.objectStoragePort ? `:${meta.objectStoragePort}` : '' }/${ meta.objectStorageBucket }`; + const useObjectStorageRemote = isRemote && meta.useObjectStorageRemote; + const objectStorageBaseUrl = useObjectStorageRemote ? meta.objectStorageRemoteBaseUrl : meta.objectStorageBaseUrl; + const objectStorageUseSSL = useObjectStorageRemote ? meta.objectStorageRemoteUseSSL : meta.objectStorageUseSSL; + const objectStorageEndpoint = useObjectStorageRemote ? meta.objectStorageRemoteEndpoint : meta.objectStorageEndpoint; + const objectStoragePort = useObjectStorageRemote ? meta.objectStorageRemotePort : meta.objectStoragePort; + const objectStorageBucket = useObjectStorageRemote ? meta.objectStorageRemoteBucket : meta.objectStorageBucket; + const objectStoragePrefix = useObjectStorageRemote ? meta.objectStorageRemotePrefix : meta.objectStoragePrefix; + + const baseUrl = objectStorageBaseUrl + ?? `${ objectStorageUseSSL ? 'https' : 'http' }://${ objectStorageEndpoint }${ objectStoragePort ? `:${objectStoragePort}` : '' }/${ objectStorageBucket }`; // for original - const key = `${meta.objectStoragePrefix}/${randomUUID()}${ext}`; + const key = `${objectStoragePrefix}/${randomUUID()}${ext}`; const url = `${ baseUrl }/${ key }`; // for alts @@ -175,23 +184,23 @@ export class DriveService { //#region Uploads this.registerLogger.info(`uploading original: ${key}`); const uploads = [ - this.upload(key, fs.createReadStream(path), type, null, name), + this.upload(key, fs.createReadStream(path), type, isRemote, null, name), ]; if (alts.webpublic) { - webpublicKey = `${meta.objectStoragePrefix}/webpublic-${randomUUID()}.${alts.webpublic.ext}`; + webpublicKey = `${objectStoragePrefix}/webpublic-${randomUUID()}.${alts.webpublic.ext}`; webpublicUrl = `${ baseUrl }/${ webpublicKey }`; this.registerLogger.info(`uploading webpublic: ${webpublicKey}`); - uploads.push(this.upload(webpublicKey, alts.webpublic.data, alts.webpublic.type, alts.webpublic.ext, name)); + uploads.push(this.upload(webpublicKey, alts.webpublic.data, alts.webpublic.type, isRemote, alts.webpublic.ext, name)); } if (alts.thumbnail) { - thumbnailKey = `${meta.objectStoragePrefix}/thumbnail-${randomUUID()}.${alts.thumbnail.ext}`; + thumbnailKey = `${objectStoragePrefix}/thumbnail-${randomUUID()}.${alts.thumbnail.ext}`; thumbnailUrl = `${ baseUrl }/${ thumbnailKey }`; this.registerLogger.info(`uploading thumbnail: ${thumbnailKey}`); - uploads.push(this.upload(thumbnailKey, alts.thumbnail.data, alts.thumbnail.type, alts.thumbnail.ext, `${name}.thumbnail`)); + uploads.push(this.upload(thumbnailKey, alts.thumbnail.data, alts.thumbnail.type, isRemote, alts.thumbnail.ext, `${name}.thumbnail`)); } await Promise.all(uploads); @@ -360,14 +369,18 @@ export class DriveService { * Upload to ObjectStorage */ @bindThis - private async upload(key: string, stream: fs.ReadStream | Buffer, type: string, ext?: string | null, filename?: string) { + private async upload(key: string, stream: fs.ReadStream | Buffer, type: string, isRemote: boolean, ext?: string | null, filename?: string) { if (type === 'image/apng') type = 'image/png'; if (!FILE_TYPE_BROWSERSAFE.includes(type)) type = 'application/octet-stream'; const meta = await this.metaService.fetch(); + const useObjectStorageRemote = isRemote && meta.useObjectStorageRemote; + const objectStorageBucket = useObjectStorageRemote ? meta.objectStorageRemoteBucket : meta.objectStorageBucket; + const objectStorageSetPublicRead = useObjectStorageRemote ? meta.objectStorageRemoteSetPublicRead : meta.objectStorageSetPublicRead; + const params = { - Bucket: meta.objectStorageBucket, + Bucket: objectStorageBucket, Key: key, Body: stream, ContentType: type, @@ -380,9 +393,9 @@ export class DriveService { // 許可されているファイル形式でしか拡張子をつけない ext ? correctFilename(filename, ext) : filename, ); - if (meta.objectStorageSetPublicRead) params.ACL = 'public-read'; + if (objectStorageSetPublicRead) params.ACL = 'public-read'; - await this.s3Service.upload(meta, params) + await this.s3Service.upload(meta, params, isRemote) .then( result => { if ('Bucket' in result) { // CompleteMultipartUploadCommandOutput @@ -618,7 +631,8 @@ export class DriveService { } } } else { - file = await (this.save(file, path, detectedName, info.type.mime, info.md5, info.size)); + const isRemote = user ? this.userEntityService.isRemoteUser(user) : false; + file = await (this.save(file, path, detectedName, info.type.mime, info.md5, info.size, isRemote)); } this.registerLogger.succ(`drive file has been created ${file.id}`); @@ -672,7 +686,7 @@ export class DriveService { } @bindThis - public async deleteFileSync(file: DriveFile, isExpired = false) { + public async deleteFileSync(file: DriveFile, isExpired = false, isRemote: boolean) { if (file.storedInternal) { this.internalStorageService.del(file.accessKey!); @@ -686,14 +700,14 @@ export class DriveService { } else if (!file.isLink) { const promises = []; - promises.push(this.deleteObjectStorageFile(file.accessKey!)); + promises.push(this.deleteObjectStorageFile(file.accessKey!, isRemote)); if (file.thumbnailUrl) { - promises.push(this.deleteObjectStorageFile(file.thumbnailAccessKey!)); + promises.push(this.deleteObjectStorageFile(file.thumbnailAccessKey!, isRemote)); } if (file.webpublicUrl) { - promises.push(this.deleteObjectStorageFile(file.webpublicAccessKey!)); + promises.push(this.deleteObjectStorageFile(file.webpublicAccessKey!, isRemote)); } await Promise.all(promises); @@ -733,15 +747,17 @@ export class DriveService { } @bindThis - public async deleteObjectStorageFile(key: string) { + public async deleteObjectStorageFile(key: string, isRemote: boolean) { const meta = await this.metaService.fetch(); + const useObjectStorageRemote = isRemote && meta.useObjectStorageRemote; + const objectStorageBucket = useObjectStorageRemote ? meta.objectStorageRemoteBucket : meta.objectStorageBucket; try { const param = { - Bucket: meta.objectStorageBucket, + Bucket: objectStorageBucket, Key: key, } as DeleteObjectCommandInput; - await this.s3Service.delete(meta, param); + await this.s3Service.delete(meta, param, isRemote); } catch (err: any) { if (err.name === 'NoSuchKey') { this.deleteLogger.warn(`The object storage had no such key to delete: ${key}. Skipping this.`, err as Error); diff --git a/packages/backend/src/core/S3Service.ts b/packages/backend/src/core/S3Service.ts index 01ce12ffdd9c..7f5f7463e536 100644 --- a/packages/backend/src/core/S3Service.ts +++ b/packages/backend/src/core/S3Service.ts @@ -23,35 +23,45 @@ export class S3Service { } @bindThis - public getS3Client(meta: Meta): S3Client { - const u = meta.objectStorageEndpoint - ? `${meta.objectStorageUseSSL ? 'https' : 'http'}://${meta.objectStorageEndpoint}` - : `${meta.objectStorageUseSSL ? 'https' : 'http'}://example.net`; // dummy url to select http(s) agent + public getS3Client(meta: Meta, isRemote: boolean): S3Client { + const useObjectStorageRemote = isRemote && meta.useObjectStorageRemote; - const agent = this.httpRequestService.getAgentByUrl(new URL(u), !meta.objectStorageUseProxy); + const objectStorageEndpoint = useObjectStorageRemote ? meta.objectStorageRemoteEndpoint : meta.objectStorageEndpoint; + const objectStorageUseSSL = useObjectStorageRemote ? meta.objectStorageRemoteUseSSL : meta.objectStorageUseSSL; + const objectStorageUseProxy = useObjectStorageRemote ? meta.objectStorageRemoteUseProxy : meta.objectStorageUseProxy; + const objectStorageAccessKey = useObjectStorageRemote ? meta.objectStorageRemoteAccessKey : meta.objectStorageAccessKey; + const objectStorageSecretKey = useObjectStorageRemote ? meta.objectStorageRemoteSecretKey : meta.objectStorageSecretKey; + const objectStorageRegion = useObjectStorageRemote ? meta.objectStorageRemoteRegion : meta.objectStorageRegion; + const objectStorageS3ForcePathStyle = useObjectStorageRemote ? meta.objectStorageRemoteS3ForcePathStyle : meta.objectStorageS3ForcePathStyle; + + const u = objectStorageEndpoint + ? `${objectStorageUseSSL ? 'https' : 'http'}://${objectStorageEndpoint}` + : `${objectStorageUseSSL ? 'https' : 'http'}://example.net`; // dummy url to select http(s) agent + + const agent = this.httpRequestService.getAgentByUrl(new URL(u), !objectStorageUseProxy); const handlerOption: NodeHttpHandlerOptions = {}; - if (meta.objectStorageUseSSL) { + if (objectStorageUseSSL) { handlerOption.httpsAgent = agent as https.Agent; } else { handlerOption.httpAgent = agent as http.Agent; } return new S3Client({ - endpoint: meta.objectStorageEndpoint ? u : undefined, - credentials: (meta.objectStorageAccessKey !== null && meta.objectStorageSecretKey !== null) ? { - accessKeyId: meta.objectStorageAccessKey, - secretAccessKey: meta.objectStorageSecretKey, + endpoint: objectStorageEndpoint ? u : undefined, + credentials: (objectStorageAccessKey !== null && objectStorageSecretKey !== null) ? { + accessKeyId: objectStorageAccessKey, + secretAccessKey: objectStorageSecretKey, } : undefined, - region: meta.objectStorageRegion ? meta.objectStorageRegion : undefined, // 空文字列もundefinedにするため ?? は使わない - tls: meta.objectStorageUseSSL, - forcePathStyle: meta.objectStorageEndpoint ? meta.objectStorageS3ForcePathStyle : false, // AWS with endPoint omitted + region: objectStorageRegion ? objectStorageRegion : undefined, // empty string is converted to undefined + tls: objectStorageUseSSL, + forcePathStyle: objectStorageEndpoint ? objectStorageS3ForcePathStyle : false, // AWS with endPoint omitted requestHandler: new NodeHttpHandler(handlerOption), }); } @bindThis - public async upload(meta: Meta, input: PutObjectCommandInput) { - const client = this.getS3Client(meta); + public async upload(meta: Meta, input: PutObjectCommandInput, isRemote: boolean) { + const client = this.getS3Client(meta, isRemote); return new Upload({ client, params: input, @@ -62,8 +72,8 @@ export class S3Service { } @bindThis - public delete(meta: Meta, input: DeleteObjectCommandInput) { - const client = this.getS3Client(meta); + public delete(meta: Meta, input: DeleteObjectCommandInput, isRemote: boolean) { + const client = this.getS3Client(meta, isRemote); return client.send(new DeleteObjectCommand(input)); } } diff --git a/packages/backend/src/models/entities/Meta.ts b/packages/backend/src/models/entities/Meta.ts index f7b4899da23a..2053e0da431b 100644 --- a/packages/backend/src/models/entities/Meta.ts +++ b/packages/backend/src/models/entities/Meta.ts @@ -433,6 +433,78 @@ export class Meta { }) public objectStorageS3ForcePathStyle: boolean; + @Column('boolean', { + default: false, + }) + public useObjectStorageRemote: boolean; + + @Column('varchar', { + length: 1024, + nullable: true, + }) + public objectStorageRemoteBucket: string | null; + + @Column('varchar', { + length: 1024, + nullable: true, + }) + public objectStorageRemotePrefix: string | null; + + @Column('varchar', { + length: 1024, + nullable: true, + }) + public objectStorageRemoteBaseUrl: string | null; + + @Column('varchar', { + length: 1024, + nullable: true, + }) + public objectStorageRemoteEndpoint: string | null; + + @Column('varchar', { + length: 1024, + nullable: true, + }) + public objectStorageRemoteRegion: string | null; + + @Column('varchar', { + length: 1024, + nullable: true, + }) + public objectStorageRemoteAccessKey: string | null; + + @Column('varchar', { + length: 1024, + nullable: true, + }) + public objectStorageRemoteSecretKey: string | null; + + @Column('integer', { + nullable: true, + }) + public objectStorageRemotePort: number | null; + + @Column('boolean', { + default: true, + }) + public objectStorageRemoteUseSSL: boolean; + + @Column('boolean', { + default: true, + }) + public objectStorageRemoteUseProxy: boolean; + + @Column('boolean', { + default: false, + }) + public objectStorageRemoteSetPublicRead: boolean; + + @Column('boolean', { + default: true, + }) + public objectStorageRemoteS3ForcePathStyle: boolean; + @Column('boolean', { default: false, }) diff --git a/packages/backend/src/queue/processors/CleanRemoteFilesProcessorService.ts b/packages/backend/src/queue/processors/CleanRemoteFilesProcessorService.ts index 6f887089eb0a..d9bfc4eafbb9 100644 --- a/packages/backend/src/queue/processors/CleanRemoteFilesProcessorService.ts +++ b/packages/backend/src/queue/processors/CleanRemoteFilesProcessorService.ts @@ -53,7 +53,7 @@ export class CleanRemoteFilesProcessorService { cursor = files.at(-1)?.id ?? null; - await Promise.all(files.map(file => this.driveService.deleteFileSync(file, true))); + await Promise.all(files.map(file => this.driveService.deleteFileSync(file, true, true))); deletedCount += 8; diff --git a/packages/backend/src/queue/processors/DeleteAccountProcessorService.ts b/packages/backend/src/queue/processors/DeleteAccountProcessorService.ts index 3e8969252a64..58ea26f5d219 100644 --- a/packages/backend/src/queue/processors/DeleteAccountProcessorService.ts +++ b/packages/backend/src/queue/processors/DeleteAccountProcessorService.ts @@ -9,6 +9,7 @@ import type { DriveFile } from '@/models/entities/DriveFile.js'; import type { Note } from '@/models/entities/Note.js'; import { EmailService } from '@/core/EmailService.js'; import { bindThis } from '@/decorators.js'; +import { UserEntityService } from '@/core/entities/UserEntityService.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; import type * as Bull from 'bullmq'; import type { DbUserDeleteJobData } from '../types.js'; @@ -34,6 +35,7 @@ export class DeleteAccountProcessorService { @Inject(DI.driveFilesRepository) private driveFilesRepository: DriveFilesRepository, + private userEntityService: UserEntityService, private driveService: DriveService, private emailService: EmailService, private queueLoggerService: QueueLoggerService, @@ -47,6 +49,7 @@ export class DeleteAccountProcessorService { this.logger.info(`Deleting account of ${job.data.user.id} ...`); const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); + const isRemote = user ? this.userEntityService.isRemoteUser(user) : false; if (user == null) { return; } @@ -104,7 +107,7 @@ export class DeleteAccountProcessorService { cursor = files.at(-1)?.id ?? null; for (const file of files) { - await this.driveService.deleteFileSync(file); + await this.driveService.deleteFileSync(file, false, isRemote); } } diff --git a/packages/backend/src/queue/processors/DeleteDriveFilesProcessorService.ts b/packages/backend/src/queue/processors/DeleteDriveFilesProcessorService.ts index 07e3762330fc..b481141eaad5 100644 --- a/packages/backend/src/queue/processors/DeleteDriveFilesProcessorService.ts +++ b/packages/backend/src/queue/processors/DeleteDriveFilesProcessorService.ts @@ -6,6 +6,7 @@ import type { Config } from '@/config.js'; import type Logger from '@/logger.js'; import { DriveService } from '@/core/DriveService.js'; import { bindThis } from '@/decorators.js'; +import { UserEntityService } from '@/core/entities/UserEntityService.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; import type * as Bull from 'bullmq'; import type { DbJobDataWithUser } from '../types.js'; @@ -25,6 +26,7 @@ export class DeleteDriveFilesProcessorService { private driveFilesRepository: DriveFilesRepository, private driveService: DriveService, + private userEntityService: UserEntityService, private queueLoggerService: QueueLoggerService, ) { this.logger = this.queueLoggerService.logger.createSubLogger('delete-drive-files'); @@ -62,7 +64,8 @@ export class DeleteDriveFilesProcessorService { cursor = files.at(-1)?.id ?? null; for (const file of files) { - await this.driveService.deleteFileSync(file); + const isRemote = file.user ? this.userEntityService.isRemoteUser(file.user) : false; + await this.driveService.deleteFileSync(file, false, isRemote); deletedCount++; } diff --git a/packages/backend/src/queue/processors/DeleteFileProcessorService.ts b/packages/backend/src/queue/processors/DeleteFileProcessorService.ts index edf87bd92174..e11d6ae57e5e 100644 --- a/packages/backend/src/queue/processors/DeleteFileProcessorService.ts +++ b/packages/backend/src/queue/processors/DeleteFileProcessorService.ts @@ -3,6 +3,8 @@ import { DI } from '@/di-symbols.js'; import type { Config } from '@/config.js'; import type Logger from '@/logger.js'; import { DriveService } from '@/core/DriveService.js'; +import { UserEntityService } from '@/core/entities/UserEntityService.js'; +import type { DriveFilesRepository } from '@/models/index.js'; import { bindThis } from '@/decorators.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; import type * as Bull from 'bullmq'; @@ -16,7 +18,11 @@ export class DeleteFileProcessorService { @Inject(DI.config) private config: Config, + @Inject(DI.driveFilesRepository) + private driveFilesRepository: DriveFilesRepository, + private driveService: DriveService, + private userEntityService: UserEntityService, private queueLoggerService: QueueLoggerService, ) { this.logger = this.queueLoggerService.logger.createSubLogger('delete-file'); @@ -26,7 +32,14 @@ export class DeleteFileProcessorService { public async process(job: Bull.Job): Promise { const key: string = job.data.key; - await this.driveService.deleteObjectStorageFile(key); + const file = await this.driveFilesRepository.createQueryBuilder('file') + .where('file.accessKey = :accessKey', { accessKey: key }) + .orWhere('file.thumbnailAccessKey = :thumbnailAccessKey', { thumbnailAccessKey: key }) + .orWhere('file.webpublicAccessKey = :webpublicAccessKey', { webpublicAccessKey: key }) + .getOne(); + const isRemote = file?.user ? this.userEntityService.isRemoteUser(file.user) : false; + + await this.driveService.deleteObjectStorageFile(key, isRemote); return 'Success'; } diff --git a/packages/backend/src/server/api/endpoints/admin/meta.ts b/packages/backend/src/server/api/endpoints/admin/meta.ts index 45e9677eeae3..ade389969256 100644 --- a/packages/backend/src/server/api/endpoints/admin/meta.ts +++ b/packages/backend/src/server/api/endpoints/admin/meta.ts @@ -249,6 +249,54 @@ export const meta = { type: 'boolean', optional: true, nullable: false, }, + useObjectStorageRemote: { + type: 'boolean', + optional: true, nullable: false, + }, + objectStorageRemoteBaseUrl: { + type: 'string', + optional: true, nullable: true, + }, + objectStorageRemoteBucket: { + type: 'string', + optional: true, nullable: true, + }, + objectStorageRemotePrefix: { + type: 'string', + optional: true, nullable: true, + }, + objectStorageRemoteEndpoint: { + type: 'string', + optional: true, nullable: true, + }, + objectStorageRemoteRegion: { + type: 'string', + optional: true, nullable: true, + }, + objectStorageRemotePort: { + type: 'number', + optional: true, nullable: true, + }, + objectStorageRemoteAccessKey: { + type: 'string', + optional: true, nullable: true, + }, + objectStorageRemoteSecretKey: { + type: 'string', + optional: true, nullable: true, + }, + objectStorageRemoteUseSSL: { + type: 'boolean', + optional: true, nullable: false, + }, + objectStorageRemoteUseProxy: { + type: 'boolean', + optional: true, nullable: false, + }, + objectStorageRemoteSetPublicRead: { + type: 'boolean', + optional: true, nullable: false, + }, enableIpLogging: { type: 'boolean', optional: true, nullable: false, @@ -370,6 +418,19 @@ export default class extends Endpoint { objectStorageUseProxy: instance.objectStorageUseProxy, objectStorageSetPublicRead: instance.objectStorageSetPublicRead, objectStorageS3ForcePathStyle: instance.objectStorageS3ForcePathStyle, + useObjectStorageRemote: instance.useObjectStorageRemote, + objectStorageRemoteBaseUrl: instance.objectStorageRemoteBaseUrl, + objectStorageRemoteBucket: instance.objectStorageRemoteBucket, + objectStorageRemotePrefix: instance.objectStorageRemotePrefix, + objectStorageRemoteEndpoint: instance.objectStorageRemoteEndpoint, + objectStorageRemoteRegion: instance.objectStorageRemoteRegion, + objectStorageRemotePort: instance.objectStorageRemotePort, + objectStorageRemoteAccessKey: instance.objectStorageRemoteAccessKey, + objectStorageRemoteSecretKey: instance.objectStorageRemoteSecretKey, + objectStorageRemoteUseSSL: instance.objectStorageRemoteUseSSL, + objectStorageRemoteUseProxy: instance.objectStorageRemoteUseProxy, + objectStorageRemoteSetPublicRead: instance.objectStorageRemoteSetPublicRead, + objectStorageRemoteS3ForcePathStyle: instance.objectStorageRemoteS3ForcePathStyle, translatorType: instance.translatorType, deeplAuthKey: instance.deeplAuthKey, deeplIsPro: instance.deeplIsPro, diff --git a/packages/backend/src/server/api/endpoints/admin/update-meta.ts b/packages/backend/src/server/api/endpoints/admin/update-meta.ts index b9830c804d1b..0855936208ff 100644 --- a/packages/backend/src/server/api/endpoints/admin/update-meta.ts +++ b/packages/backend/src/server/api/endpoints/admin/update-meta.ts @@ -99,6 +99,19 @@ export const paramDef = { objectStorageUseProxy: { type: 'boolean' }, objectStorageSetPublicRead: { type: 'boolean' }, objectStorageS3ForcePathStyle: { type: 'boolean' }, + useObjectStorageRemote: { type: 'boolean' }, + objectStorageRemoteBaseUrl: { type: 'string', nullable: true }, + objectStorageRemoteBucket: { type: 'string', nullable: true }, + objectStorageRemotePrefix: { type: 'string', nullable: true }, + objectStorageRemoteEndpoint: { type: 'string', nullable: true }, + objectStorageRemoteRegion: { type: 'string', nullable: true }, + objectStorageRemotePort: { type: 'integer', nullable: true }, + objectStorageRemoteAccessKey: { type: 'string', nullable: true }, + objectStorageRemoteSecretKey: { type: 'string', nullable: true }, + objectStorageRemoteUseSSL: { type: 'boolean' }, + objectStorageRemoteUseProxy: { type: 'boolean' }, + objectStorageRemoteSetPublicRead: { type: 'boolean' }, + objectStorageRemoteS3ForcePathStyle: { type: 'boolean' }, enableIpLogging: { type: 'boolean' }, enableActiveEmailValidation: { type: 'boolean' }, enableChartsForRemoteUser: { type: 'boolean' }, @@ -384,6 +397,58 @@ export default class extends Endpoint { set.objectStorageS3ForcePathStyle = ps.objectStorageS3ForcePathStyle; } + if (ps.useObjectStorageRemote !== undefined) { + set.useObjectStorageRemote = ps.useObjectStorageRemote; + } + + if (ps.objectStorageRemoteBaseUrl !== undefined) { + set.objectStorageRemoteBaseUrl = ps.objectStorageRemoteBaseUrl; + } + + if (ps.objectStorageRemoteBucket !== undefined) { + set.objectStorageRemoteBucket = ps.objectStorageRemoteBucket; + } + + if (ps.objectStorageRemotePrefix !== undefined) { + set.objectStorageRemotePrefix = ps.objectStorageRemotePrefix; + } + + if (ps.objectStorageRemoteEndpoint !== undefined) { + set.objectStorageRemoteEndpoint = ps.objectStorageRemoteEndpoint; + } + + if (ps.objectStorageRemoteRegion !== undefined) { + set.objectStorageRemoteRegion = ps.objectStorageRemoteRegion; + } + + if (ps.objectStorageRemotePort !== undefined) { + set.objectStorageRemotePort = ps.objectStorageRemotePort; + } + + if (ps.objectStorageRemoteAccessKey !== undefined) { + set.objectStorageRemoteAccessKey = ps.objectStorageRemoteAccessKey; + } + + if (ps.objectStorageRemoteSecretKey !== undefined) { + set.objectStorageRemoteSecretKey = ps.objectStorageRemoteSecretKey; + } + + if (ps.objectStorageRemoteUseSSL !== undefined) { + set.objectStorageRemoteUseSSL = ps.objectStorageRemoteUseSSL; + } + + if (ps.objectStorageRemoteUseProxy !== undefined) { + set.objectStorageRemoteUseProxy = ps.objectStorageRemoteUseProxy; + } + + if (ps.objectStorageRemoteSetPublicRead !== undefined) { + set.objectStorageRemoteSetPublicRead = ps.objectStorageRemoteSetPublicRead; + } + + if (ps.objectStorageRemoteS3ForcePathStyle !== undefined) { + set.objectStorageRemoteS3ForcePathStyle = ps.objectStorageRemoteS3ForcePathStyle; + } + if (ps.translatorType !== undefined) { set.translatorType = ps.translatorType; } diff --git a/packages/frontend/src/pages/admin/object-storage.vue b/packages/frontend/src/pages/admin/object-storage.vue index e569aad1b88c..5686886bb332 100644 --- a/packages/frontend/src/pages/admin/object-storage.vue +++ b/packages/frontend/src/pages/admin/object-storage.vue @@ -4,66 +4,139 @@
- {{ i18n.ts.useObjectStorage }} - - + + + +
+ {{ i18n.ts.useObjectStorage }} + + +
+ + + + + +
+ {{ i18n.ts.useObjectStorage }} + + +
+
@@ -80,6 +153,7 @@