From 4e2633588a22a60170c2e8bc0f12ebcd8057a106 Mon Sep 17 00:00:00 2001 From: caipira113 Date: Tue, 22 Oct 2024 00:16:45 +0900 Subject: [PATCH] feat: configure separate remote object storage --- .../1729518620697-remoteObjectStorage.js | 38 +++ packages/backend/src/core/DriveService.ts | 80 ++++-- packages/backend/src/core/S3Service.ts | 63 +++-- packages/backend/src/models/Meta.ts | 72 ++++++ .../CleanRemoteFilesProcessorService.ts | 2 +- .../DeleteAccountProcessorService.ts | 5 +- .../DeleteDriveFilesProcessorService.ts | 5 +- .../processors/DeleteFileProcessorService.ts | 18 +- .../src/server/api/endpoints/admin/meta.ts | 77 ++++++ .../server/api/endpoints/admin/update-meta.ts | 65 +++++ .../src/pages/admin/object-storage.vue | 239 +++++++++++++----- packages/misskey-js/src/autogen/types.ts | 26 ++ 12 files changed, 583 insertions(+), 107 deletions(-) create mode 100644 packages/backend/migration/1729518620697-remoteObjectStorage.js diff --git a/packages/backend/migration/1729518620697-remoteObjectStorage.js b/packages/backend/migration/1729518620697-remoteObjectStorage.js new file mode 100644 index 000000000..361eaedec --- /dev/null +++ b/packages/backend/migration/1729518620697-remoteObjectStorage.js @@ -0,0 +1,38 @@ +/* + * SPDX-FileCopyrightText: syuilo and misskey-project + * SPDX-License-Identifier: AGPL-3.0-only + */ + +export class RemoteObjectStorage1729518620697 { + async up(queryRunner) { + await queryRunner.query(`ALTER TABLE "meta" ADD COLUMN useRemoteObjectStorage BOOLEAN DEFAULT false;`); + await queryRunner.query(`ALTER TABLE "meta" ADD COLUMN remoteObjectStorageBucket VARCHAR(1024) NOT NULL;`); + await queryRunner.query(`ALTER TABLE "meta" ADD COLUMN remoteObjectStoragePrefix VARCHAR(1024) NOT NULL;`); + await queryRunner.query(`ALTER TABLE "meta" ADD COLUMN remoteObjectStorageBaseUrl VARCHAR(1024);`); + await queryRunner.query(`ALTER TABLE "meta" ADD COLUMN remoteObjectStorageEndpoint VARCHAR(1024);`); + await queryRunner.query(`ALTER TABLE "meta" ADD COLUMN remoteObjectStorageRegion VARCHAR(1024);`); + await queryRunner.query(`ALTER TABLE "meta" ADD COLUMN remoteObjectStorageAccessKey VARCHAR(1024);`); + await queryRunner.query(`ALTER TABLE "meta" ADD COLUMN remoteObjectStorageSecretKey VARCHAR(1024);`); + await queryRunner.query(`ALTER TABLE "meta" ADD COLUMN remoteObjectStoragePort INTEGER;`); + await queryRunner.query(`ALTER TABLE "meta" ADD COLUMN remoteObjectStorageUseSSL BOOLEAN DEFAULT true;`); + await queryRunner.query(`ALTER TABLE "meta" ADD COLUMN remoteObjectStorageUseProxy BOOLEAN DEFAULT true;`); + await queryRunner.query(`ALTER TABLE "meta" ADD COLUMN remoteObjectStorageSetPublicRead BOOLEAN DEFAULT false;`); + await queryRunner.query(`ALTER TABLE "meta" ADD COLUMN remoteObjectStorageS3ForcePathStyle BOOLEAN DEFAULT true;`); + } + + async down(queryRunner) { + await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN remoteObjectStorageS3ForcePathStyle;`); + await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN remoteObjectStorageSetPublicRead;`); + await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN remoteObjectStorageUseProxy;`); + await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN remoteObjectStorageUseSSL;`); + await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN remoteObjectStoragePort;`); + await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN remoteObjectStorageSecretKey;`); + await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN remoteObjectStorageAccessKey;`); + await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN remoteObjectStorageRegion;`); + await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN remoteObjectStorageEndpoint;`); + await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN remoteObjectStorageBaseUrl;`); + await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN remoteObjectStoragePrefix;`); + await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN remoteObjectStorageBucket;`); + await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN useRemoteObjectStorage;`); + } +} diff --git a/packages/backend/src/core/DriveService.ts b/packages/backend/src/core/DriveService.ts index c332e5a0a..5dadbf094 100644 --- a/packages/backend/src/core/DriveService.ts +++ b/packages/backend/src/core/DriveService.ts @@ -146,7 +146,7 @@ export class DriveService { * @param size Size for original */ @bindThis - private async save(file: MiDriveFile, path: string, name: string, type: string, hash: string, size: number): Promise { + private async save(file: MiDriveFile, path: string, name: string, type: string, hash: string, size: number, isRemote = false): Promise { // thunbnail, webpublic を必要なら生成 const alts = await this.generateAlts(path, type, !file.uri); @@ -169,11 +169,37 @@ export class DriveService { ext = ''; } - const baseUrl = this.meta.objectStorageBaseUrl - ?? `${ this.meta.objectStorageUseSSL ? 'https' : 'http' }://${ this.meta.objectStorageEndpoint }${ this.meta.objectStoragePort ? `:${this.meta.objectStoragePort}` : '' }/${ this.meta.objectStorageBucket }`; + const useRemoteObjectStorage = isRemote && this.meta.useRemoteObjectStorage; + + const objectStorageBaseUrl = useRemoteObjectStorage + ? this.meta.remoteObjectStorageBaseUrl + : this.meta.objectStorageBaseUrl; + + const objectStorageUseSSL = useRemoteObjectStorage + ? this.meta.remoteObjectStorageUseSSL + : this.meta.objectStorageUseSSL; + + const objectStorageEndpoint = useRemoteObjectStorage + ? this.meta.remoteObjectStorageEndpoint + : this.meta.objectStorageEndpoint; + + const objectStoragePort = useRemoteObjectStorage + ? this.meta.remoteObjectStoragePort + : this.meta.objectStoragePort; + + const objectStorageBucket = useRemoteObjectStorage + ? this.meta.remoteObjectStorageBucket + : this.meta.objectStorageBucket; + + const objectStoragePrefix = useRemoteObjectStorage + ? this.meta.remoteObjectStoragePrefix + : this.meta.objectStoragePrefix; + + const baseUrl = objectStorageBaseUrl + ?? `${objectStorageUseSSL ? 'https' : 'http'}://${objectStorageEndpoint}${objectStoragePort ? `:${objectStoragePort}` : ''}/${objectStorageBucket}`; // for original - const key = `${this.meta.objectStoragePrefix}/${randomUUID()}${ext}`; + const key = `${objectStoragePrefix}/${randomUUID()}${ext}`; const url = `${ baseUrl }/${ key }`; // for alts @@ -186,7 +212,7 @@ export class DriveService { //#region Uploads this.registerLogger.info(`uploading original: ${key}`); const uploads = [ - this.upload(key, fs.createReadStream(path), type, null, name), + this.upload(key, fs.createReadStream(path), type, isRemote, null, name), ]; if (alts.webpublic) { @@ -194,7 +220,7 @@ export class DriveService { webpublicUrl = `${ baseUrl }/${ webpublicKey }`; this.registerLogger.info(`uploading webpublic: ${webpublicKey}`); - uploads.push(this.upload(webpublicKey, alts.webpublic.data, alts.webpublic.type, alts.webpublic.ext, name)); + uploads.push(this.upload(webpublicKey, alts.webpublic.data, alts.webpublic.type, isRemote, alts.webpublic.ext, name)); } if (alts.thumbnail) { @@ -202,7 +228,7 @@ export class DriveService { thumbnailUrl = `${ baseUrl }/${ thumbnailKey }`; this.registerLogger.info(`uploading thumbnail: ${thumbnailKey}`); - uploads.push(this.upload(thumbnailKey, alts.thumbnail.data, alts.thumbnail.type, alts.thumbnail.ext, `${name}.thumbnail`)); + uploads.push(this.upload(thumbnailKey, alts.thumbnail.data, alts.thumbnail.type, isRemote, alts.thumbnail.ext, `${name}.thumbnail`)); } await Promise.all(uploads); @@ -371,12 +397,22 @@ export class DriveService { * Upload to ObjectStorage */ @bindThis - private async upload(key: string, stream: fs.ReadStream | Buffer, type: string, ext?: string | null, filename?: string) { + private async upload(key: string, stream: fs.ReadStream | Buffer, type: string, isRemote = false, ext?: string | null, filename?: string) { if (type === 'image/apng') type = 'image/png'; if (!FILE_TYPE_BROWSERSAFE.includes(type)) type = 'application/octet-stream'; + const useRemoteObjectStorage = isRemote && this.meta.useRemoteObjectStorage; + + const objectStorageBucket = useRemoteObjectStorage + ? this.meta.remoteObjectStorageBucket + : this.meta.objectStorageBucket; + + const objectStorageSetPublicRead = useRemoteObjectStorage + ? this.meta.remoteObjectStorageSetPublicRead + : this.meta.objectStorageSetPublicRead; + const params = { - Bucket: this.meta.objectStorageBucket, + Bucket: objectStorageBucket, Key: key, Body: stream, ContentType: type, @@ -389,9 +425,9 @@ export class DriveService { // 許可されているファイル形式でしか拡張子をつけない ext ? correctFilename(filename, ext) : filename, ); - if (this.meta.objectStorageSetPublicRead) params.ACL = 'public-read'; + if (objectStorageSetPublicRead) params.ACL = 'public-read'; - await this.s3Service.upload(this.meta, params) + await this.s3Service.upload(this.meta, params, isRemote) .then( result => { if ('Bucket' in result) { // CompleteMultipartUploadCommandOutput @@ -630,7 +666,8 @@ export class DriveService { } } } else { - file = await (this.save(file, path, detectedName, info.type.mime, info.md5, info.size)); + const isRemote = user ? this.userEntityService.isRemoteUser(user) : false; + file = await (this.save(file, path, detectedName, info.type.mime, info.md5, info.size, isRemote)); } this.registerLogger.succ(`drive file has been created ${file.id}`); @@ -740,7 +777,7 @@ export class DriveService { } @bindThis - public async deleteFileSync(file: MiDriveFile, isExpired = false, deleter?: MiUser) { + public async deleteFileSync(file: MiDriveFile, isExpired = false, isRemote = false, deleter?: MiUser) { if (file.storedInternal) { this.internalStorageService.del(file.accessKey!); @@ -754,14 +791,14 @@ export class DriveService { } else if (!file.isLink) { const promises = []; - promises.push(this.deleteObjectStorageFile(file.accessKey!)); + promises.push(this.deleteObjectStorageFile(file.accessKey!, isRemote)); if (file.thumbnailUrl) { - promises.push(this.deleteObjectStorageFile(file.thumbnailAccessKey!)); + promises.push(this.deleteObjectStorageFile(file.thumbnailAccessKey!, isRemote)); } if (file.webpublicUrl) { - promises.push(this.deleteObjectStorageFile(file.webpublicAccessKey!)); + promises.push(this.deleteObjectStorageFile(file.webpublicAccessKey!, isRemote)); } await Promise.all(promises); @@ -815,14 +852,19 @@ export class DriveService { } @bindThis - public async deleteObjectStorageFile(key: string) { + public async deleteObjectStorageFile(key: string, isRemote = false) { + const useRemoteObjectStorage = isRemote && this.meta.useRemoteObjectStorage; + const objectStorageBucket = useRemoteObjectStorage + ? this.meta.remoteObjectStorageBucket + : this.meta.objectStorageBucket; + try { const param = { - Bucket: this.meta.objectStorageBucket, + Bucket: objectStorageBucket, Key: key, } as DeleteObjectCommandInput; - await this.s3Service.delete(this.meta, param); + await this.s3Service.delete(this.meta, param, isRemote); } catch (err: any) { if (err.name === 'NoSuchKey') { this.deleteLogger.warn(`The object storage had no such key to delete: ${key}. Skipping this.`, err as Error); diff --git a/packages/backend/src/core/S3Service.ts b/packages/backend/src/core/S3Service.ts index bb2a46335..a72b16b74 100644 --- a/packages/backend/src/core/S3Service.ts +++ b/packages/backend/src/core/S3Service.ts @@ -19,39 +19,64 @@ import type { DeleteObjectCommandInput, PutObjectCommandInput } from '@aws-sdk/c export class S3Service { constructor( private httpRequestService: HttpRequestService, - ) { - } + ) {} @bindThis - public getS3Client(meta: MiMeta): S3Client { - const u = meta.objectStorageEndpoint - ? `${meta.objectStorageUseSSL ? 'https' : 'http'}://${meta.objectStorageEndpoint}` - : `${meta.objectStorageUseSSL ? 'https' : 'http'}://example.net`; // dummy url to select http(s) agent + public getS3Client(meta: MiMeta, isRemote = false): S3Client { + const useRemoteObjectStorage = isRemote && meta.useRemoteObjectStorage; - const agent = this.httpRequestService.getAgentByUrl(new URL(u), !meta.objectStorageUseProxy); + const objectStorageEndpoint = useRemoteObjectStorage + ? meta.remoteObjectStorageEndpoint + : meta.objectStorageEndpoint; + + const objectStorageUseSSL = useRemoteObjectStorage + ? meta.remoteObjectStorageUseSSL + : meta.objectStorageUseSSL; + + const objectStorageAccessKey = useRemoteObjectStorage + ? meta.remoteObjectStorageAccessKey + : meta.objectStorageAccessKey; + + const objectStorageSecretKey = useRemoteObjectStorage + ? meta.remoteObjectStorageSecretKey + : meta.objectStorageSecretKey; + + const objectStorageRegion = useRemoteObjectStorage + ? meta.remoteObjectStorageRegion + : meta.objectStorageRegion; + + const objectStorageS3ForcePathStyle = useRemoteObjectStorage + ? meta.remoteObjectStorageS3ForcePathStyle + : meta.objectStorageS3ForcePathStyle; + + const u = objectStorageEndpoint + ? `${objectStorageUseSSL ? 'https' : 'http'}://${objectStorageEndpoint}` + : `${objectStorageUseSSL ? 'https' : 'http'}://example.net`; // dummy url to select http(s) agent + + const agent = this.httpRequestService.getAgentByUrl(new URL(u), !objectStorageUseSSL); const handlerOption: NodeHttpHandlerOptions = {}; - if (meta.objectStorageUseSSL) { + if (objectStorageUseSSL) { handlerOption.httpsAgent = agent as https.Agent; } else { handlerOption.httpAgent = agent as http.Agent; } return new S3Client({ - endpoint: meta.objectStorageEndpoint ? u : undefined, - credentials: (meta.objectStorageAccessKey !== null && meta.objectStorageSecretKey !== null) ? { - accessKeyId: meta.objectStorageAccessKey, - secretAccessKey: meta.objectStorageSecretKey, + endpoint: objectStorageEndpoint ? u : undefined, + credentials: (objectStorageAccessKey !== null && objectStorageSecretKey !== null) ? { + accessKeyId: objectStorageAccessKey, + secretAccessKey: objectStorageSecretKey, } : undefined, - region: meta.objectStorageRegion ? meta.objectStorageRegion : undefined, // 空文字列もundefinedにするため ?? は使わない - tls: meta.objectStorageUseSSL, - forcePathStyle: meta.objectStorageEndpoint ? meta.objectStorageS3ForcePathStyle : false, // AWS with endPoint omitted + region: objectStorageRegion || undefined, // 空文字列もundefinedにするため ?? は使わない + tls: objectStorageUseSSL, + forcePathStyle: objectStorageEndpoint ? objectStorageS3ForcePathStyle : false, // AWS with endPoint omitted requestHandler: new NodeHttpHandler(handlerOption), }); } @bindThis - public async upload(meta: MiMeta, input: PutObjectCommandInput) { - const client = this.getS3Client(meta); + public async upload(meta: MiMeta, input: PutObjectCommandInput, isRemote = false) { + const client = this.getS3Client(meta, isRemote); return new Upload({ client, params: input, @@ -62,8 +87,8 @@ export class S3Service { } @bindThis - public delete(meta: MiMeta, input: DeleteObjectCommandInput) { - const client = this.getS3Client(meta); + public delete(meta: MiMeta, input: DeleteObjectCommandInput, isRemote = false) { + const client = this.getS3Client(meta, isRemote); return client.send(new DeleteObjectCommand(input)); } } diff --git a/packages/backend/src/models/Meta.ts b/packages/backend/src/models/Meta.ts index ad5e31ad6..62c94e233 100644 --- a/packages/backend/src/models/Meta.ts +++ b/packages/backend/src/models/Meta.ts @@ -481,6 +481,78 @@ export class MiMeta { }) public objectStorageS3ForcePathStyle: boolean; + @Column('boolean', { + default: false, + }) + public useRemoteObjectStorage: boolean; + + @Column('varchar', { + length: 1024, + nullable: true, + }) + public remoteObjectStorageBucket: string | null; + + @Column('varchar', { + length: 1024, + nullable: true, + }) + public remoteObjectStoragePrefix: string | null; + + @Column('varchar', { + length: 1024, + nullable: true, + }) + public remoteObjectStorageBaseUrl: string | null; + + @Column('varchar', { + length: 1024, + nullable: true, + }) + public remoteObjectStorageEndpoint: string | null; + + @Column('varchar', { + length: 1024, + nullable: true, + }) + public remoteObjectStorageRegion: string | null; + + @Column('varchar', { + length: 1024, + nullable: true, + }) + public remoteObjectStorageAccessKey: string | null; + + @Column('varchar', { + length: 1024, + nullable: true, + }) + public remoteObjectStorageSecretKey: string | null; + + @Column('integer', { + nullable: true, + }) + public remoteObjectStoragePort: number | null; + + @Column('boolean', { + default: true, + }) + public remoteObjectStorageUseSSL: boolean; + + @Column('boolean', { + default: true, + }) + public remoteObjectStorageUseProxy: boolean; + + @Column('boolean', { + default: false, + }) + public remoteObjectStorageSetPublicRead: boolean; + + @Column('boolean', { + default: true, + }) + public remoteObjectStorageS3ForcePathStyle: boolean; + @Column('boolean', { default: false, }) diff --git a/packages/backend/src/queue/processors/CleanRemoteFilesProcessorService.ts b/packages/backend/src/queue/processors/CleanRemoteFilesProcessorService.ts index 728fc9e72..1c1739a7b 100644 --- a/packages/backend/src/queue/processors/CleanRemoteFilesProcessorService.ts +++ b/packages/backend/src/queue/processors/CleanRemoteFilesProcessorService.ts @@ -54,7 +54,7 @@ export class CleanRemoteFilesProcessorService { cursor = files.at(-1)?.id ?? null; - await Promise.all(files.map(file => this.driveService.deleteFileSync(file, true))); + await Promise.all(files.map(file => this.driveService.deleteFileSync(file, true, true))); deletedCount += 8; diff --git a/packages/backend/src/queue/processors/DeleteAccountProcessorService.ts b/packages/backend/src/queue/processors/DeleteAccountProcessorService.ts index 14a53e0c4..9aaac2c6e 100644 --- a/packages/backend/src/queue/processors/DeleteAccountProcessorService.ts +++ b/packages/backend/src/queue/processors/DeleteAccountProcessorService.ts @@ -17,6 +17,7 @@ import { SearchService } from '@/core/SearchService.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; import type * as Bull from 'bullmq'; import type { DbUserDeleteJobData } from '../types.js'; +import { UserEntityService } from '@/core/entities/UserEntityService.js'; @Injectable() export class DeleteAccountProcessorService { @@ -35,6 +36,7 @@ export class DeleteAccountProcessorService { @Inject(DI.driveFilesRepository) private driveFilesRepository: DriveFilesRepository, + private userEntityService: UserEntityService, private driveService: DriveService, private emailService: EmailService, private queueLoggerService: QueueLoggerService, @@ -48,6 +50,7 @@ export class DeleteAccountProcessorService { this.logger.info(`Deleting account of ${job.data.user.id} ...`); const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); + const isRemote = user ? this.userEntityService.isRemoteUser(user) : false; if (user == null) { return; } @@ -105,7 +108,7 @@ export class DeleteAccountProcessorService { cursor = files.at(-1)?.id ?? null; for (const file of files) { - await this.driveService.deleteFileSync(file); + await this.driveService.deleteFileSync(file, undefined, isRemote); } } diff --git a/packages/backend/src/queue/processors/DeleteDriveFilesProcessorService.ts b/packages/backend/src/queue/processors/DeleteDriveFilesProcessorService.ts index 291fa4a6d..3b6e401c9 100644 --- a/packages/backend/src/queue/processors/DeleteDriveFilesProcessorService.ts +++ b/packages/backend/src/queue/processors/DeleteDriveFilesProcessorService.ts @@ -13,6 +13,7 @@ import { bindThis } from '@/decorators.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; import type * as Bull from 'bullmq'; import type { DbJobDataWithUser } from '../types.js'; +import { UserEntityService } from "@/core/entities/UserEntityService.js"; @Injectable() export class DeleteDriveFilesProcessorService { @@ -25,6 +26,7 @@ export class DeleteDriveFilesProcessorService { @Inject(DI.driveFilesRepository) private driveFilesRepository: DriveFilesRepository, + private userEntityService: UserEntityService, private driveService: DriveService, private queueLoggerService: QueueLoggerService, ) { @@ -36,6 +38,7 @@ export class DeleteDriveFilesProcessorService { this.logger.info(`Deleting drive files of ${job.data.user.id} ...`); const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); + const isRemote = user ? this.userEntityService.isRemoteUser(user) : false; if (user == null) { return; } @@ -63,7 +66,7 @@ export class DeleteDriveFilesProcessorService { cursor = files.at(-1)?.id ?? null; for (const file of files) { - await this.driveService.deleteFileSync(file); + await this.driveService.deleteFileSync(file, undefined, isRemote); deletedCount++; } diff --git a/packages/backend/src/queue/processors/DeleteFileProcessorService.ts b/packages/backend/src/queue/processors/DeleteFileProcessorService.ts index fc1dd93ce..03f350eba 100644 --- a/packages/backend/src/queue/processors/DeleteFileProcessorService.ts +++ b/packages/backend/src/queue/processors/DeleteFileProcessorService.ts @@ -3,19 +3,26 @@ * SPDX-License-Identifier: AGPL-3.0-only */ -import { Injectable } from '@nestjs/common'; +import { Injectable, Inject } from '@nestjs/common'; import type Logger from '@/logger.js'; import { DriveService } from '@/core/DriveService.js'; import { bindThis } from '@/decorators.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; import type * as Bull from 'bullmq'; import type { ObjectStorageFileJobData } from '../types.js'; +import type { DriveFilesRepository } from "@/models/_.js"; +import { UserEntityService } from "@/core/entities/UserEntityService.js"; +import { DI } from '@/di-symbols.js'; @Injectable() export class DeleteFileProcessorService { private logger: Logger; constructor( + @Inject(DI.driveFilesRepository) + private driveFilesRepository: DriveFilesRepository, + + private userEntityService: UserEntityService, private driveService: DriveService, private queueLoggerService: QueueLoggerService, ) { @@ -26,7 +33,14 @@ export class DeleteFileProcessorService { public async process(job: Bull.Job): Promise { const key: string = job.data.key; - await this.driveService.deleteObjectStorageFile(key); + const file = await this.driveFilesRepository.createQueryBuilder('file') + .where('file.accessKey = :accessKey', { accessKey: key }) + .orWhere('file.thumbnailAccessKey = :thumbnailAccessKey', { thumbnailAccessKey: key }) + .orWhere('file.webpublicAccessKey = :webpublicAccessKey', { webpublicAccessKey: key }) + .getOne(); + const isRemote = file?.user ? this.userEntityService.isRemoteUser(file.user) : false; + + await this.driveService.deleteObjectStorageFile(key, isRemote); return 'Success'; } diff --git a/packages/backend/src/server/api/endpoints/admin/meta.ts b/packages/backend/src/server/api/endpoints/admin/meta.ts index 64e3cc33b..30cc7dc2e 100644 --- a/packages/backend/src/server/api/endpoints/admin/meta.ts +++ b/packages/backend/src/server/api/endpoints/admin/meta.ts @@ -312,6 +312,66 @@ export const meta = { type: 'boolean', optional: false, nullable: false, }, + useRemoteObjectStorage: { + type: 'boolean', + optional: false, + nullable: false, + }, + remoteObjectStorageBaseUrl: { + type: 'string', + optional: false, + nullable: true, + }, + remoteObjectStorageBucket: { + type: 'string', + optional: false, + nullable: true, + }, + remoteObjectStoragePrefix: { + type: 'string', + optional: false, + nullable: true, + }, + remoteObjectStorageEndpoint: { + type: 'string', + optional: false, + nullable: true, + }, + remoteObjectStorageRegion: { + type: 'string', + optional: false, + nullable: true, + }, + remoteObjectStoragePort: { + type: 'number', + optional: false, + nullable: true, + }, + remoteObjectStorageAccessKey: { + type: 'string', + optional: false, + nullable: true, + }, + remoteObjectStorageSecretKey: { + type: 'string', + optional: false, + nullable: true, + }, + remoteObjectStorageUseSSL: { + type: 'boolean', + optional: false, + nullable: false, + }, + remoteObjectStorageUseProxy: { + type: 'boolean', + optional: false, + nullable: false, + }, + remoteObjectStorageSetPublicRead: { + type: 'boolean', + optional: false, + nullable: false, + }, enableIpLogging: { type: 'boolean', optional: false, nullable: false, @@ -452,6 +512,10 @@ export const meta = { type: 'boolean', optional: false, nullable: false, }, + remoteObjectStorageS3ForcePathStyle: { + type: 'boolean', + optional: false, nullable: false, + }, privacyPolicyUrl: { type: 'string', optional: false, nullable: true, @@ -628,6 +692,19 @@ export default class extends Endpoint { // eslint- objectStorageUseProxy: instance.objectStorageUseProxy, objectStorageSetPublicRead: instance.objectStorageSetPublicRead, objectStorageS3ForcePathStyle: instance.objectStorageS3ForcePathStyle, + useRemoteObjectStorage: instance.useRemoteObjectStorage, + remoteObjectStorageBaseUrl: instance.remoteObjectStorageBaseUrl, + remoteObjectStorageBucket: instance.remoteObjectStorageBucket, + remoteObjectStoragePrefix: instance.remoteObjectStoragePrefix, + remoteObjectStorageEndpoint: instance.remoteObjectStorageEndpoint, + remoteObjectStorageRegion: instance.remoteObjectStorageRegion, + remoteObjectStoragePort: instance.remoteObjectStoragePort, + remoteObjectStorageAccessKey: instance.remoteObjectStorageAccessKey, + remoteObjectStorageSecretKey: instance.remoteObjectStorageSecretKey, + remoteObjectStorageUseSSL: instance.remoteObjectStorageUseSSL, + remoteObjectStorageUseProxy: instance.remoteObjectStorageUseProxy, + remoteObjectStorageSetPublicRead: instance.remoteObjectStorageSetPublicRead, + remoteObjectStorageS3ForcePathStyle: instance.remoteObjectStorageS3ForcePathStyle, deeplAuthKey: instance.deeplAuthKey, deeplIsPro: instance.deeplIsPro, enableIpLogging: instance.enableIpLogging, diff --git a/packages/backend/src/server/api/endpoints/admin/update-meta.ts b/packages/backend/src/server/api/endpoints/admin/update-meta.ts index 38ef0d1de..f71f7cdfc 100644 --- a/packages/backend/src/server/api/endpoints/admin/update-meta.ts +++ b/packages/backend/src/server/api/endpoints/admin/update-meta.ts @@ -127,6 +127,19 @@ export const paramDef = { objectStorageUseProxy: { type: 'boolean' }, objectStorageSetPublicRead: { type: 'boolean' }, objectStorageS3ForcePathStyle: { type: 'boolean' }, + useRemoteObjectStorage: { type: 'boolean' }, + remoteObjectStorageBaseUrl: { type: 'string', nullable: true }, + remoteObjectStorageBucket: { type: 'string', nullable: true }, + remoteObjectStoragePrefix: { type: 'string', nullable: true }, + remoteObjectStorageEndpoint: { type: 'string', nullable: true }, + remoteObjectStorageRegion: { type: 'string', nullable: true }, + remoteObjectStoragePort: { type: 'integer', nullable: true }, + remoteObjectStorageAccessKey: { type: 'string', nullable: true }, + remoteObjectStorageSecretKey: { type: 'string', nullable: true }, + remoteObjectStorageUseSSL: { type: 'boolean' }, + remoteObjectStorageUseProxy: { type: 'boolean' }, + remoteObjectStorageSetPublicRead: { type: 'boolean' }, + remoteObjectStorageS3ForcePathStyle: { type: 'boolean' }, enableIpLogging: { type: 'boolean' }, enableActiveEmailValidation: { type: 'boolean' }, enableVerifymailApi: { type: 'boolean' }, @@ -519,6 +532,58 @@ export default class extends Endpoint { // eslint- set.objectStorageS3ForcePathStyle = ps.objectStorageS3ForcePathStyle; } + if (ps.useRemoteObjectStorage !== undefined) { + set.useRemoteObjectStorage = ps.useRemoteObjectStorage; + } + + if (ps.remoteObjectStorageBaseUrl !== undefined) { + set.remoteObjectStorageBaseUrl = ps.remoteObjectStorageBaseUrl; + } + + if (ps.remoteObjectStorageBucket !== undefined) { + set.remoteObjectStorageBucket = ps.remoteObjectStorageBucket; + } + + if (ps.remoteObjectStoragePrefix !== undefined) { + set.remoteObjectStoragePrefix = ps.remoteObjectStoragePrefix; + } + + if (ps.remoteObjectStorageEndpoint !== undefined) { + set.remoteObjectStorageEndpoint = ps.remoteObjectStorageEndpoint; + } + + if (ps.remoteObjectStorageRegion !== undefined) { + set.remoteObjectStorageRegion = ps.remoteObjectStorageRegion; + } + + if (ps.remoteObjectStoragePort !== undefined) { + set.remoteObjectStoragePort = ps.remoteObjectStoragePort; + } + + if (ps.remoteObjectStorageAccessKey !== undefined) { + set.remoteObjectStorageAccessKey = ps.remoteObjectStorageAccessKey; + } + + if (ps.remoteObjectStorageSecretKey !== undefined) { + set.remoteObjectStorageSecretKey = ps.remoteObjectStorageSecretKey; + } + + if (ps.remoteObjectStorageUseSSL !== undefined) { + set.remoteObjectStorageUseSSL = ps.remoteObjectStorageUseSSL; + } + + if (ps.remoteObjectStorageUseProxy !== undefined) { + set.remoteObjectStorageUseProxy = ps.remoteObjectStorageUseProxy; + } + + if (ps.remoteObjectStorageSetPublicRead !== undefined) { + set.remoteObjectStorageSetPublicRead = ps.remoteObjectStorageSetPublicRead; + } + + if (ps.remoteObjectStorageS3ForcePathStyle !== undefined) { + set.remoteObjectStorageS3ForcePathStyle = ps.remoteObjectStorageS3ForcePathStyle; + } + if (ps.deeplAuthKey !== undefined) { if (ps.deeplAuthKey === '') { set.deeplAuthKey = null; diff --git a/packages/frontend/src/pages/admin/object-storage.vue b/packages/frontend/src/pages/admin/object-storage.vue index d5a664934..8cf8668cd 100644 --- a/packages/frontend/src/pages/admin/object-storage.vue +++ b/packages/frontend/src/pages/admin/object-storage.vue @@ -4,82 +4,153 @@ SPDX-License-Identifier: AGPL-3.0-only -->