mirror of
https://github.com/kokonect-link/cherrypick
synced 2024-10-30 14:51:48 +09:00
リモートファイルの削除が重い問題を修正
This commit is contained in:
parent
ed0070c470
commit
03e999875a
@ -194,6 +194,13 @@ export function createDeleteObjectStorageFileJob(key: string) {
|
||||
});
|
||||
}
|
||||
|
||||
export function createCleanRemoteFilesJob() {
|
||||
return objectStorageQueue.add('cleanRemoteFiles', {}, {
|
||||
removeOnComplete: true,
|
||||
removeOnFail: true
|
||||
});
|
||||
}
|
||||
|
||||
export default function() {
|
||||
if (!program.onlyServer) {
|
||||
deliverQueue.process(128, processDeliver);
|
||||
|
@ -1,7 +1,7 @@
|
||||
import * as Bull from 'bull';
|
||||
|
||||
import { queueLogger } from '../../logger';
|
||||
import { deleteFile } from '../../../services/drive/delete-file';
|
||||
import { deleteFileSync } from '../../../services/drive/delete-file';
|
||||
import { Users, DriveFiles } from '../../../models';
|
||||
import { MoreThan } from 'typeorm';
|
||||
|
||||
@ -39,7 +39,7 @@ export async function deleteDriveFiles(job: Bull.Job, done: any): Promise<void>
|
||||
cursor = files[files.length - 1].id;
|
||||
|
||||
for (const file of files) {
|
||||
await deleteFile(file);
|
||||
await deleteFileSync(file);
|
||||
deletedCount++;
|
||||
}
|
||||
|
||||
|
50
src/queue/processors/object-storage/clean-remote-files.ts
Normal file
50
src/queue/processors/object-storage/clean-remote-files.ts
Normal file
@ -0,0 +1,50 @@
|
||||
import * as Bull from 'bull';
|
||||
|
||||
import { queueLogger } from '../../logger';
|
||||
import { deleteFileSync } from '../../../services/drive/delete-file';
|
||||
import { DriveFiles } from '../../../models';
|
||||
import { MoreThan, Not, IsNull } from 'typeorm';
|
||||
|
||||
const logger = queueLogger.createSubLogger('clean-remote-files');
|
||||
|
||||
export default async function cleanRemoteFiles(job: Bull.Job, done: any): Promise<void> {
|
||||
logger.info(`Deleting cached remote files...`);
|
||||
|
||||
let deletedCount = 0;
|
||||
let cursor: any = null;
|
||||
|
||||
while (true) {
|
||||
const files = await DriveFiles.find({
|
||||
where: {
|
||||
userHost: Not(IsNull()),
|
||||
isLink: false,
|
||||
...(cursor ? { id: MoreThan(cursor) } : {})
|
||||
},
|
||||
take: 8,
|
||||
order: {
|
||||
id: 1
|
||||
}
|
||||
});
|
||||
|
||||
if (files.length === 0) {
|
||||
job.progress(100);
|
||||
break;
|
||||
}
|
||||
|
||||
cursor = files[files.length - 1].id;
|
||||
|
||||
await Promise.all(files.map(file => deleteFileSync(file, true)));
|
||||
|
||||
deletedCount += 8;
|
||||
|
||||
const total = await DriveFiles.count({
|
||||
userHost: Not(IsNull()),
|
||||
isLink: false,
|
||||
});
|
||||
|
||||
job.progress(deletedCount / total);
|
||||
}
|
||||
|
||||
logger.succ(`All cahced remote files has been deleted.`);
|
||||
done();
|
||||
}
|
@ -1,22 +1,10 @@
|
||||
import * as Bull from 'bull';
|
||||
import * as Minio from 'minio';
|
||||
import { fetchMeta } from '../../../misc/fetch-meta';
|
||||
import { deleteObjectStorageFile } from '../../../services/drive/delete-file';
|
||||
|
||||
export default async (job: Bull.Job) => {
|
||||
const meta = await fetchMeta();
|
||||
|
||||
const minio = new Minio.Client({
|
||||
endPoint: meta.objectStorageEndpoint!,
|
||||
region: meta.objectStorageRegion ? meta.objectStorageRegion : undefined,
|
||||
port: meta.objectStoragePort ? meta.objectStoragePort : undefined,
|
||||
useSSL: meta.objectStorageUseSSL,
|
||||
accessKey: meta.objectStorageAccessKey!,
|
||||
secretKey: meta.objectStorageSecretKey!,
|
||||
});
|
||||
|
||||
const key: string = job.data.key;
|
||||
|
||||
await minio.removeObject(meta.objectStorageBucket!, key);
|
||||
await deleteObjectStorageFile(key);
|
||||
|
||||
return 'Success';
|
||||
};
|
||||
|
@ -1,8 +1,10 @@
|
||||
import * as Bull from 'bull';
|
||||
import deleteFile from './delete-file';
|
||||
import cleanRemoteFiles from './clean-remote-files';
|
||||
|
||||
const jobs = {
|
||||
deleteFile,
|
||||
cleanRemoteFiles,
|
||||
} as any;
|
||||
|
||||
export default function(q: Bull.Queue) {
|
||||
|
@ -1,7 +1,5 @@
|
||||
import { Not, IsNull } from 'typeorm';
|
||||
import define from '../../../define';
|
||||
import { deleteFile } from '../../../../../services/drive/delete-file';
|
||||
import { DriveFiles } from '../../../../../models';
|
||||
import { createCleanRemoteFilesJob } from '../../../../../queue';
|
||||
|
||||
export const meta = {
|
||||
tags: ['admin'],
|
||||
@ -11,12 +9,5 @@ export const meta = {
|
||||
};
|
||||
|
||||
export default define(meta, async (ps, me) => {
|
||||
const files = await DriveFiles.find({
|
||||
userHost: Not(IsNull()),
|
||||
isLink: false,
|
||||
});
|
||||
|
||||
for (const file of files) {
|
||||
deleteFile(file, true);
|
||||
}
|
||||
createCleanRemoteFilesJob();
|
||||
});
|
||||
|
@ -1,8 +1,10 @@
|
||||
import * as Minio from 'minio';
|
||||
import { DriveFile } from '../../models/entities/drive-file';
|
||||
import { InternalStorage } from './internal-storage';
|
||||
import { DriveFiles, Instances, Notes } from '../../models';
|
||||
import { driveChart, perUserDriveChart, instanceChart } from '../chart';
|
||||
import { createDeleteObjectStorageFileJob } from '../../queue';
|
||||
import { fetchMeta } from '../../misc/fetch-meta';
|
||||
|
||||
export async function deleteFile(file: DriveFile, isExpired = false) {
|
||||
if (file.storedInternal) {
|
||||
@ -27,6 +29,40 @@ export async function deleteFile(file: DriveFile, isExpired = false) {
|
||||
}
|
||||
}
|
||||
|
||||
postProcess(file, isExpired);
|
||||
}
|
||||
|
||||
export async function deleteFileSync(file: DriveFile, isExpired = false) {
|
||||
if (file.storedInternal) {
|
||||
InternalStorage.del(file.accessKey!);
|
||||
|
||||
if (file.thumbnailUrl) {
|
||||
InternalStorage.del(file.thumbnailAccessKey!);
|
||||
}
|
||||
|
||||
if (file.webpublicUrl) {
|
||||
InternalStorage.del(file.webpublicAccessKey!);
|
||||
}
|
||||
} else if (!file.isLink) {
|
||||
const promises = [];
|
||||
|
||||
promises.push(deleteObjectStorageFile(file.accessKey!));
|
||||
|
||||
if (file.thumbnailUrl) {
|
||||
promises.push(deleteObjectStorageFile(file.thumbnailAccessKey!));
|
||||
}
|
||||
|
||||
if (file.webpublicUrl) {
|
||||
promises.push(deleteObjectStorageFile(file.webpublicAccessKey!));
|
||||
}
|
||||
|
||||
await Promise.all(promises);
|
||||
}
|
||||
|
||||
postProcess(file, isExpired);
|
||||
}
|
||||
|
||||
function postProcess(file: DriveFile, isExpired = false) {
|
||||
// リモートファイル期限切れ削除後は直リンクにする
|
||||
if (isExpired && file.userHost !== null && file.uri != null) {
|
||||
DriveFiles.update(file.id, {
|
||||
@ -53,3 +89,18 @@ export async function deleteFile(file: DriveFile, isExpired = false) {
|
||||
Instances.decrement({ host: file.userHost }, 'driveFiles', 1);
|
||||
}
|
||||
}
|
||||
|
||||
export async function deleteObjectStorageFile(key: string) {
|
||||
const meta = await fetchMeta();
|
||||
|
||||
const minio = new Minio.Client({
|
||||
endPoint: meta.objectStorageEndpoint!,
|
||||
region: meta.objectStorageRegion ? meta.objectStorageRegion : undefined,
|
||||
port: meta.objectStoragePort ? meta.objectStoragePort : undefined,
|
||||
useSSL: meta.objectStorageUseSSL,
|
||||
accessKey: meta.objectStorageAccessKey!,
|
||||
secretKey: meta.objectStorageSecretKey!,
|
||||
});
|
||||
|
||||
await minio.removeObject(meta.objectStorageBucket!, key);
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user