feat: queueing bulk follow/unfollow and block/unblock (#10544)
* wrap follow/unfollow and block/unblock as job queue * create import job to follow in each iteration * make relationship jobs concurrent * replace to job queue if called repeatedly * use addBulk to import * omit stream when importing * fix job caller * use ThinUser instead of User to reduce redis memory consumption * createImportFollowingToDbJobの呼び出し方を変える, 型補強 * Force ThinUser * オブジェクト操作のみのメソッド名はgenerate...Data * Force ThinUser in generateRelationshipJobData * silent bulk unfollow at admin api endpoint --------- Co-authored-by: tamaina <tamaina@hotmail.co.jp>
This commit is contained in:
Namekuji
GitHub
parent
b463490d9f
commit
da83322200
23 changed files with 418 additions and 186 deletions
|
|
@ -13,6 +13,7 @@ import { EndedPollNotificationProcessorService } from './processors/EndedPollNot
|
|||
import { DeliverProcessorService } from './processors/DeliverProcessorService.js';
|
||||
import { InboxProcessorService } from './processors/InboxProcessorService.js';
|
||||
import { QueueLoggerService } from './QueueLoggerService.js';
|
||||
import { RelationshipQueueProcessorsService } from './RelationshipQueueProcessorsService.js';
|
||||
|
||||
@Injectable()
|
||||
export class QueueProcessorService {
|
||||
|
|
@ -27,6 +28,7 @@ export class QueueProcessorService {
|
|||
private systemQueueProcessorsService: SystemQueueProcessorsService,
|
||||
private objectStorageQueueProcessorsService: ObjectStorageQueueProcessorsService,
|
||||
private dbQueueProcessorsService: DbQueueProcessorsService,
|
||||
private relationshipQueueProcessorsService: RelationshipQueueProcessorsService,
|
||||
private webhookDeliverProcessorService: WebhookDeliverProcessorService,
|
||||
private endedPollNotificationProcessorService: EndedPollNotificationProcessorService,
|
||||
private deliverProcessorService: DeliverProcessorService,
|
||||
|
|
@ -52,14 +54,15 @@ export class QueueProcessorService {
|
|||
};
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
const systemLogger = this.logger.createSubLogger('system');
|
||||
const deliverLogger = this.logger.createSubLogger('deliver');
|
||||
const webhookLogger = this.logger.createSubLogger('webhook');
|
||||
const inboxLogger = this.logger.createSubLogger('inbox');
|
||||
const dbLogger = this.logger.createSubLogger('db');
|
||||
const relationshipLogger = this.logger.createSubLogger('relationship');
|
||||
const objectStorageLogger = this.logger.createSubLogger('objectStorage');
|
||||
|
||||
|
||||
this.queueService.systemQueue
|
||||
.on('waiting', (jobId) => systemLogger.debug(`waiting id=${jobId}`))
|
||||
.on('active', (job) => systemLogger.debug(`active id=${job.id}`))
|
||||
|
|
@ -67,7 +70,7 @@ export class QueueProcessorService {
|
|||
.on('failed', (job, err) => systemLogger.warn(`failed(${err}) id=${job.id}`, { job, e: renderError(err) }))
|
||||
.on('error', (job: any, err: Error) => systemLogger.error(`error ${err}`, { job, e: renderError(err) }))
|
||||
.on('stalled', (job) => systemLogger.warn(`stalled id=${job.id}`));
|
||||
|
||||
|
||||
this.queueService.deliverQueue
|
||||
.on('waiting', (jobId) => deliverLogger.debug(`waiting id=${jobId}`))
|
||||
.on('active', (job) => deliverLogger.debug(`active ${getJobInfo(job, true)} to=${job.data.to}`))
|
||||
|
|
@ -75,7 +78,7 @@ export class QueueProcessorService {
|
|||
.on('failed', (job, err) => deliverLogger.warn(`failed(${err}) ${getJobInfo(job)} to=${job.data.to}`))
|
||||
.on('error', (job: any, err: Error) => deliverLogger.error(`error ${err}`, { job, e: renderError(err) }))
|
||||
.on('stalled', (job) => deliverLogger.warn(`stalled ${getJobInfo(job)} to=${job.data.to}`));
|
||||
|
||||
|
||||
this.queueService.inboxQueue
|
||||
.on('waiting', (jobId) => inboxLogger.debug(`waiting id=${jobId}`))
|
||||
.on('active', (job) => inboxLogger.debug(`active ${getJobInfo(job, true)}`))
|
||||
|
|
@ -83,7 +86,7 @@ export class QueueProcessorService {
|
|||
.on('failed', (job, err) => inboxLogger.warn(`failed(${err}) ${getJobInfo(job)} activity=${job.data.activity ? job.data.activity.id : 'none'}`, { job, e: renderError(err) }))
|
||||
.on('error', (job: any, err: Error) => inboxLogger.error(`error ${err}`, { job, e: renderError(err) }))
|
||||
.on('stalled', (job) => inboxLogger.warn(`stalled ${getJobInfo(job)} activity=${job.data.activity ? job.data.activity.id : 'none'}`));
|
||||
|
||||
|
||||
this.queueService.dbQueue
|
||||
.on('waiting', (jobId) => dbLogger.debug(`waiting id=${jobId}`))
|
||||
.on('active', (job) => dbLogger.debug(`active id=${job.id}`))
|
||||
|
|
@ -91,7 +94,15 @@ export class QueueProcessorService {
|
|||
.on('failed', (job, err) => dbLogger.warn(`failed(${err}) id=${job.id}`, { job, e: renderError(err) }))
|
||||
.on('error', (job: any, err: Error) => dbLogger.error(`error ${err}`, { job, e: renderError(err) }))
|
||||
.on('stalled', (job) => dbLogger.warn(`stalled id=${job.id}`));
|
||||
|
||||
|
||||
this.queueService.relationshipQueue
|
||||
.on('waiting', (jobId) => relationshipLogger.debug(`waiting id=${jobId}`))
|
||||
.on('active', (job) => relationshipLogger.debug(`active id=${job.id}`))
|
||||
.on('completed', (job, result) => relationshipLogger.debug(`completed(${result}) id=${job.id}`))
|
||||
.on('failed', (job, err) => relationshipLogger.warn(`failed(${err}) id=${job.id}`, { job, e: renderError(err) }))
|
||||
.on('error', (job: any, err: Error) => relationshipLogger.error(`error ${err}`, { job, e: renderError(err) }))
|
||||
.on('stalled', (job) => relationshipLogger.warn(`stalled id=${job.id}`));
|
||||
|
||||
this.queueService.objectStorageQueue
|
||||
.on('waiting', (jobId) => objectStorageLogger.debug(`waiting id=${jobId}`))
|
||||
.on('active', (job) => objectStorageLogger.debug(`active id=${job.id}`))
|
||||
|
|
@ -99,7 +110,7 @@ export class QueueProcessorService {
|
|||
.on('failed', (job, err) => objectStorageLogger.warn(`failed(${err}) id=${job.id}`, { job, e: renderError(err) }))
|
||||
.on('error', (job: any, err: Error) => objectStorageLogger.error(`error ${err}`, { job, e: renderError(err) }))
|
||||
.on('stalled', (job) => objectStorageLogger.warn(`stalled id=${job.id}`));
|
||||
|
||||
|
||||
this.queueService.webhookDeliverQueue
|
||||
.on('waiting', (jobId) => webhookLogger.debug(`waiting id=${jobId}`))
|
||||
.on('active', (job) => webhookLogger.debug(`active ${getJobInfo(job, true)} to=${job.data.to}`))
|
||||
|
|
@ -107,26 +118,27 @@ export class QueueProcessorService {
|
|||
.on('failed', (job, err) => webhookLogger.warn(`failed(${err}) ${getJobInfo(job)} to=${job.data.to}`))
|
||||
.on('error', (job: any, err: Error) => webhookLogger.error(`error ${err}`, { job, e: renderError(err) }))
|
||||
.on('stalled', (job) => webhookLogger.warn(`stalled ${getJobInfo(job)} to=${job.data.to}`));
|
||||
|
||||
|
||||
this.queueService.deliverQueue.process(this.config.deliverJobConcurrency ?? 128, (job) => this.deliverProcessorService.process(job));
|
||||
this.queueService.inboxQueue.process(this.config.inboxJobConcurrency ?? 16, (job) => this.inboxProcessorService.process(job));
|
||||
this.queueService.endedPollNotificationQueue.process((job, done) => this.endedPollNotificationProcessorService.process(job, done));
|
||||
this.queueService.webhookDeliverQueue.process(64, (job) => this.webhookDeliverProcessorService.process(job));
|
||||
this.dbQueueProcessorsService.start(this.queueService.dbQueue);
|
||||
this.relationshipQueueProcessorsService.start(this.queueService.relationshipQueue);
|
||||
this.objectStorageQueueProcessorsService.start(this.queueService.objectStorageQueue);
|
||||
|
||||
|
||||
this.queueService.systemQueue.add('tickCharts', {
|
||||
}, {
|
||||
repeat: { cron: '55 * * * *' },
|
||||
removeOnComplete: true,
|
||||
});
|
||||
|
||||
|
||||
this.queueService.systemQueue.add('resyncCharts', {
|
||||
}, {
|
||||
repeat: { cron: '0 0 * * *' },
|
||||
removeOnComplete: true,
|
||||
});
|
||||
|
||||
|
||||
this.queueService.systemQueue.add('cleanCharts', {
|
||||
}, {
|
||||
repeat: { cron: '0 0 * * *' },
|
||||
|
|
@ -138,19 +150,19 @@ export class QueueProcessorService {
|
|||
repeat: { cron: '0 0 * * *' },
|
||||
removeOnComplete: true,
|
||||
});
|
||||
|
||||
|
||||
this.queueService.systemQueue.add('clean', {
|
||||
}, {
|
||||
repeat: { cron: '0 0 * * *' },
|
||||
removeOnComplete: true,
|
||||
});
|
||||
|
||||
|
||||
this.queueService.systemQueue.add('checkExpiredMutings', {
|
||||
}, {
|
||||
repeat: { cron: '*/5 * * * *' },
|
||||
removeOnComplete: true,
|
||||
});
|
||||
|
||||
|
||||
this.systemQueueProcessorsService.start(this.queueService.systemQueue);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue