From 27ac3d795e7098124a7eea0dd59f6f1ef8f32394 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E3=81=8B=E3=81=A3=E3=81=93=E3=81=8B=E3=82=8A?= <67428053+kakkokari-gtyih@users.noreply.github.com> Date: Tue, 4 Jun 2024 13:14:37 +0900 Subject: [PATCH 1/9] Update about-misskey.vue --- packages/frontend/src/pages/about-misskey.vue | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/packages/frontend/src/pages/about-misskey.vue b/packages/frontend/src/pages/about-misskey.vue index b55ae220d8..629f00689d 100644 --- a/packages/frontend/src/pages/about-misskey.vue +++ b/packages/frontend/src/pages/about-misskey.vue @@ -102,13 +102,16 @@ SPDX-License-Identifier: AGPL-3.0-only
- Mask Network + Mask Network
- XServer + XServer
- Skeb + Skeb +
+
+ Skeb
From 43cccaaee9be42fab38eaa9ca04bb5e55b5d8db7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E3=81=8B=E3=81=A3=E3=81=93=E3=81=8B=E3=82=8A?= <67428053+kakkokari-gtyih@users.noreply.github.com> Date: Tue, 4 Jun 2024 13:15:35 +0900 Subject: [PATCH 2/9] fix --- packages/frontend/src/pages/about-misskey.vue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/frontend/src/pages/about-misskey.vue b/packages/frontend/src/pages/about-misskey.vue index 629f00689d..cc0394f401 100644 --- a/packages/frontend/src/pages/about-misskey.vue +++ b/packages/frontend/src/pages/about-misskey.vue @@ -111,7 +111,7 @@ SPDX-License-Identifier: AGPL-3.0-only Skeb
- Skeb + GMO Pepabo
From d4a8c63264939c4ec36af628f7e1516d2ae60254 Mon Sep 17 00:00:00 2001 From: syuilo <4439005+syuilo@users.noreply.github.com> Date: Thu, 6 Jun 2024 09:32:04 +0900 Subject: [PATCH 3/9] enhance(backend): sentry integration for job queues --- .../src/queue/QueueProcessorService.ts | 389 +++++++++++------- 1 file changed, 231 insertions(+), 158 deletions(-) diff --git a/packages/backend/src/queue/QueueProcessorService.ts b/packages/backend/src/queue/QueueProcessorService.ts index ce999d9cef..eb1901d069 100644 --- a/packages/backend/src/queue/QueueProcessorService.ts +++ b/packages/backend/src/queue/QueueProcessorService.ts @@ -5,6 +5,7 @@ import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common'; import * as Bull from 'bullmq'; +import * as Sentry from '@sentry/node'; import type { Config } from '@/config.js'; import { DI } from '@/di-symbols.js'; import type Logger from '@/logger.js'; @@ -135,199 +136,271 @@ export class QueueProcessorService implements OnApplicationShutdown { } //#region system - this.systemQueueWorker = new Bull.Worker(QUEUE.SYSTEM, (job) => { - switch (job.name) { - case 'tickCharts': return this.tickChartsProcessorService.process(); - case 'resyncCharts': return this.resyncChartsProcessorService.process(); - case 'cleanCharts': return this.cleanChartsProcessorService.process(); - case 'aggregateRetention': return this.aggregateRetentionProcessorService.process(); - case 'checkExpiredMutings': return this.checkExpiredMutingsProcessorService.process(); - case 'clean': return this.cleanProcessorService.process(); - default: throw new Error(`unrecognized job type ${job.name} for system`); - } - }, { - ...baseQueueOptions(this.config, QUEUE.SYSTEM), - autorun: false, - }); + { + const processer = (job: Bull.Job) => { + switch (job.name) { + case 'tickCharts': return this.tickChartsProcessorService.process(); + case 'resyncCharts': return this.resyncChartsProcessorService.process(); + case 'cleanCharts': return this.cleanChartsProcessorService.process(); + case 'aggregateRetention': return this.aggregateRetentionProcessorService.process(); + case 'checkExpiredMutings': return this.checkExpiredMutingsProcessorService.process(); + case 'clean': return this.cleanProcessorService.process(); + default: throw new Error(`unrecognized job type ${job.name} for system`); + } + }; - const systemLogger = this.logger.createSubLogger('system'); + this.systemQueueWorker = new Bull.Worker(QUEUE.SYSTEM, (job) => { + if (this.config.sentryForBackend) { + return Sentry.startSpan({ name: 'Queue: System: ' + job.name }, () => processer(job)); + } else { + return processer(job); + } + }, { + ...baseQueueOptions(this.config, QUEUE.SYSTEM), + autorun: false, + }); - this.systemQueueWorker - .on('active', (job) => systemLogger.debug(`active id=${job.id}`)) - .on('completed', (job, result) => systemLogger.debug(`completed(${result}) id=${job.id}`)) - .on('failed', (job, err) => systemLogger.warn(`failed(${err.stack}) id=${job ? job.id : '-'}`, { job, e: renderError(err) })) - .on('error', (err: Error) => systemLogger.error(`error ${err.stack}`, { e: renderError(err) })) - .on('stalled', (jobId) => systemLogger.warn(`stalled id=${jobId}`)); + const systemLogger = this.logger.createSubLogger('system'); + + this.systemQueueWorker + .on('active', (job) => systemLogger.debug(`active id=${job.id}`)) + .on('completed', (job, result) => systemLogger.debug(`completed(${result}) id=${job.id}`)) + .on('failed', (job, err) => systemLogger.warn(`failed(${err.stack}) id=${job ? job.id : '-'}`, { job, e: renderError(err) })) + .on('error', (err: Error) => systemLogger.error(`error ${err.stack}`, { e: renderError(err) })) + .on('stalled', (jobId) => systemLogger.warn(`stalled id=${jobId}`)); + } //#endregion //#region db - this.dbQueueWorker = new Bull.Worker(QUEUE.DB, (job) => { - switch (job.name) { - case 'deleteDriveFiles': return this.deleteDriveFilesProcessorService.process(job); - case 'exportCustomEmojis': return this.exportCustomEmojisProcessorService.process(job); - case 'exportNotes': return this.exportNotesProcessorService.process(job); - case 'exportClips': return this.exportClipsProcessorService.process(job); - case 'exportFavorites': return this.exportFavoritesProcessorService.process(job); - case 'exportFollowing': return this.exportFollowingProcessorService.process(job); - case 'exportMuting': return this.exportMutingProcessorService.process(job); - case 'exportBlocking': return this.exportBlockingProcessorService.process(job); - case 'exportUserLists': return this.exportUserListsProcessorService.process(job); - case 'exportAntennas': return this.exportAntennasProcessorService.process(job); - case 'importFollowing': return this.importFollowingProcessorService.process(job); - case 'importFollowingToDb': return this.importFollowingProcessorService.processDb(job); - case 'importMuting': return this.importMutingProcessorService.process(job); - case 'importBlocking': return this.importBlockingProcessorService.process(job); - case 'importBlockingToDb': return this.importBlockingProcessorService.processDb(job); - case 'importUserLists': return this.importUserListsProcessorService.process(job); - case 'importCustomEmojis': return this.importCustomEmojisProcessorService.process(job); - case 'importAntennas': return this.importAntennasProcessorService.process(job); - case 'deleteAccount': return this.deleteAccountProcessorService.process(job); - default: throw new Error(`unrecognized job type ${job.name} for db`); - } - }, { - ...baseQueueOptions(this.config, QUEUE.DB), - autorun: false, - }); + { + const processer = (job: Bull.Job) => { + switch (job.name) { + case 'deleteDriveFiles': return this.deleteDriveFilesProcessorService.process(job); + case 'exportCustomEmojis': return this.exportCustomEmojisProcessorService.process(job); + case 'exportNotes': return this.exportNotesProcessorService.process(job); + case 'exportClips': return this.exportClipsProcessorService.process(job); + case 'exportFavorites': return this.exportFavoritesProcessorService.process(job); + case 'exportFollowing': return this.exportFollowingProcessorService.process(job); + case 'exportMuting': return this.exportMutingProcessorService.process(job); + case 'exportBlocking': return this.exportBlockingProcessorService.process(job); + case 'exportUserLists': return this.exportUserListsProcessorService.process(job); + case 'exportAntennas': return this.exportAntennasProcessorService.process(job); + case 'importFollowing': return this.importFollowingProcessorService.process(job); + case 'importFollowingToDb': return this.importFollowingProcessorService.processDb(job); + case 'importMuting': return this.importMutingProcessorService.process(job); + case 'importBlocking': return this.importBlockingProcessorService.process(job); + case 'importBlockingToDb': return this.importBlockingProcessorService.processDb(job); + case 'importUserLists': return this.importUserListsProcessorService.process(job); + case 'importCustomEmojis': return this.importCustomEmojisProcessorService.process(job); + case 'importAntennas': return this.importAntennasProcessorService.process(job); + case 'deleteAccount': return this.deleteAccountProcessorService.process(job); + default: throw new Error(`unrecognized job type ${job.name} for db`); + } + }; - const dbLogger = this.logger.createSubLogger('db'); + this.dbQueueWorker = new Bull.Worker(QUEUE.DB, (job) => { + if (this.config.sentryForBackend) { + return Sentry.startSpan({ name: 'Queue: DB: ' + job.name }, () => processer(job)); + } else { + return processer(job); + } + }, { + ...baseQueueOptions(this.config, QUEUE.DB), + autorun: false, + }); - this.dbQueueWorker - .on('active', (job) => dbLogger.debug(`active id=${job.id}`)) - .on('completed', (job, result) => dbLogger.debug(`completed(${result}) id=${job.id}`)) - .on('failed', (job, err) => dbLogger.warn(`failed(${err.stack}) id=${job ? job.id : '-'}`, { job, e: renderError(err) })) - .on('error', (err: Error) => dbLogger.error(`error ${err.stack}`, { e: renderError(err) })) - .on('stalled', (jobId) => dbLogger.warn(`stalled id=${jobId}`)); + const dbLogger = this.logger.createSubLogger('db'); + + this.dbQueueWorker + .on('active', (job) => dbLogger.debug(`active id=${job.id}`)) + .on('completed', (job, result) => dbLogger.debug(`completed(${result}) id=${job.id}`)) + .on('failed', (job, err) => dbLogger.warn(`failed(${err.stack}) id=${job ? job.id : '-'}`, { job, e: renderError(err) })) + .on('error', (err: Error) => dbLogger.error(`error ${err.stack}`, { e: renderError(err) })) + .on('stalled', (jobId) => dbLogger.warn(`stalled id=${jobId}`)); + } //#endregion //#region deliver - this.deliverQueueWorker = new Bull.Worker(QUEUE.DELIVER, (job) => this.deliverProcessorService.process(job), { - ...baseQueueOptions(this.config, QUEUE.DELIVER), - autorun: false, - concurrency: this.config.deliverJobConcurrency ?? 128, - limiter: { - max: this.config.deliverJobPerSec ?? 128, - duration: 1000, - }, - settings: { - backoffStrategy: httpRelatedBackoff, - }, - }); + { + this.deliverQueueWorker = new Bull.Worker(QUEUE.DELIVER, (job) => { + if (this.config.sentryForBackend) { + return Sentry.startSpan({ name: 'Queue: Deliver' }, () => this.deliverProcessorService.process(job)); + } else { + return this.deliverProcessorService.process(job); + } + }, { + ...baseQueueOptions(this.config, QUEUE.DELIVER), + autorun: false, + concurrency: this.config.deliverJobConcurrency ?? 128, + limiter: { + max: this.config.deliverJobPerSec ?? 128, + duration: 1000, + }, + settings: { + backoffStrategy: httpRelatedBackoff, + }, + }); - const deliverLogger = this.logger.createSubLogger('deliver'); + const deliverLogger = this.logger.createSubLogger('deliver'); - this.deliverQueueWorker - .on('active', (job) => deliverLogger.debug(`active ${getJobInfo(job, true)} to=${job.data.to}`)) - .on('completed', (job, result) => deliverLogger.debug(`completed(${result}) ${getJobInfo(job, true)} to=${job.data.to}`)) - .on('failed', (job, err) => deliverLogger.warn(`failed(${err.stack}) ${getJobInfo(job)} to=${job ? job.data.to : '-'}`)) - .on('error', (err: Error) => deliverLogger.error(`error ${err.stack}`, { e: renderError(err) })) - .on('stalled', (jobId) => deliverLogger.warn(`stalled id=${jobId}`)); + this.deliverQueueWorker + .on('active', (job) => deliverLogger.debug(`active ${getJobInfo(job, true)} to=${job.data.to}`)) + .on('completed', (job, result) => deliverLogger.debug(`completed(${result}) ${getJobInfo(job, true)} to=${job.data.to}`)) + .on('failed', (job, err) => deliverLogger.warn(`failed(${err.stack}) ${getJobInfo(job)} to=${job ? job.data.to : '-'}`)) + .on('error', (err: Error) => deliverLogger.error(`error ${err.stack}`, { e: renderError(err) })) + .on('stalled', (jobId) => deliverLogger.warn(`stalled id=${jobId}`)); + } //#endregion //#region inbox - this.inboxQueueWorker = new Bull.Worker(QUEUE.INBOX, (job) => this.inboxProcessorService.process(job), { - ...baseQueueOptions(this.config, QUEUE.INBOX), - autorun: false, - concurrency: this.config.inboxJobConcurrency ?? 16, - limiter: { - max: this.config.inboxJobPerSec ?? 32, - duration: 1000, - }, - settings: { - backoffStrategy: httpRelatedBackoff, - }, - }); + { + this.inboxQueueWorker = new Bull.Worker(QUEUE.INBOX, (job) => { + if (this.config.sentryForBackend) { + return Sentry.startSpan({ name: 'Queue: Inbox' }, () => this.inboxProcessorService.process(job)); + } else { + return this.inboxProcessorService.process(job); + } + }, { + ...baseQueueOptions(this.config, QUEUE.INBOX), + autorun: false, + concurrency: this.config.inboxJobConcurrency ?? 16, + limiter: { + max: this.config.inboxJobPerSec ?? 32, + duration: 1000, + }, + settings: { + backoffStrategy: httpRelatedBackoff, + }, + }); - const inboxLogger = this.logger.createSubLogger('inbox'); + const inboxLogger = this.logger.createSubLogger('inbox'); - this.inboxQueueWorker - .on('active', (job) => inboxLogger.debug(`active ${getJobInfo(job, true)}`)) - .on('completed', (job, result) => inboxLogger.debug(`completed(${result}) ${getJobInfo(job, true)}`)) - .on('failed', (job, err) => inboxLogger.warn(`failed(${err.stack}) ${getJobInfo(job)} activity=${job ? (job.data.activity ? job.data.activity.id : 'none') : '-'}`, { job, e: renderError(err) })) - .on('error', (err: Error) => inboxLogger.error(`error ${err.stack}`, { e: renderError(err) })) - .on('stalled', (jobId) => inboxLogger.warn(`stalled id=${jobId}`)); + this.inboxQueueWorker + .on('active', (job) => inboxLogger.debug(`active ${getJobInfo(job, true)}`)) + .on('completed', (job, result) => inboxLogger.debug(`completed(${result}) ${getJobInfo(job, true)}`)) + .on('failed', (job, err) => inboxLogger.warn(`failed(${err.stack}) ${getJobInfo(job)} activity=${job ? (job.data.activity ? job.data.activity.id : 'none') : '-'}`, { job, e: renderError(err) })) + .on('error', (err: Error) => inboxLogger.error(`error ${err.stack}`, { e: renderError(err) })) + .on('stalled', (jobId) => inboxLogger.warn(`stalled id=${jobId}`)); + } //#endregion //#region webhook deliver - this.webhookDeliverQueueWorker = new Bull.Worker(QUEUE.WEBHOOK_DELIVER, (job) => this.webhookDeliverProcessorService.process(job), { - ...baseQueueOptions(this.config, QUEUE.WEBHOOK_DELIVER), - autorun: false, - concurrency: 64, - limiter: { - max: 64, - duration: 1000, - }, - settings: { - backoffStrategy: httpRelatedBackoff, - }, - }); + { + this.webhookDeliverQueueWorker = new Bull.Worker(QUEUE.WEBHOOK_DELIVER, (job) => { + if (this.config.sentryForBackend) { + return Sentry.startSpan({ name: 'Queue: WebhookDeliver' }, () => this.webhookDeliverProcessorService.process(job)); + } else { + return this.webhookDeliverProcessorService.process(job); + } + }, { + ...baseQueueOptions(this.config, QUEUE.WEBHOOK_DELIVER), + autorun: false, + concurrency: 64, + limiter: { + max: 64, + duration: 1000, + }, + settings: { + backoffStrategy: httpRelatedBackoff, + }, + }); - const webhookLogger = this.logger.createSubLogger('webhook'); + const webhookLogger = this.logger.createSubLogger('webhook'); - this.webhookDeliverQueueWorker - .on('active', (job) => webhookLogger.debug(`active ${getJobInfo(job, true)} to=${job.data.to}`)) - .on('completed', (job, result) => webhookLogger.debug(`completed(${result}) ${getJobInfo(job, true)} to=${job.data.to}`)) - .on('failed', (job, err) => webhookLogger.warn(`failed(${err.stack}) ${getJobInfo(job)} to=${job ? job.data.to : '-'}`)) - .on('error', (err: Error) => webhookLogger.error(`error ${err.stack}`, { e: renderError(err) })) - .on('stalled', (jobId) => webhookLogger.warn(`stalled id=${jobId}`)); + this.webhookDeliverQueueWorker + .on('active', (job) => webhookLogger.debug(`active ${getJobInfo(job, true)} to=${job.data.to}`)) + .on('completed', (job, result) => webhookLogger.debug(`completed(${result}) ${getJobInfo(job, true)} to=${job.data.to}`)) + .on('failed', (job, err) => webhookLogger.warn(`failed(${err.stack}) ${getJobInfo(job)} to=${job ? job.data.to : '-'}`)) + .on('error', (err: Error) => webhookLogger.error(`error ${err.stack}`, { e: renderError(err) })) + .on('stalled', (jobId) => webhookLogger.warn(`stalled id=${jobId}`)); + } //#endregion //#region relationship - this.relationshipQueueWorker = new Bull.Worker(QUEUE.RELATIONSHIP, (job) => { - switch (job.name) { - case 'follow': return this.relationshipProcessorService.processFollow(job); - case 'unfollow': return this.relationshipProcessorService.processUnfollow(job); - case 'block': return this.relationshipProcessorService.processBlock(job); - case 'unblock': return this.relationshipProcessorService.processUnblock(job); - default: throw new Error(`unrecognized job type ${job.name} for relationship`); - } - }, { - ...baseQueueOptions(this.config, QUEUE.RELATIONSHIP), - autorun: false, - concurrency: this.config.relationshipJobConcurrency ?? 16, - limiter: { - max: this.config.relationshipJobPerSec ?? 64, - duration: 1000, - }, - }); + { + const processer = (job: Bull.Job) => { + switch (job.name) { + case 'follow': return this.relationshipProcessorService.processFollow(job); + case 'unfollow': return this.relationshipProcessorService.processUnfollow(job); + case 'block': return this.relationshipProcessorService.processBlock(job); + case 'unblock': return this.relationshipProcessorService.processUnblock(job); + default: throw new Error(`unrecognized job type ${job.name} for relationship`); + } + }; - const relationshipLogger = this.logger.createSubLogger('relationship'); + this.relationshipQueueWorker = new Bull.Worker(QUEUE.RELATIONSHIP, (job) => { + if (this.config.sentryForBackend) { + return Sentry.startSpan({ name: 'Queue: Relationship: ' + job.name }, () => processer(job)); + } else { + return processer(job); + } + }, { + ...baseQueueOptions(this.config, QUEUE.RELATIONSHIP), + autorun: false, + concurrency: this.config.relationshipJobConcurrency ?? 16, + limiter: { + max: this.config.relationshipJobPerSec ?? 64, + duration: 1000, + }, + }); - this.relationshipQueueWorker - .on('active', (job) => relationshipLogger.debug(`active id=${job.id}`)) - .on('completed', (job, result) => relationshipLogger.debug(`completed(${result}) id=${job.id}`)) - .on('failed', (job, err) => relationshipLogger.warn(`failed(${err.stack}) id=${job ? job.id : '-'}`, { job, e: renderError(err) })) - .on('error', (err: Error) => relationshipLogger.error(`error ${err.stack}`, { e: renderError(err) })) - .on('stalled', (jobId) => relationshipLogger.warn(`stalled id=${jobId}`)); + const relationshipLogger = this.logger.createSubLogger('relationship'); + + this.relationshipQueueWorker + .on('active', (job) => relationshipLogger.debug(`active id=${job.id}`)) + .on('completed', (job, result) => relationshipLogger.debug(`completed(${result}) id=${job.id}`)) + .on('failed', (job, err) => relationshipLogger.warn(`failed(${err.stack}) id=${job ? job.id : '-'}`, { job, e: renderError(err) })) + .on('error', (err: Error) => relationshipLogger.error(`error ${err.stack}`, { e: renderError(err) })) + .on('stalled', (jobId) => relationshipLogger.warn(`stalled id=${jobId}`)); + } //#endregion //#region object storage - this.objectStorageQueueWorker = new Bull.Worker(QUEUE.OBJECT_STORAGE, (job) => { - switch (job.name) { - case 'deleteFile': return this.deleteFileProcessorService.process(job); - case 'cleanRemoteFiles': return this.cleanRemoteFilesProcessorService.process(job); - default: throw new Error(`unrecognized job type ${job.name} for objectStorage`); - } - }, { - ...baseQueueOptions(this.config, QUEUE.OBJECT_STORAGE), - autorun: false, - concurrency: 16, - }); + { + const processer = (job: Bull.Job) => { + switch (job.name) { + case 'deleteFile': return this.deleteFileProcessorService.process(job); + case 'cleanRemoteFiles': return this.cleanRemoteFilesProcessorService.process(job); + default: throw new Error(`unrecognized job type ${job.name} for objectStorage`); + } + }; - const objectStorageLogger = this.logger.createSubLogger('objectStorage'); + this.objectStorageQueueWorker = new Bull.Worker(QUEUE.OBJECT_STORAGE, (job) => { + if (this.config.sentryForBackend) { + return Sentry.startSpan({ name: 'Queue: ObjectStorage: ' + job.name }, () => processer(job)); + } else { + return processer(job); + } + }, { + ...baseQueueOptions(this.config, QUEUE.OBJECT_STORAGE), + autorun: false, + concurrency: 16, + }); - this.objectStorageQueueWorker - .on('active', (job) => objectStorageLogger.debug(`active id=${job.id}`)) - .on('completed', (job, result) => objectStorageLogger.debug(`completed(${result}) id=${job.id}`)) - .on('failed', (job, err) => objectStorageLogger.warn(`failed(${err.stack}) id=${job ? job.id : '-'}`, { job, e: renderError(err) })) - .on('error', (err: Error) => objectStorageLogger.error(`error ${err.stack}`, { e: renderError(err) })) - .on('stalled', (jobId) => objectStorageLogger.warn(`stalled id=${jobId}`)); + const objectStorageLogger = this.logger.createSubLogger('objectStorage'); + + this.objectStorageQueueWorker + .on('active', (job) => objectStorageLogger.debug(`active id=${job.id}`)) + .on('completed', (job, result) => objectStorageLogger.debug(`completed(${result}) id=${job.id}`)) + .on('failed', (job, err) => objectStorageLogger.warn(`failed(${err.stack}) id=${job ? job.id : '-'}`, { job, e: renderError(err) })) + .on('error', (err: Error) => objectStorageLogger.error(`error ${err.stack}`, { e: renderError(err) })) + .on('stalled', (jobId) => objectStorageLogger.warn(`stalled id=${jobId}`)); + } //#endregion //#region ended poll notification - this.endedPollNotificationQueueWorker = new Bull.Worker(QUEUE.ENDED_POLL_NOTIFICATION, (job) => this.endedPollNotificationProcessorService.process(job), { - ...baseQueueOptions(this.config, QUEUE.ENDED_POLL_NOTIFICATION), - autorun: false, - }); + { + this.endedPollNotificationQueueWorker = new Bull.Worker(QUEUE.ENDED_POLL_NOTIFICATION, (job) => { + if (this.config.sentryForBackend) { + return Sentry.startSpan({ name: 'Queue: EndedPollNotification' }, () => this.endedPollNotificationProcessorService.process(job)); + } else { + return this.endedPollNotificationProcessorService.process(job); + } + }, { + ...baseQueueOptions(this.config, QUEUE.ENDED_POLL_NOTIFICATION), + autorun: false, + }); + } //#endregion } From dbf9e1194bf5b84ec711c14f27c11d2bfeb37f20 Mon Sep 17 00:00:00 2001 From: syuilo <4439005+syuilo@users.noreply.github.com> Date: Thu, 6 Jun 2024 10:01:50 +0900 Subject: [PATCH 4/9] refactor(backend): remove unused logger option --- packages/backend/src/core/LoggerService.ts | 4 ++-- .../backend/src/core/chart/ChartLoggerService.ts | 2 +- packages/backend/src/logger.ts | 14 +++++--------- packages/backend/src/server/FileServerService.ts | 2 +- packages/backend/src/server/ServerService.ts | 2 +- 5 files changed, 10 insertions(+), 14 deletions(-) diff --git a/packages/backend/src/core/LoggerService.ts b/packages/backend/src/core/LoggerService.ts index 96d9b09992..f102461a50 100644 --- a/packages/backend/src/core/LoggerService.ts +++ b/packages/backend/src/core/LoggerService.ts @@ -15,7 +15,7 @@ export class LoggerService { } @bindThis - public getLogger(domain: string, color?: KEYWORD | undefined, store?: boolean) { - return new Logger(domain, color, store); + public getLogger(domain: string, color?: KEYWORD | undefined) { + return new Logger(domain, color); } } diff --git a/packages/backend/src/core/chart/ChartLoggerService.ts b/packages/backend/src/core/chart/ChartLoggerService.ts index afc728d564..20815ea968 100644 --- a/packages/backend/src/core/chart/ChartLoggerService.ts +++ b/packages/backend/src/core/chart/ChartLoggerService.ts @@ -14,6 +14,6 @@ export class ChartLoggerService { constructor( private loggerService: LoggerService, ) { - this.logger = this.loggerService.getLogger('chart', 'white', process.env.NODE_ENV !== 'test'); + this.logger = this.loggerService.getLogger('chart', 'white'); } } diff --git a/packages/backend/src/logger.ts b/packages/backend/src/logger.ts index d4705af601..ff5363a425 100644 --- a/packages/backend/src/logger.ts +++ b/packages/backend/src/logger.ts @@ -22,31 +22,27 @@ type Level = 'error' | 'success' | 'warning' | 'debug' | 'info'; export default class Logger { private context: Context; private parentLogger: Logger | null = null; - private store: boolean; - constructor(context: string, color?: KEYWORD, store = true) { + constructor(context: string, color?: KEYWORD) { this.context = { name: context, color: color, }; - this.store = store; } @bindThis - public createSubLogger(context: string, color?: KEYWORD, store = true): Logger { - const logger = new Logger(context, color, store); + public createSubLogger(context: string, color?: KEYWORD): Logger { + const logger = new Logger(context, color); logger.parentLogger = this; return logger; } @bindThis - private log(level: Level, message: string, data?: Record | null, important = false, subContexts: Context[] = [], store = true): void { + private log(level: Level, message: string, data?: Record | null, important = false, subContexts: Context[] = []): void { if (envOption.quiet) return; - if (!this.store) store = false; - if (level === 'debug') store = false; if (this.parentLogger) { - this.parentLogger.log(level, message, data, important, [this.context].concat(subContexts), store); + this.parentLogger.log(level, message, data, important, [this.context].concat(subContexts)); return; } diff --git a/packages/backend/src/server/FileServerService.ts b/packages/backend/src/server/FileServerService.ts index 9db3aa1bfb..77a637d895 100644 --- a/packages/backend/src/server/FileServerService.ts +++ b/packages/backend/src/server/FileServerService.ts @@ -53,7 +53,7 @@ export class FileServerService { private internalStorageService: InternalStorageService, private loggerService: LoggerService, ) { - this.logger = this.loggerService.getLogger('server', 'gray', false); + this.logger = this.loggerService.getLogger('server', 'gray'); //this.createServer = this.createServer.bind(this); } diff --git a/packages/backend/src/server/ServerService.ts b/packages/backend/src/server/ServerService.ts index 3572f16627..9c849480f2 100644 --- a/packages/backend/src/server/ServerService.ts +++ b/packages/backend/src/server/ServerService.ts @@ -68,7 +68,7 @@ export class ServerService implements OnApplicationShutdown { private loggerService: LoggerService, private oauth2ProviderService: OAuth2ProviderService, ) { - this.logger = this.loggerService.getLogger('server', 'gray', false); + this.logger = this.loggerService.getLogger('server', 'gray'); } @bindThis From 65d19279a2c19c3e6be1c4c50cc9b01c94420d6c Mon Sep 17 00:00:00 2001 From: syuilo <4439005+syuilo@users.noreply.github.com> Date: Thu, 6 Jun 2024 10:11:43 +0900 Subject: [PATCH 5/9] fix --- packages/backend/src/boot/master.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/backend/src/boot/master.ts b/packages/backend/src/boot/master.ts index 75e1a80cd1..4bc5c799cf 100644 --- a/packages/backend/src/boot/master.ts +++ b/packages/backend/src/boot/master.ts @@ -25,7 +25,7 @@ const _dirname = dirname(_filename); const meta = JSON.parse(fs.readFileSync(`${_dirname}/../../../../built/meta.json`, 'utf-8')); const logger = new Logger('core', 'cyan'); -const bootLogger = logger.createSubLogger('boot', 'magenta', false); +const bootLogger = logger.createSubLogger('boot', 'magenta'); const themeColor = chalk.hex('#86b300'); From ab69e113f4921462b72f1f352dfefe52b37862f5 Mon Sep 17 00:00:00 2001 From: syuilo <4439005+syuilo@users.noreply.github.com> Date: Thu, 6 Jun 2024 11:20:54 +0900 Subject: [PATCH 6/9] enhance(backend): improve sentry integration --- packages/backend/src/boot/worker.ts | 23 +++++++++++++++++++ .../src/queue/QueueProcessorService.ts | 4 ++-- .../backend/src/server/api/ApiCallService.ts | 14 +++++++---- 3 files changed, 35 insertions(+), 6 deletions(-) diff --git a/packages/backend/src/boot/worker.ts b/packages/backend/src/boot/worker.ts index d4a7cd56e5..5d4a15b29f 100644 --- a/packages/backend/src/boot/worker.ts +++ b/packages/backend/src/boot/worker.ts @@ -4,13 +4,36 @@ */ import cluster from 'node:cluster'; +import * as Sentry from '@sentry/node'; +import { nodeProfilingIntegration } from '@sentry/profiling-node'; import { envOption } from '@/env.js'; +import { loadConfig } from '@/config.js'; import { jobQueue, server } from './common.js'; /** * Init worker process */ export async function workerMain() { + const config = loadConfig(); + + if (config.sentryForBackend) { + Sentry.init({ + integrations: [ + ...(config.sentryForBackend.enableNodeProfiling ? [nodeProfilingIntegration()] : []), + ], + + // Performance Monitoring + tracesSampleRate: 1.0, // Capture 100% of the transactions + + // Set sampling rate for profiling - this is relative to tracesSampleRate + profilesSampleRate: 1.0, + + maxBreadcrumbs: 0, + + ...config.sentryForBackend.options, + }); + } + if (envOption.onlyServer) { await server(); } else if (envOption.onlyQueue) { diff --git a/packages/backend/src/queue/QueueProcessorService.ts b/packages/backend/src/queue/QueueProcessorService.ts index eb1901d069..4f333df791 100644 --- a/packages/backend/src/queue/QueueProcessorService.ts +++ b/packages/backend/src/queue/QueueProcessorService.ts @@ -165,7 +165,7 @@ export class QueueProcessorService implements OnApplicationShutdown { this.systemQueueWorker .on('active', (job) => systemLogger.debug(`active id=${job.id}`)) .on('completed', (job, result) => systemLogger.debug(`completed(${result}) id=${job.id}`)) - .on('failed', (job, err) => systemLogger.warn(`failed(${err.stack}) id=${job ? job.id : '-'}`, { job, e: renderError(err) })) + .on('failed', (job, err) => systemLogger.error(`failed(${err.stack}) id=${job ? job.id : '-'}`, { job, e: renderError(err) })) .on('error', (err: Error) => systemLogger.error(`error ${err.stack}`, { e: renderError(err) })) .on('stalled', (jobId) => systemLogger.warn(`stalled id=${jobId}`)); } @@ -214,7 +214,7 @@ export class QueueProcessorService implements OnApplicationShutdown { this.dbQueueWorker .on('active', (job) => dbLogger.debug(`active id=${job.id}`)) .on('completed', (job, result) => dbLogger.debug(`completed(${result}) id=${job.id}`)) - .on('failed', (job, err) => dbLogger.warn(`failed(${err.stack}) id=${job ? job.id : '-'}`, { job, e: renderError(err) })) + .on('failed', (job, err) => dbLogger.error(`failed(${err.stack}) id=${job ? job.id : '-'}`, { job, e: renderError(err) })) .on('error', (err: Error) => dbLogger.error(`error ${err.stack}`, { e: renderError(err) })) .on('stalled', (jobId) => dbLogger.warn(`stalled id=${jobId}`)); } diff --git a/packages/backend/src/server/api/ApiCallService.ts b/packages/backend/src/server/api/ApiCallService.ts index 271ef80554..e21a5e90ab 100644 --- a/packages/backend/src/server/api/ApiCallService.ts +++ b/packages/backend/src/server/api/ApiCallService.ts @@ -93,7 +93,7 @@ export class ApiCallService implements OnApplicationShutdown { } } - #onExecError(ep: IEndpoint, data: any, err: Error): void { + #onExecError(ep: IEndpoint, data: any, err: Error, userId?: MiUser['id']): void { if (err instanceof ApiError || err instanceof AuthenticationError) { throw err; } else { @@ -108,10 +108,12 @@ export class ApiCallService implements OnApplicationShutdown { id: errId, }, }); - console.error(err, errId); if (this.config.sentryForBackend) { Sentry.captureMessage(`Internal error occurred in ${ep.name}: ${err.message}`, { + user: { + id: userId, + }, extra: { ep: ep.name, ps: data, @@ -410,9 +412,13 @@ export class ApiCallService implements OnApplicationShutdown { // API invoking if (this.config.sentryForBackend) { - return await Sentry.startSpan({ name: 'API: ' + ep.name }, () => ep.exec(data, user, token, file, request.ip, request.headers).catch((err: Error) => this.#onExecError(ep, data, err))); + return await Sentry.startSpan({ + name: 'API: ' + ep.name, + }, () => ep.exec(data, user, token, file, request.ip, request.headers) + .catch((err: Error) => this.#onExecError(ep, data, err, user?.id))); } else { - return await ep.exec(data, user, token, file, request.ip, request.headers).catch((err: Error) => this.#onExecError(ep, data, err)); + return await ep.exec(data, user, token, file, request.ip, request.headers) + .catch((err: Error) => this.#onExecError(ep, data, err, user?.id)); } } From a697a7f97b401d1545a7f61694b2704b4d3ac9fc Mon Sep 17 00:00:00 2001 From: syuilo <4439005+syuilo@users.noreply.github.com> Date: Thu, 6 Jun 2024 11:38:34 +0900 Subject: [PATCH 7/9] enhance(backend): improve sentry integration --- .../src/queue/QueueProcessorService.ts | 63 ++++++++++++++++--- 1 file changed, 56 insertions(+), 7 deletions(-) diff --git a/packages/backend/src/queue/QueueProcessorService.ts b/packages/backend/src/queue/QueueProcessorService.ts index 4f333df791..fdeb6a9518 100644 --- a/packages/backend/src/queue/QueueProcessorService.ts +++ b/packages/backend/src/queue/QueueProcessorService.ts @@ -165,7 +165,14 @@ export class QueueProcessorService implements OnApplicationShutdown { this.systemQueueWorker .on('active', (job) => systemLogger.debug(`active id=${job.id}`)) .on('completed', (job, result) => systemLogger.debug(`completed(${result}) id=${job.id}`)) - .on('failed', (job, err) => systemLogger.error(`failed(${err.stack}) id=${job ? job.id : '-'}`, { job, e: renderError(err) })) + .on('failed', (job, err) => { + systemLogger.error(`failed(${err.stack}) id=${job ? job.id : '-'}`, { job, e: renderError(err) }); + if (config.sentryForBackend) { + Sentry.captureMessage(`Queue: System: ${job?.name ?? '?'}`, { + extra: { job, err }, + }); + } + }) .on('error', (err: Error) => systemLogger.error(`error ${err.stack}`, { e: renderError(err) })) .on('stalled', (jobId) => systemLogger.warn(`stalled id=${jobId}`)); } @@ -214,7 +221,14 @@ export class QueueProcessorService implements OnApplicationShutdown { this.dbQueueWorker .on('active', (job) => dbLogger.debug(`active id=${job.id}`)) .on('completed', (job, result) => dbLogger.debug(`completed(${result}) id=${job.id}`)) - .on('failed', (job, err) => dbLogger.error(`failed(${err.stack}) id=${job ? job.id : '-'}`, { job, e: renderError(err) })) + .on('failed', (job, err) => { + dbLogger.error(`failed(${err.stack}) id=${job ? job.id : '-'}`, { job, e: renderError(err) }); + if (config.sentryForBackend) { + Sentry.captureMessage(`Queue: DB: ${job?.name ?? '?'}`, { + extra: { job, err }, + }); + } + }) .on('error', (err: Error) => dbLogger.error(`error ${err.stack}`, { e: renderError(err) })) .on('stalled', (jobId) => dbLogger.warn(`stalled id=${jobId}`)); } @@ -246,7 +260,14 @@ export class QueueProcessorService implements OnApplicationShutdown { this.deliverQueueWorker .on('active', (job) => deliverLogger.debug(`active ${getJobInfo(job, true)} to=${job.data.to}`)) .on('completed', (job, result) => deliverLogger.debug(`completed(${result}) ${getJobInfo(job, true)} to=${job.data.to}`)) - .on('failed', (job, err) => deliverLogger.warn(`failed(${err.stack}) ${getJobInfo(job)} to=${job ? job.data.to : '-'}`)) + .on('failed', (job, err) => { + deliverLogger.error(`failed(${err.stack}) ${getJobInfo(job)} to=${job ? job.data.to : '-'}`); + if (config.sentryForBackend) { + Sentry.captureMessage('Queue: Deliver', { + extra: { job, err }, + }); + } + }) .on('error', (err: Error) => deliverLogger.error(`error ${err.stack}`, { e: renderError(err) })) .on('stalled', (jobId) => deliverLogger.warn(`stalled id=${jobId}`)); } @@ -278,7 +299,14 @@ export class QueueProcessorService implements OnApplicationShutdown { this.inboxQueueWorker .on('active', (job) => inboxLogger.debug(`active ${getJobInfo(job, true)}`)) .on('completed', (job, result) => inboxLogger.debug(`completed(${result}) ${getJobInfo(job, true)}`)) - .on('failed', (job, err) => inboxLogger.warn(`failed(${err.stack}) ${getJobInfo(job)} activity=${job ? (job.data.activity ? job.data.activity.id : 'none') : '-'}`, { job, e: renderError(err) })) + .on('failed', (job, err) => { + inboxLogger.error(`failed(${err.stack}) ${getJobInfo(job)} activity=${job ? (job.data.activity ? job.data.activity.id : 'none') : '-'}`, { job, e: renderError(err) }); + if (config.sentryForBackend) { + Sentry.captureMessage('Queue: Inbox', { + extra: { job, err }, + }); + } + }) .on('error', (err: Error) => inboxLogger.error(`error ${err.stack}`, { e: renderError(err) })) .on('stalled', (jobId) => inboxLogger.warn(`stalled id=${jobId}`)); } @@ -310,7 +338,14 @@ export class QueueProcessorService implements OnApplicationShutdown { this.webhookDeliverQueueWorker .on('active', (job) => webhookLogger.debug(`active ${getJobInfo(job, true)} to=${job.data.to}`)) .on('completed', (job, result) => webhookLogger.debug(`completed(${result}) ${getJobInfo(job, true)} to=${job.data.to}`)) - .on('failed', (job, err) => webhookLogger.warn(`failed(${err.stack}) ${getJobInfo(job)} to=${job ? job.data.to : '-'}`)) + .on('failed', (job, err) => { + webhookLogger.error(`failed(${err.stack}) ${getJobInfo(job)} to=${job ? job.data.to : '-'}`); + if (config.sentryForBackend) { + Sentry.captureMessage('Queue: WebhookDeliver', { + extra: { job, err }, + }); + } + }) .on('error', (err: Error) => webhookLogger.error(`error ${err.stack}`, { e: renderError(err) })) .on('stalled', (jobId) => webhookLogger.warn(`stalled id=${jobId}`)); } @@ -349,7 +384,14 @@ export class QueueProcessorService implements OnApplicationShutdown { this.relationshipQueueWorker .on('active', (job) => relationshipLogger.debug(`active id=${job.id}`)) .on('completed', (job, result) => relationshipLogger.debug(`completed(${result}) id=${job.id}`)) - .on('failed', (job, err) => relationshipLogger.warn(`failed(${err.stack}) id=${job ? job.id : '-'}`, { job, e: renderError(err) })) + .on('failed', (job, err) => { + relationshipLogger.error(`failed(${err.stack}) id=${job ? job.id : '-'}`, { job, e: renderError(err) }); + if (config.sentryForBackend) { + Sentry.captureMessage(`Queue: Relationship: ${job?.name ?? '?'}`, { + extra: { job, err }, + }); + } + }) .on('error', (err: Error) => relationshipLogger.error(`error ${err.stack}`, { e: renderError(err) })) .on('stalled', (jobId) => relationshipLogger.warn(`stalled id=${jobId}`)); } @@ -382,7 +424,14 @@ export class QueueProcessorService implements OnApplicationShutdown { this.objectStorageQueueWorker .on('active', (job) => objectStorageLogger.debug(`active id=${job.id}`)) .on('completed', (job, result) => objectStorageLogger.debug(`completed(${result}) id=${job.id}`)) - .on('failed', (job, err) => objectStorageLogger.warn(`failed(${err.stack}) id=${job ? job.id : '-'}`, { job, e: renderError(err) })) + .on('failed', (job, err) => { + objectStorageLogger.error(`failed(${err.stack}) id=${job ? job.id : '-'}`, { job, e: renderError(err) }); + if (config.sentryForBackend) { + Sentry.captureMessage(`Queue: ObjectStorage: ${job?.name ?? '?'}`, { + extra: { job, err }, + }); + } + }) .on('error', (err: Error) => objectStorageLogger.error(`error ${err.stack}`, { e: renderError(err) })) .on('stalled', (jobId) => objectStorageLogger.warn(`stalled id=${jobId}`)); } From d55e638a231b380e733dc0ada3c3de410f918a93 Mon Sep 17 00:00:00 2001 From: syuilo <4439005+syuilo@users.noreply.github.com> Date: Thu, 6 Jun 2024 11:40:11 +0900 Subject: [PATCH 8/9] lint fixes --- packages/backend/src/NestLogger.ts | 2 +- packages/backend/src/boot/entry.ts | 2 +- packages/backend/src/postgres.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/backend/src/NestLogger.ts b/packages/backend/src/NestLogger.ts index 80f1f7a024..d0be19664f 100644 --- a/packages/backend/src/NestLogger.ts +++ b/packages/backend/src/NestLogger.ts @@ -7,7 +7,7 @@ import { LoggerService } from '@nestjs/common'; import Logger from '@/logger.js'; const logger = new Logger('core', 'cyan'); -const nestLogger = logger.createSubLogger('nest', 'green', false); +const nestLogger = logger.createSubLogger('nest', 'green'); export class NestLogger implements LoggerService { /** diff --git a/packages/backend/src/boot/entry.ts b/packages/backend/src/boot/entry.ts index 04c6ca9723..25375c3015 100644 --- a/packages/backend/src/boot/entry.ts +++ b/packages/backend/src/boot/entry.ts @@ -25,7 +25,7 @@ Error.stackTraceLimit = Infinity; EventEmitter.defaultMaxListeners = 128; const logger = new Logger('core', 'cyan'); -const clusterLogger = logger.createSubLogger('cluster', 'orange', false); +const clusterLogger = logger.createSubLogger('cluster', 'orange'); const ev = new Xev(); //#region Events diff --git a/packages/backend/src/postgres.ts b/packages/backend/src/postgres.ts index 2d14537bbb..aa2aa5e373 100644 --- a/packages/backend/src/postgres.ts +++ b/packages/backend/src/postgres.ts @@ -85,7 +85,7 @@ import { bindThis } from '@/decorators.js'; export const dbLogger = new MisskeyLogger('db'); -const sqlLogger = dbLogger.createSubLogger('sql', 'gray', false); +const sqlLogger = dbLogger.createSubLogger('sql', 'gray'); class MyCustomLogger implements Logger { @bindThis From 8f833d742fdb10f088479c78ad489461773a8b81 Mon Sep 17 00:00:00 2001 From: syuilo <4439005+syuilo@users.noreply.github.com> Date: Thu, 6 Jun 2024 11:51:31 +0900 Subject: [PATCH 9/9] enhance(backend): improve sentry integration --- .../backend/src/queue/QueueProcessorService.ts | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/packages/backend/src/queue/QueueProcessorService.ts b/packages/backend/src/queue/QueueProcessorService.ts index fdeb6a9518..6a87be021e 100644 --- a/packages/backend/src/queue/QueueProcessorService.ts +++ b/packages/backend/src/queue/QueueProcessorService.ts @@ -165,10 +165,10 @@ export class QueueProcessorService implements OnApplicationShutdown { this.systemQueueWorker .on('active', (job) => systemLogger.debug(`active id=${job.id}`)) .on('completed', (job, result) => systemLogger.debug(`completed(${result}) id=${job.id}`)) - .on('failed', (job, err) => { + .on('failed', (job, err: Error) => { systemLogger.error(`failed(${err.stack}) id=${job ? job.id : '-'}`, { job, e: renderError(err) }); if (config.sentryForBackend) { - Sentry.captureMessage(`Queue: System: ${job?.name ?? '?'}`, { + Sentry.captureMessage(`Queue: System: ${job?.name ?? '?'}: ${err.message}`, { extra: { job, err }, }); } @@ -224,7 +224,7 @@ export class QueueProcessorService implements OnApplicationShutdown { .on('failed', (job, err) => { dbLogger.error(`failed(${err.stack}) id=${job ? job.id : '-'}`, { job, e: renderError(err) }); if (config.sentryForBackend) { - Sentry.captureMessage(`Queue: DB: ${job?.name ?? '?'}`, { + Sentry.captureMessage(`Queue: DB: ${job?.name ?? '?'}: ${err.message}`, { extra: { job, err }, }); } @@ -263,7 +263,7 @@ export class QueueProcessorService implements OnApplicationShutdown { .on('failed', (job, err) => { deliverLogger.error(`failed(${err.stack}) ${getJobInfo(job)} to=${job ? job.data.to : '-'}`); if (config.sentryForBackend) { - Sentry.captureMessage('Queue: Deliver', { + Sentry.captureMessage(`Queue: Deliver: ${err.message}`, { extra: { job, err }, }); } @@ -302,7 +302,7 @@ export class QueueProcessorService implements OnApplicationShutdown { .on('failed', (job, err) => { inboxLogger.error(`failed(${err.stack}) ${getJobInfo(job)} activity=${job ? (job.data.activity ? job.data.activity.id : 'none') : '-'}`, { job, e: renderError(err) }); if (config.sentryForBackend) { - Sentry.captureMessage('Queue: Inbox', { + Sentry.captureMessage(`Queue: Inbox: ${err.message}`, { extra: { job, err }, }); } @@ -341,7 +341,7 @@ export class QueueProcessorService implements OnApplicationShutdown { .on('failed', (job, err) => { webhookLogger.error(`failed(${err.stack}) ${getJobInfo(job)} to=${job ? job.data.to : '-'}`); if (config.sentryForBackend) { - Sentry.captureMessage('Queue: WebhookDeliver', { + Sentry.captureMessage(`Queue: WebhookDeliver: ${err.message}`, { extra: { job, err }, }); } @@ -387,7 +387,7 @@ export class QueueProcessorService implements OnApplicationShutdown { .on('failed', (job, err) => { relationshipLogger.error(`failed(${err.stack}) id=${job ? job.id : '-'}`, { job, e: renderError(err) }); if (config.sentryForBackend) { - Sentry.captureMessage(`Queue: Relationship: ${job?.name ?? '?'}`, { + Sentry.captureMessage(`Queue: Relationship: ${job?.name ?? '?'}: ${err.message}`, { extra: { job, err }, }); } @@ -427,7 +427,7 @@ export class QueueProcessorService implements OnApplicationShutdown { .on('failed', (job, err) => { objectStorageLogger.error(`failed(${err.stack}) id=${job ? job.id : '-'}`, { job, e: renderError(err) }); if (config.sentryForBackend) { - Sentry.captureMessage(`Queue: ObjectStorage: ${job?.name ?? '?'}`, { + Sentry.captureMessage(`Queue: ObjectStorage: ${job?.name ?? '?'}: ${err.message}`, { extra: { job, err }, }); }