fix(backend): キューのエラーログを簡略化するように (#14748)
* reduce federation log spam * Don't record stack trace for unrecoverable errors. * Avoid logging duplicate stace traces. (cherry picked from commit ed0570110bf8cb8e8959591dccfa3c35999106ce) * improve error summaries (cherry picked from commit 20dd66f735d9778df0371001e303549dce619260) * fix lint errors (cherry picked from commit 83869e1c470b12b3bf4b23d885514d926620662a) * condense job info (cherry picked from commit 786702e076ad1af14538849512ad31c0ced7afe6) * fix maxAttempts calculation (cherry picked from commit b4d10aa8f821e594ec9c907eb2a5bdb3c73c67d5) * condense error info (cherry picked from commit f62cd8941ced74a4865aa5eae4f4a1c7aa1d30f1) * normalize ID logging (cherry picked from commit d8e1e4890d28347239162e26235eb68b1ff96654) * further condense error details (cherry picked from commit d867c2089b3b24680df0713a2aa0914789e45670) * collapse AbortErrors (cherry picked from commit 5171ba7113ebc7242527768afb9ab4cec534e3b3) * don't log job name unless it has one (cherry picked from commit a5316c06ed770b60f7b4c7ff5aa8c71cc0558db7) * Update Changelog * Record origin --------- Co-authored-by: Hazel K <acomputerdog@gmail.com>
This commit is contained in:
かっこかり
GitHub
parent
777804605e
commit
2f09d69773
@ -12,6 +12,10 @@
|
||||
- Feat: モデレータ権限を持つユーザが全員7日間活動しなかった場合は自動的に招待制へと移行するように ( #13437 )
|
||||
- Fix: `admin/emoji/update`エンドポイントのidのみ指定した時不正なエラーが発生するバグを修正
|
||||
|
||||
### Server
|
||||
- Fix: キューのエラーログを簡略化するように
|
||||
(Cherry-picked from https://activitypub.software/TransFem-org/Sharkey/-/merge_requests/649)
|
||||
|
||||
## 2024.10.0
|
||||
|
||||
### Note
|
||||
|
||||
@ -67,7 +67,7 @@ function getJobInfo(job: Bull.Job | undefined, increment = false): string {
|
||||
|
||||
// onActiveとかonCompletedのattemptsMadeがなぜか0始まりなのでインクリメントする
|
||||
const currentAttempts = job.attemptsMade + (increment ? 1 : 0);
|
||||
const maxAttempts = job.opts ? job.opts.attempts : 0;
|
||||
const maxAttempts = job.opts.attempts ?? 0;
|
||||
|
||||
return `id=${job.id} attempts=${currentAttempts}/${maxAttempts} age=${formated}`;
|
||||
}
|
||||
@ -126,20 +126,30 @@ export class QueueProcessorService implements OnApplicationShutdown {
|
||||
) {
|
||||
this.logger = this.queueLoggerService.logger;
|
||||
|
||||
function renderError(e: Error): any {
|
||||
if (e) { // 何故かeがundefinedで来ることがある
|
||||
function renderError(e?: Error) {
|
||||
// 何故かeがundefinedで来ることがある
|
||||
if (!e) return '?';
|
||||
|
||||
if (e instanceof Bull.UnrecoverableError || e.name === 'AbortError') {
|
||||
return `${e.name}: ${e.message}`;
|
||||
}
|
||||
|
||||
return {
|
||||
stack: e.stack,
|
||||
message: e.message,
|
||||
name: e.name,
|
||||
};
|
||||
} else {
|
||||
return {
|
||||
stack: '?',
|
||||
message: '?',
|
||||
name: '?',
|
||||
};
|
||||
}
|
||||
|
||||
function renderJob(job?: Bull.Job) {
|
||||
if (!job) return '?';
|
||||
|
||||
return {
|
||||
name: job.name || undefined,
|
||||
info: getJobInfo(job),
|
||||
failedReason: job.failedReason || undefined,
|
||||
data: job.data,
|
||||
};
|
||||
}
|
||||
|
||||
//#region system
|
||||
@ -175,15 +185,15 @@ export class QueueProcessorService implements OnApplicationShutdown {
|
||||
.on('active', (job) => logger.debug(`active id=${job.id}`))
|
||||
.on('completed', (job, result) => logger.debug(`completed(${result}) id=${job.id}`))
|
||||
.on('failed', (job, err: Error) => {
|
||||
logger.error(`failed(${err.stack}) id=${job ? job.id : '-'}`, { job, e: renderError(err) });
|
||||
logger.error(`failed(${err.name}: ${err.message}) id=${job?.id ?? '?'}`, { job: renderJob(job), e: renderError(err) });
|
||||
if (config.sentryForBackend) {
|
||||
Sentry.captureMessage(`Queue: System: ${job?.name ?? '?'}: ${err.message}`, {
|
||||
Sentry.captureMessage(`Queue: System: ${job?.name ?? '?'}: ${err.name}: ${err.message}`, {
|
||||
level: 'error',
|
||||
extra: { job, err },
|
||||
});
|
||||
}
|
||||
})
|
||||
.on('error', (err: Error) => logger.error(`error ${err.stack}`, { e: renderError(err) }))
|
||||
.on('error', (err: Error) => logger.error(`error ${err.name}: ${err.message}`, { e: renderError(err) }))
|
||||
.on('stalled', (jobId) => logger.warn(`stalled id=${jobId}`));
|
||||
}
|
||||
//#endregion
|
||||
@ -232,15 +242,15 @@ export class QueueProcessorService implements OnApplicationShutdown {
|
||||
.on('active', (job) => logger.debug(`active id=${job.id}`))
|
||||
.on('completed', (job, result) => logger.debug(`completed(${result}) id=${job.id}`))
|
||||
.on('failed', (job, err) => {
|
||||
logger.error(`failed(${err.stack}) id=${job ? job.id : '-'}`, { job, e: renderError(err) });
|
||||
logger.error(`failed(${err.name}: ${err.message}) id=${job?.id ?? '?'}`, { job: renderJob(job), e: renderError(err) });
|
||||
if (config.sentryForBackend) {
|
||||
Sentry.captureMessage(`Queue: DB: ${job?.name ?? '?'}: ${err.message}`, {
|
||||
Sentry.captureMessage(`Queue: DB: ${job?.name ?? '?'}: ${err.name}: ${err.message}`, {
|
||||
level: 'error',
|
||||
extra: { job, err },
|
||||
});
|
||||
}
|
||||
})
|
||||
.on('error', (err: Error) => logger.error(`error ${err.stack}`, { e: renderError(err) }))
|
||||
.on('error', (err: Error) => logger.error(`error ${err.name}: ${err.message}`, { e: renderError(err) }))
|
||||
.on('stalled', (jobId) => logger.warn(`stalled id=${jobId}`));
|
||||
}
|
||||
//#endregion
|
||||
@ -272,15 +282,15 @@ export class QueueProcessorService implements OnApplicationShutdown {
|
||||
.on('active', (job) => logger.debug(`active ${getJobInfo(job, true)} to=${job.data.to}`))
|
||||
.on('completed', (job, result) => logger.debug(`completed(${result}) ${getJobInfo(job, true)} to=${job.data.to}`))
|
||||
.on('failed', (job, err) => {
|
||||
logger.error(`failed(${err.stack}) ${getJobInfo(job)} to=${job ? job.data.to : '-'}`);
|
||||
logger.error(`failed(${err.name}: ${err.message}) ${getJobInfo(job)} to=${job ? job.data.to : '-'}`);
|
||||
if (config.sentryForBackend) {
|
||||
Sentry.captureMessage(`Queue: Deliver: ${err.message}`, {
|
||||
Sentry.captureMessage(`Queue: Deliver: ${err.name}: ${err.message}`, {
|
||||
level: 'error',
|
||||
extra: { job, err },
|
||||
});
|
||||
}
|
||||
})
|
||||
.on('error', (err: Error) => logger.error(`error ${err.stack}`, { e: renderError(err) }))
|
||||
.on('error', (err: Error) => logger.error(`error ${err.name}: ${err.message}`, { e: renderError(err) }))
|
||||
.on('stalled', (jobId) => logger.warn(`stalled id=${jobId}`));
|
||||
}
|
||||
//#endregion
|
||||
@ -312,15 +322,15 @@ export class QueueProcessorService implements OnApplicationShutdown {
|
||||
.on('active', (job) => logger.debug(`active ${getJobInfo(job, true)}`))
|
||||
.on('completed', (job, result) => logger.debug(`completed(${result}) ${getJobInfo(job, true)}`))
|
||||
.on('failed', (job, err) => {
|
||||
logger.error(`failed(${err.stack}) ${getJobInfo(job)} activity=${job ? (job.data.activity ? job.data.activity.id : 'none') : '-'}`, { job, e: renderError(err) });
|
||||
logger.error(`failed(${err.name}: ${err.message}) ${getJobInfo(job)} activity=${job ? (job.data.activity ? job.data.activity.id : 'none') : '-'}`, { job: renderJob(job), e: renderError(err) });
|
||||
if (config.sentryForBackend) {
|
||||
Sentry.captureMessage(`Queue: Inbox: ${err.message}`, {
|
||||
Sentry.captureMessage(`Queue: Inbox: ${err.name}: ${err.message}`, {
|
||||
level: 'error',
|
||||
extra: { job, err },
|
||||
});
|
||||
}
|
||||
})
|
||||
.on('error', (err: Error) => logger.error(`error ${err.stack}`, { e: renderError(err) }))
|
||||
.on('error', (err: Error) => logger.error(`error ${err.name}: ${err.message}`, { e: renderError(err) }))
|
||||
.on('stalled', (jobId) => logger.warn(`stalled id=${jobId}`));
|
||||
}
|
||||
//#endregion
|
||||
@ -352,15 +362,15 @@ export class QueueProcessorService implements OnApplicationShutdown {
|
||||
.on('active', (job) => logger.debug(`active ${getJobInfo(job, true)} to=${job.data.to}`))
|
||||
.on('completed', (job, result) => logger.debug(`completed(${result}) ${getJobInfo(job, true)} to=${job.data.to}`))
|
||||
.on('failed', (job, err) => {
|
||||
logger.error(`failed(${err.stack}) ${getJobInfo(job)} to=${job ? job.data.to : '-'}`);
|
||||
logger.error(`failed(${err.name}: ${err.message}) ${getJobInfo(job)} to=${job ? job.data.to : '-'}`);
|
||||
if (config.sentryForBackend) {
|
||||
Sentry.captureMessage(`Queue: UserWebhookDeliver: ${err.message}`, {
|
||||
Sentry.captureMessage(`Queue: UserWebhookDeliver: ${err.name}: ${err.message}`, {
|
||||
level: 'error',
|
||||
extra: { job, err },
|
||||
});
|
||||
}
|
||||
})
|
||||
.on('error', (err: Error) => logger.error(`error ${err.stack}`, { e: renderError(err) }))
|
||||
.on('error', (err: Error) => logger.error(`error ${err.name}: ${err.message}`, { e: renderError(err) }))
|
||||
.on('stalled', (jobId) => logger.warn(`stalled id=${jobId}`));
|
||||
}
|
||||
//#endregion
|
||||
@ -392,15 +402,15 @@ export class QueueProcessorService implements OnApplicationShutdown {
|
||||
.on('active', (job) => logger.debug(`active ${getJobInfo(job, true)} to=${job.data.to}`))
|
||||
.on('completed', (job, result) => logger.debug(`completed(${result}) ${getJobInfo(job, true)} to=${job.data.to}`))
|
||||
.on('failed', (job, err) => {
|
||||
logger.error(`failed(${err.stack}) ${getJobInfo(job)} to=${job ? job.data.to : '-'}`);
|
||||
logger.error(`failed(${err.name}: ${err.message}) ${getJobInfo(job)} to=${job ? job.data.to : '-'}`);
|
||||
if (config.sentryForBackend) {
|
||||
Sentry.captureMessage(`Queue: SystemWebhookDeliver: ${err.message}`, {
|
||||
Sentry.captureMessage(`Queue: SystemWebhookDeliver: ${err.name}: ${err.message}`, {
|
||||
level: 'error',
|
||||
extra: { job, err },
|
||||
});
|
||||
}
|
||||
})
|
||||
.on('error', (err: Error) => logger.error(`error ${err.stack}`, { e: renderError(err) }))
|
||||
.on('error', (err: Error) => logger.error(`error ${err.name}: ${err.message}`, { e: renderError(err) }))
|
||||
.on('stalled', (jobId) => logger.warn(`stalled id=${jobId}`));
|
||||
}
|
||||
//#endregion
|
||||
@ -439,15 +449,15 @@ export class QueueProcessorService implements OnApplicationShutdown {
|
||||
.on('active', (job) => logger.debug(`active id=${job.id}`))
|
||||
.on('completed', (job, result) => logger.debug(`completed(${result}) id=${job.id}`))
|
||||
.on('failed', (job, err) => {
|
||||
logger.error(`failed(${err.stack}) id=${job ? job.id : '-'}`, { job, e: renderError(err) });
|
||||
logger.error(`failed(${err.name}: ${err.message}) id=${job?.id ?? '?'}`, { job: renderJob(job), e: renderError(err) });
|
||||
if (config.sentryForBackend) {
|
||||
Sentry.captureMessage(`Queue: Relationship: ${job?.name ?? '?'}: ${err.message}`, {
|
||||
Sentry.captureMessage(`Queue: Relationship: ${job?.name ?? '?'}: ${err.name}: ${err.message}`, {
|
||||
level: 'error',
|
||||
extra: { job, err },
|
||||
});
|
||||
}
|
||||
})
|
||||
.on('error', (err: Error) => logger.error(`error ${err.stack}`, { e: renderError(err) }))
|
||||
.on('error', (err: Error) => logger.error(`error ${err.name}: ${err.message}`, { e: renderError(err) }))
|
||||
.on('stalled', (jobId) => logger.warn(`stalled id=${jobId}`));
|
||||
}
|
||||
//#endregion
|
||||
@ -480,15 +490,15 @@ export class QueueProcessorService implements OnApplicationShutdown {
|
||||
.on('active', (job) => logger.debug(`active id=${job.id}`))
|
||||
.on('completed', (job, result) => logger.debug(`completed(${result}) id=${job.id}`))
|
||||
.on('failed', (job, err) => {
|
||||
logger.error(`failed(${err.stack}) id=${job ? job.id : '-'}`, { job, e: renderError(err) });
|
||||
logger.error(`failed(${err.name}: ${err.message}) id=${job?.id ?? '?'}`, { job: renderJob(job), e: renderError(err) });
|
||||
if (config.sentryForBackend) {
|
||||
Sentry.captureMessage(`Queue: ObjectStorage: ${job?.name ?? '?'}: ${err.message}`, {
|
||||
Sentry.captureMessage(`Queue: ObjectStorage: ${job?.name ?? '?'}: ${err.name}: ${err.message}`, {
|
||||
level: 'error',
|
||||
extra: { job, err },
|
||||
});
|
||||
}
|
||||
})
|
||||
.on('error', (err: Error) => logger.error(`error ${err.stack}`, { e: renderError(err) }))
|
||||
.on('error', (err: Error) => logger.error(`error ${err.name}: ${err.message}`, { e: renderError(err) }))
|
||||
.on('stalled', (jobId) => logger.warn(`stalled id=${jobId}`));
|
||||
}
|
||||
//#endregion
|
||||
|
||||
Loading…
Reference in New Issue
Block a user