From d2b3b4b9e757e66c33fbad64aee270c6cfcfdad0 Mon Sep 17 00:00:00 2001 From: Xeltica Date: Thu, 23 Feb 2023 19:37:10 +0900 Subject: [PATCH] =?UTF-8?q?=E3=82=A2=E3=83=A9=E3=83=BC=E3=83=88=E6=A9=9F?= =?UTF-8?q?=E8=83=BD=E3=81=AE=E5=85=A8=E9=9D=A2=E7=9A=84=E3=81=AA=E6=9B=B8?= =?UTF-8?q?=E3=81=8D=E7=9B=B4=E3=81=97=20close=20#67?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/backend/functions/error-to-string.ts | 8 ++ src/backend/functions/get-scores.ts | 24 ++-- src/backend/functions/update-score.ts | 11 +- src/backend/models/acct.ts | 3 + src/backend/models/count.ts | 6 + src/backend/services/misskey.ts | 44 ++++++- src/backend/services/send-alert.ts | 30 +---- src/backend/services/worker.ts | 153 ++++++++++++++--------- src/backend/store.ts | 19 ++- src/backend/utils/delay.ts | 1 + src/backend/utils/group-by.ts | 13 ++ src/backend/views/frontend.pug | 3 - src/common/functions/format.ts | 10 +- 13 files changed, 207 insertions(+), 118 deletions(-) create mode 100644 src/backend/functions/error-to-string.ts create mode 100644 src/backend/models/acct.ts create mode 100644 src/backend/models/count.ts create mode 100644 src/backend/utils/delay.ts create mode 100644 src/backend/utils/group-by.ts diff --git a/src/backend/functions/error-to-string.ts b/src/backend/functions/error-to-string.ts new file mode 100644 index 0000000..37e6934 --- /dev/null +++ b/src/backend/functions/error-to-string.ts @@ -0,0 +1,8 @@ +import {MisskeyError} from '../services/misskey'; + +export const errorToString = (e: Error) => { + if (e instanceof MisskeyError) { + return JSON.stringify(e.error); + } + return `${e.name}: ${e.message}\n${e.stack}`; +}; diff --git a/src/backend/functions/get-scores.ts b/src/backend/functions/get-scores.ts index 88b66cd..70adf25 100644 --- a/src/backend/functions/get-scores.ts +++ b/src/backend/functions/get-scores.ts @@ -1,25 +1,17 @@ import { User } from '../models/entities/user'; -import { Score } from '../../common/types/score'; -import { api } from '../services/misskey'; import { toSignedString } from '../../common/functions/to-signed-string'; +import {Count} from '../models/count'; /** - * ユーザーのスコアを取得します。 + * ユーザーのスコア差分を取得します。 * @param user ユーザー - * @returns ユーザーのスコア + * @param count 統計 + * @returns ユーザーのスコア差分 */ -export const getScores = async (user: User): Promise => { - // TODO 毎回取ってくるのも微妙なので、ある程度キャッシュしたいかも - const miUser = await api>(user.host, 'users/show', { username: user.username }, user.token); - if (miUser.error) { - throw miUser.error; - } +export const getDelta = (user: User, count: Count) => { return { - notesCount: miUser.notesCount, - followingCount: miUser.followingCount, - followersCount: miUser.followersCount, - notesDelta: toSignedString(miUser.notesCount - user.prevNotesCount), - followingDelta: toSignedString(miUser.followingCount - user.prevFollowingCount), - followersDelta: toSignedString(miUser.followersCount - user.prevFollowersCount), + notesDelta: toSignedString(count.notesCount - user.prevNotesCount), + followingDelta: toSignedString(count.followingCount - user.prevFollowingCount), + followersDelta: toSignedString(count.followersCount - user.prevFollowersCount), }; }; diff --git a/src/backend/functions/update-score.ts b/src/backend/functions/update-score.ts index 018893e..2e840c3 100644 --- a/src/backend/functions/update-score.ts +++ b/src/backend/functions/update-score.ts @@ -1,5 +1,6 @@ import { User } from '../models/entities/user'; import { updateUser } from './users'; +import {Count} from '../models/count'; /** * Misskeyのユーザーモデル @@ -14,12 +15,12 @@ export type MiUser = { /** * スコアを更新します * @param user ユーザー - * @param miUser Misskeyのユーザー + * @param count 統計 */ -export const updateScore = async (user: User, miUser: MiUser): Promise => { +export const updateScore = async (user: User, count: Count): Promise => { await updateUser(user.username, user.host, { - prevNotesCount: miUser.notesCount ?? 0, - prevFollowingCount: miUser.followingCount ?? 0, - prevFollowersCount: miUser.followersCount ?? 0, + prevNotesCount: count.notesCount ?? 0, + prevFollowingCount: count.followingCount ?? 0, + prevFollowersCount: count.followersCount ?? 0, }); }; diff --git a/src/backend/models/acct.ts b/src/backend/models/acct.ts new file mode 100644 index 0000000..138bc2d --- /dev/null +++ b/src/backend/models/acct.ts @@ -0,0 +1,3 @@ +export type Acct = `@${string}@${string}`; + +export const toAcct = (it: {username: string, host: string}): Acct => `@${it.username}@${it.host}`; diff --git a/src/backend/models/count.ts b/src/backend/models/count.ts new file mode 100644 index 0000000..65e058e --- /dev/null +++ b/src/backend/models/count.ts @@ -0,0 +1,6 @@ +export interface Count +{ + notesCount: number; + followingCount: number; + followersCount: number; +} diff --git a/src/backend/services/misskey.ts b/src/backend/services/misskey.ts index 3428a04..3fe6e71 100644 --- a/src/backend/services/misskey.ts +++ b/src/backend/services/misskey.ts @@ -1,16 +1,35 @@ import axios from 'axios'; +import {printLog} from '../store'; +import {delay} from '../utils/delay'; -export const ua = `Mozilla/5.0 MisskeyTools +https://github.com/Xeltica/MisskeyTools Node/${process.version}`; +export const ua = `Mozilla/5.0 MisskeyTools +https://github.com/shrimpia/misskey-tools Node/${process.version}`; + +const RETRY_COUNT = 5; /** * Misskey APIを呼び出す */ -export const api = >(host: string, endpoint: string, arg: Record, i?: string): Promise => { +export const api = async >(host: string, endpoint: string, arg: Record, token?: string): Promise => { const a = { ...arg }; - if (i) { - a.i = i; + if (token) { + a.i = token; } - return axios.post(`https://${host}/api/${endpoint}`, a).then(res => res.data); + + for (let i = 0; i < RETRY_COUNT; i++) { + let data: T; + try { + data = await axios.post(`https://${host}/api/${endpoint}`, a).then(res => res.data); + } catch (e) { + printLog(`接続エラー: ${host}/api/${endpoint} リトライ中 (${i + 1} / ${RETRY_COUNT})\n${e}`, 'error'); + await delay(3000); + continue; + } + if (!('error' in data)) { + return data; + } + throw new MisskeyError((data as any).error); + } + throw new TimedOutError(); }; /** @@ -27,3 +46,18 @@ export const apiAvailable = async (host: string, i: string): Promise => return false; } }; + +export class TimedOutError extends Error {} + +export class MisskeyError extends Error { + constructor(public error: MisskeyErrorObject) { + super(); + } +} + +export interface MisskeyErrorObject { + message: string; + code: string; + id: string; + kind: string; +} diff --git a/src/backend/services/send-alert.ts b/src/backend/services/send-alert.ts index 737d71d..d313278 100644 --- a/src/backend/services/send-alert.ts +++ b/src/backend/services/send-alert.ts @@ -1,36 +1,12 @@ import { User } from '../models/entities/user'; -import { format } from '../../common/functions/format'; -import { getScores } from '../functions/get-scores'; import { api } from './misskey'; -/** - * アラートを送信する - * @param user ユーザー - */ -export const sendAlert = async (user: User) => { - const text = format(await getScores(user), user); - switch (user.alertMode) { - case 'note': - await sendNoteAlert(text, user); - break; - case 'notification': - await sendNotificationAlert(text, user); - break; - case 'both': - await Promise.all([ - sendNotificationAlert(text, user), - sendNoteAlert(text, user), - ]); - break; - } -}; - /** * ノートアラートを送信する * @param text 通知内容 * @param user ユーザー */ -const sendNoteAlert = async (text: string, user: User) => { +export const sendNoteAlert = async (text: string, user: User) => { const res = await api>(user.host, 'notes/create', { text, visibility: user.visibility, @@ -48,9 +24,9 @@ const sendNoteAlert = async (text: string, user: User) => { * @param text 通知内容 * @param user ユーザー */ -const sendNotificationAlert = async (text: string, user: User) => { +export const sendNotificationAlert = async (text: string, user: User) => { const res = await api(user.host, 'notifications/create', { - header: 'みす廃あらーと', + header: 'Misskey Tools', icon: 'https://i.imgur.com/B991yTl.png', body: text, }, user.token); diff --git a/src/backend/services/worker.ts b/src/backend/services/worker.ts index 35e13e7..853d6b7 100644 --- a/src/backend/services/worker.ts +++ b/src/backend/services/worker.ts @@ -1,16 +1,23 @@ import cron from 'node-cron'; -import delay from 'delay'; -import { Not } from 'typeorm'; - import { deleteUser } from '../functions/users'; import { MiUser, updateScore } from '../functions/update-score'; import { updateRating } from '../functions/update-rating'; -import { AlertMode } from '../../common/types/alert-mode'; import { Users } from '../models'; -import { sendAlert } from './send-alert'; -import { api } from './misskey'; +import {sendNoteAlert, sendNotificationAlert} from './send-alert'; +import {api, MisskeyError, TimedOutError} from './misskey'; import * as Store from '../store'; import { User } from '../models/entities/user'; +import {groupBy} from '../utils/group-by'; +import {clearLog, printLog} from '../store'; +import {errorToString} from '../functions/error-to-string'; +import {Acct, toAcct} from '../models/acct'; +import {Count} from '../models/count'; +import {format} from '../../common/functions/format'; + +const ERROR_CODES_USER_REMOVED = ['NO_SUCH_USER', 'AUTHENTICATION_FAILED', 'YOUR_ACCOUNT_SUSPENDED']; + +// TODO: Redisで持つようにしたい +const userScoreCache = new Map(); export default (): void => { cron.schedule('0 0 0 * * *', work); @@ -20,71 +27,99 @@ export const work = async () => { Store.dispatch({ nowCalculating: true }); clearLog(); - printLog('Started.'); try { - const users = await Users.find({ alertMode: Not('nothing') }); - printLog('will process ' + users.length + ' accounts.'); - for (const user of users) { - await update(user).catch(e => handleError(user, e)); - printLog(`processed for ${user.username}@${user.host}`); + const users = await Users.find(); + const groupedUsers = groupBy(users, u => u.host); - if (user.alertMode === 'note') { - await delay(3000); - } - } - printLog('finished successfully.'); + printLog(`${users.length} アカウントのレート計算を開始します。`); + await calculateAllRating(groupedUsers); + Store.dispatch({ nowCalculating: false }); + + printLog(`${users.length} アカウントのアラート送信を開始します。`); + await sendAllAlerts(groupedUsers); + + printLog('ミス廃アラートワーカーは正常に完了しました。'); } catch (e) { - const msg = String(e instanceof Error ? e.stack : e); - printLog(msg); - printLog('stopped wrongly.'); + printLog('ミス廃アラートワーカーが異常終了しました。', 'error'); + printLog(e instanceof Error ? errorToString(e) : e, 'error'); } finally { Store.dispatch({ nowCalculating: false }); } }; -const clearLog = () => { - Store.dispatch({ misshaiWorkerLog: [] }); +const calculateAllRating = async (groupedUsers: [string, User[]][]) => { + return await Promise.all(groupedUsers.map(kv => calculateRating(...kv))); }; -const printLog = (log: any) => { - Store.dispatch({ misshaiWorkerLog: [ - ...Store.getState().misshaiWorkerLog, - String(log), - ] }); -}; - -/** - * アラートを送信します。 - * @param user アラートの送信先ユーザー - */ -const update = async (user: User) => { - const miUser = await api(user.host, 'users/show', { username: user.username }, user.token); - if (miUser.error) throw miUser.error; - - await updateRating(user, miUser); - await sendAlert(user); - - await updateScore(user, miUser); -}; - -/** - * アラート送信失敗のエラーをハンドリングします。 - * @param user 送信に失敗したアラートの送信先ユーザー - * @param e エラー。ErrorだったりObjectだったりするのでanyだけど、いずれ型定義したい - */ -const handleError = async (user: User, e: any) => { - if (e.code) { - if (e.code === 'NO_SUCH_USER' || e.code === 'AUTHENTICATION_FAILED') { - // ユーザーが削除されている場合、レコードからも消してとりやめ - printLog(`${user.username}@${user.host} is deleted, so delete this user from the system`); - await deleteUser(user.username, user.host); - } else { - printLog(`Misskey Error: ${JSON.stringify(e)}`); +const calculateRating = async (host: string, users: User[]) => { + for (const user of users) { + let miUser: MiUser; + try { + miUser = await api(user.host, 'i', {}, user.token); + } catch (e) { + if (!(e instanceof Error)) { + printLog('バグ：エラーオブジェクトはErrorを継承していないといけない', 'error'); + } else if (e instanceof MisskeyError) { + if (ERROR_CODES_USER_REMOVED.includes(e.error.code)) { + // ユーザーが削除されている場合、レコードからも消してとりやめ + printLog(`アカウント ${toAcct(user)} は削除されているか、凍結されているか、トークンを失効しています。そのため、本システムからアカウントを削除します。`, 'warn'); + await deleteUser(user.username, user.host); + } else { + printLog(`Misskey エラー: ${JSON.stringify(e.error)}`, 'error'); + } + } else if (e instanceof TimedOutError) { + printLog(`サーバー ${user.host} との接続に失敗したため、このサーバーのレート計算を中断します。`, 'error'); + return; + } else { + // おそらく通信エラー + printLog(`不明なエラーが発生しました。\n${errorToString(e)}`, 'error'); + } + continue; } - } else { - // おそらく通信エラー - printLog(`Unknown error: ${e.name} ${e.message}`); + userScoreCache.set(toAcct(user), miUser); + + await updateRating(user, miUser); } + printLog(`${host} ユーザー(${users.length}人) のレート計算が完了しました。`); +}; + +const sendAllAlerts = async (groupedUsers: [string, User[]][]) => { + return await Promise.all(groupedUsers.map(kv => sendAlerts(...kv))); +}; + +const sendAlerts = async (host: string, users: User[]) => { + const models = users + .map(user => { + const count = userScoreCache.get(toAcct(user)); + if (count == null) return null; + return { + user, + count, + message: format(user, count), + }; + }) + .filter(u => u != null) as {user: User, count: Count, message: string}[]; + + // 何もしない + for (const {user, count} of models.filter(m => m.user.alertMode === 'nothing')) { + await updateScore(user, count); + } + + // 通知 + for (const {user, count, message} of models.filter(m => m.user.alertMode === 'notification' || m.user.alertMode === 'both')) { + await sendNotificationAlert(message, user); + if (user.alertMode === 'notification') { + await updateScore(user, count); + } + } + + // 通知 + for (const {user, count, message} of models.filter(m => m.user.alertMode === 'note')) { + await sendNoteAlert(message, user); + await updateScore(user, count); + } + + printLog(`${host} ユーザー(${users.length}人) へのアラート送信が完了しました。`); }; diff --git a/src/backend/store.ts b/src/backend/store.ts index 0bfe19d..a724797 100644 --- a/src/backend/store.ts +++ b/src/backend/store.ts @@ -17,7 +17,7 @@ let _state: Readonly = defaultState; */ export type State = { nowCalculating: boolean, - misshaiWorkerLog: string[], + misshaiWorkerLog: Log[], }; /** @@ -36,3 +36,20 @@ export const dispatch = (mutation: Partial) => { ...mutation, }; }; + +export type Log = { + text: string; + level: 'error' | 'warn' | 'info'; + timestamp: Date; +} + +export const clearLog = () => { + dispatch({ misshaiWorkerLog: [] }); +}; + +export const printLog = (log: unknown, level: Log['level'] = 'info') => { + dispatch({ misshaiWorkerLog: [ + ...getState().misshaiWorkerLog, + { text: String(log), level, timestamp: new Date() }, + ] }); +}; diff --git a/src/backend/utils/delay.ts b/src/backend/utils/delay.ts new file mode 100644 index 0000000..3c85a4d --- /dev/null +++ b/src/backend/utils/delay.ts @@ -0,0 +1 @@ +export const delay = (ms: number) => new Promise(res => setTimeout(res, ms)); diff --git a/src/backend/utils/group-by.ts b/src/backend/utils/group-by.ts new file mode 100644 index 0000000..56c35ed --- /dev/null +++ b/src/backend/utils/group-by.ts @@ -0,0 +1,13 @@ +type GetKeyFunction = (cur: V, idx: number, src: readonly V[]) => K; + +export const groupBy = (array: readonly V[], getKey: GetKeyFunction) => { + return Array.from( + array.reduce((map, cur, idx, src) => { + const key = getKey(cur, idx, src); + const list = map.get(key); + if (list) list.push(cur); + else map.set(key, [cur]); + return map; + }, new Map()) + ); +}; diff --git a/src/backend/views/frontend.pug b/src/backend/views/frontend.pug index 3cb8724..cd9a4b6 100644 --- a/src/backend/views/frontend.pug +++ b/src/backend/views/frontend.pug @@ -11,9 +11,6 @@ html meta(property='og:title' content=title) meta(property='og:description' content=desc) meta(property='og:type' content='website') - meta(name='twitter:card' content='summary') - meta(name='twitter:site' content='@Xeltica') - meta(name='twitter:creator' content='@Xeltica') link(rel="preload" href="https://koruri.chillout.chat/koruri.css") link(rel="preload", href="/assets/otadesign_rounded.woff") link(rel="preload", href="/assets/otadesign_rounded.woff2") diff --git a/src/common/functions/format.ts b/src/common/functions/format.ts index d1c9c35..e3f5654 100644 --- a/src/common/functions/format.ts +++ b/src/common/functions/format.ts @@ -3,6 +3,8 @@ import { Score } from '../types/score'; import { defaultTemplate } from '../../backend/const'; import { IUser } from '../types/user'; import { createGacha } from './create-gacha'; +import {Count} from '../../backend/models/count'; +import {getDelta} from '../../backend/functions/get-scores'; /** * 埋め込み変数の型 @@ -30,11 +32,15 @@ const variableRegex = /\{([a-zA-Z0-9_]+?)\}/g; /** * スコア情報とユーザー情報からテキストを生成する - * @param score スコア情報 * @param user ユーザー情報 + * @param count カウント * @returns 生成したテキスト */ -export const format = (score: Score, user: IUser): string => { +export const format = (user: IUser, count: Count): string => { + const score: Score = { + ...count, + ...getDelta(user, count), + }; const template = user.template || defaultTemplate; return template.replace(variableRegex, (m, name) => { const v = variables[name];