From f4838e50b4043f917020dd1cfa7b75da087ff8f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E3=81=8B=E3=81=A3=E3=81=93=E3=81=8B=E3=82=8A?= <67428053+kakkokari-gtyih@users.noreply.github.com> Date: Thu, 21 Mar 2024 07:51:01 +0900 Subject: [PATCH 1/9] =?UTF-8?q?enhance(antenna):=20Bot=E3=81=AE=E6=8A=95?= =?UTF-8?q?=E7=A8=BF=E3=82=92=E9=99=A4=E5=A4=96=E3=81=A7=E3=81=8D=E3=82=8B?= =?UTF-8?q?=E3=82=88=E3=81=86=E3=81=AB=20(#13603)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * enhance(antenna): Botの投稿を除外できるように (MisskeyIO#545) (cherry picked from commit a95ce067c6cf0a93647e358aabc984bdbe99e952) * Update Changelog * remove translations * spdx --------- Co-authored-by: まっちゃとーにゅ <17376330+u1-liquid@users.noreply.github.com> --- CHANGELOG.md | 2 ++ locales/index.d.ts | 4 ++++ locales/ja-JP.yml | 1 + .../1710919614510-antenna-exclude-bots.js | 16 ++++++++++++++++ packages/backend/src/core/AntennaService.ts | 6 ++++-- .../src/core/entities/AntennaEntityService.ts | 1 + packages/backend/src/models/Antenna.ts | 5 +++++ .../backend/src/models/json-schema/antenna.ts | 5 +++++ .../processors/ExportAntennasProcessorService.ts | 1 + .../processors/ImportAntennasProcessorService.ts | 2 ++ .../src/server/api/endpoints/antennas/create.ts | 2 ++ .../src/server/api/endpoints/antennas/update.ts | 2 ++ packages/backend/test/e2e/antennas.ts | 2 ++ .../frontend/src/pages/my-antennas/create.vue | 1 + .../frontend/src/pages/my-antennas/editor.vue | 3 +++ packages/misskey-js/src/autogen/types.ts | 4 ++++ 16 files changed, 55 insertions(+), 2 deletions(-) create mode 100644 packages/backend/migration/1710919614510-antenna-exclude-bots.js diff --git a/CHANGELOG.md b/CHANGELOG.md index 18dd07f1c..0dce1a049 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,8 @@ ## Unreleased ### General +- Enhance: アンテナでBotによるノートを除外できるように + (Cherry-picked from https://github.com/MisskeyIO/misskey/pull/545) - Fix: Play作成時に設定した公開範囲が機能していない問題を修正 ### Client diff --git a/locales/index.d.ts b/locales/index.d.ts index 7f4ec7ecb..afb4adac6 100644 --- a/locales/index.d.ts +++ b/locales/index.d.ts @@ -1616,6 +1616,10 @@ export interface Locale extends ILocale { * 除外キーワード */ "antennaExcludeKeywords": string; + /** + * Botアカウントを除外 + */ + "antennaExcludeBots": string; /** * スペースで区切るとAND指定になり、改行で区切るとOR指定になります */ diff --git a/locales/ja-JP.yml b/locales/ja-JP.yml index 8b44ac212..a64c83b10 100644 --- a/locales/ja-JP.yml +++ b/locales/ja-JP.yml @@ -400,6 +400,7 @@ name: "名前" antennaSource: "受信ソース" antennaKeywords: "受信キーワード" antennaExcludeKeywords: "除外キーワード" +antennaExcludeBots: "Botアカウントを除外" antennaKeywordsDescription: "スペースで区切るとAND指定になり、改行で区切るとOR指定になります" notifyAntenna: "新しいノートを通知する" withFileAntenna: "ファイルが添付されたノートのみ" diff --git a/packages/backend/migration/1710919614510-antenna-exclude-bots.js b/packages/backend/migration/1710919614510-antenna-exclude-bots.js new file mode 100644 index 000000000..fac84317c --- /dev/null +++ b/packages/backend/migration/1710919614510-antenna-exclude-bots.js @@ -0,0 +1,16 @@ +/* + * SPDX-FileCopyrightText: syuilo and misskey-project + * SPDX-License-Identifier: AGPL-3.0-only + */ + +export class AntennaExcludeBots1710919614510 { + name = 'AntennaExcludeBots1710919614510' + + async up(queryRunner) { + await queryRunner.query(`ALTER TABLE "antenna" ADD "excludeBots" boolean NOT NULL DEFAULT false`); + } + + async down(queryRunner) { + await queryRunner.query(`ALTER TABLE "antenna" DROP COLUMN "excludeBots"`); + } +} diff --git a/packages/backend/src/core/AntennaService.ts b/packages/backend/src/core/AntennaService.ts index 4f956a43e..793d8974b 100644 --- a/packages/backend/src/core/AntennaService.ts +++ b/packages/backend/src/core/AntennaService.ts @@ -92,7 +92,7 @@ export class AntennaService implements OnApplicationShutdown { } @bindThis - public async addNoteToAntennas(note: MiNote, noteUser: { id: MiUser['id']; username: string; host: string | null; }): Promise { + public async addNoteToAntennas(note: MiNote, noteUser: { id: MiUser['id']; username: string; host: string | null; isBot: boolean; }): Promise { const antennas = await this.getAntennas(); const antennasWithMatchResult = await Promise.all(antennas.map(antenna => this.checkHitAntenna(antenna, note, noteUser).then(hit => [antenna, hit] as const))); const matchedAntennas = antennasWithMatchResult.filter(([, hit]) => hit).map(([antenna]) => antenna); @@ -110,10 +110,12 @@ export class AntennaService implements OnApplicationShutdown { // NOTE: フォローしているユーザーのノート、リストのユーザーのノート、グループのユーザーのノート指定はパフォーマンス上の理由で無効になっている @bindThis - public async checkHitAntenna(antenna: MiAntenna, note: (MiNote | Packed<'Note'>), noteUser: { id: MiUser['id']; username: string; host: string | null; }): Promise { + public async checkHitAntenna(antenna: MiAntenna, note: (MiNote | Packed<'Note'>), noteUser: { id: MiUser['id']; username: string; host: string | null; isBot: boolean; }): Promise { if (note.visibility === 'specified') return false; if (note.visibility === 'followers') return false; + if (antenna.excludeBots && noteUser.isBot) return false; + if (antenna.localOnly && noteUser.host != null) return false; if (!antenna.withReplies && note.replyId != null) return false; diff --git a/packages/backend/src/core/entities/AntennaEntityService.ts b/packages/backend/src/core/entities/AntennaEntityService.ts index 64d6a3c97..3ec8efa6b 100644 --- a/packages/backend/src/core/entities/AntennaEntityService.ts +++ b/packages/backend/src/core/entities/AntennaEntityService.ts @@ -39,6 +39,7 @@ export class AntennaEntityService { caseSensitive: antenna.caseSensitive, localOnly: antenna.localOnly, notify: antenna.notify, + excludeBots: antenna.excludeBots, withReplies: antenna.withReplies, withFile: antenna.withFile, isActive: antenna.isActive, diff --git a/packages/backend/src/models/Antenna.ts b/packages/backend/src/models/Antenna.ts index 332a89976..f5e819059 100644 --- a/packages/backend/src/models/Antenna.ts +++ b/packages/backend/src/models/Antenna.ts @@ -72,6 +72,11 @@ export class MiAntenna { }) public caseSensitive: boolean; + @Column('boolean', { + default: false, + }) + public excludeBots: boolean; + @Column('boolean', { default: false, }) diff --git a/packages/backend/src/models/json-schema/antenna.ts b/packages/backend/src/models/json-schema/antenna.ts index 74622b619..78cf6d3ba 100644 --- a/packages/backend/src/models/json-schema/antenna.ts +++ b/packages/backend/src/models/json-schema/antenna.ts @@ -76,6 +76,11 @@ export const packedAntennaSchema = { type: 'boolean', optional: false, nullable: false, }, + excludeBots: { + type: 'boolean', + optional: false, nullable: false, + default: false, + }, withReplies: { type: 'boolean', optional: false, nullable: false, diff --git a/packages/backend/src/queue/processors/ExportAntennasProcessorService.ts b/packages/backend/src/queue/processors/ExportAntennasProcessorService.ts index af48bad41..1d8e90f36 100644 --- a/packages/backend/src/queue/processors/ExportAntennasProcessorService.ts +++ b/packages/backend/src/queue/processors/ExportAntennasProcessorService.ts @@ -81,6 +81,7 @@ export class ExportAntennasProcessorService { }) : null, caseSensitive: antenna.caseSensitive, localOnly: antenna.localOnly, + excludeBots: antenna.excludeBots, withReplies: antenna.withReplies, withFile: antenna.withFile, notify: antenna.notify, diff --git a/packages/backend/src/queue/processors/ImportAntennasProcessorService.ts b/packages/backend/src/queue/processors/ImportAntennasProcessorService.ts index 951b56059..ff1c04de0 100644 --- a/packages/backend/src/queue/processors/ImportAntennasProcessorService.ts +++ b/packages/backend/src/queue/processors/ImportAntennasProcessorService.ts @@ -44,6 +44,7 @@ const validate = new Ajv().compile({ } }, caseSensitive: { type: 'boolean' }, localOnly: { type: 'boolean' }, + excludeBots: { type: 'boolean' }, withReplies: { type: 'boolean' }, withFile: { type: 'boolean' }, notify: { type: 'boolean' }, @@ -88,6 +89,7 @@ export class ImportAntennasProcessorService { users: (antenna.src === 'list' && antenna.userListAccts !== null ? antenna.userListAccts : antenna.users).filter(Boolean), caseSensitive: antenna.caseSensitive, localOnly: antenna.localOnly, + excludeBots: antenna.excludeBots, withReplies: antenna.withReplies, withFile: antenna.withFile, notify: antenna.notify, diff --git a/packages/backend/src/server/api/endpoints/antennas/create.ts b/packages/backend/src/server/api/endpoints/antennas/create.ts index 191de8f83..57c8eb495 100644 --- a/packages/backend/src/server/api/endpoints/antennas/create.ts +++ b/packages/backend/src/server/api/endpoints/antennas/create.ts @@ -64,6 +64,7 @@ export const paramDef = { } }, caseSensitive: { type: 'boolean' }, localOnly: { type: 'boolean' }, + excludeBots: { type: 'boolean' }, withReplies: { type: 'boolean' }, withFile: { type: 'boolean' }, notify: { type: 'boolean' }, @@ -124,6 +125,7 @@ export default class extends Endpoint { // eslint- users: ps.users, caseSensitive: ps.caseSensitive, localOnly: ps.localOnly, + excludeBots: ps.excludeBots, withReplies: ps.withReplies, withFile: ps.withFile, notify: ps.notify, diff --git a/packages/backend/src/server/api/endpoints/antennas/update.ts b/packages/backend/src/server/api/endpoints/antennas/update.ts index 76a34924a..e6720aacf 100644 --- a/packages/backend/src/server/api/endpoints/antennas/update.ts +++ b/packages/backend/src/server/api/endpoints/antennas/update.ts @@ -63,6 +63,7 @@ export const paramDef = { } }, caseSensitive: { type: 'boolean' }, localOnly: { type: 'boolean' }, + excludeBots: { type: 'boolean' }, withReplies: { type: 'boolean' }, withFile: { type: 'boolean' }, notify: { type: 'boolean' }, @@ -120,6 +121,7 @@ export default class extends Endpoint { // eslint- users: ps.users, caseSensitive: ps.caseSensitive, localOnly: ps.localOnly, + excludeBots: ps.excludeBots, withReplies: ps.withReplies, withFile: ps.withFile, notify: ps.notify, diff --git a/packages/backend/test/e2e/antennas.ts b/packages/backend/test/e2e/antennas.ts index 7370b1963..cf5c7dd13 100644 --- a/packages/backend/test/e2e/antennas.ts +++ b/packages/backend/test/e2e/antennas.ts @@ -44,6 +44,7 @@ describe('アンテナ', () => { users: [''], withFile: false, withReplies: false, + excludeBots: false, }; let root: User; @@ -156,6 +157,7 @@ describe('アンテナ', () => { users: [''], withFile: false, withReplies: false, + excludeBots: false, localOnly: false, }; assert.deepStrictEqual(response, expected); diff --git a/packages/frontend/src/pages/my-antennas/create.vue b/packages/frontend/src/pages/my-antennas/create.vue index 8b3b3cfbf..2d026d2fa 100644 --- a/packages/frontend/src/pages/my-antennas/create.vue +++ b/packages/frontend/src/pages/my-antennas/create.vue @@ -26,6 +26,7 @@ const draft = ref({ users: [], keywords: [], excludeKeywords: [], + excludeBots: false, withReplies: false, caseSensitive: false, localOnly: false, diff --git a/packages/frontend/src/pages/my-antennas/editor.vue b/packages/frontend/src/pages/my-antennas/editor.vue index c6dcbadd9..97edbc44c 100644 --- a/packages/frontend/src/pages/my-antennas/editor.vue +++ b/packages/frontend/src/pages/my-antennas/editor.vue @@ -26,6 +26,7 @@ SPDX-License-Identifier: AGPL-3.0-only + {{ i18n.ts.antennaExcludeBots }} {{ i18n.ts.withReplies }} @@ -78,6 +79,7 @@ const keywords = ref(props.antenna.keywords.map(x => x.join(' ')).join(' const excludeKeywords = ref(props.antenna.excludeKeywords.map(x => x.join(' ')).join('\n')); const caseSensitive = ref(props.antenna.caseSensitive); const localOnly = ref(props.antenna.localOnly); +const excludeBots = ref(props.antenna.excludeBots); const withReplies = ref(props.antenna.withReplies); const withFile = ref(props.antenna.withFile); const notify = ref(props.antenna.notify); @@ -94,6 +96,7 @@ async function saveAntenna() { name: name.value, src: src.value, userListId: userListId.value, + excludeBots: excludeBots.value, withReplies: withReplies.value, withFile: withFile.value, notify: notify.value, diff --git a/packages/misskey-js/src/autogen/types.ts b/packages/misskey-js/src/autogen/types.ts index 3c862f690..636bc62aa 100644 --- a/packages/misskey-js/src/autogen/types.ts +++ b/packages/misskey-js/src/autogen/types.ts @@ -4434,6 +4434,8 @@ export type components = { localOnly: boolean; notify: boolean; /** @default false */ + excludeBots: boolean; + /** @default false */ withReplies: boolean; withFile: boolean; isActive: boolean; @@ -9654,6 +9656,7 @@ export type operations = { users: string[]; caseSensitive: boolean; localOnly?: boolean; + excludeBots?: boolean; withReplies: boolean; withFile: boolean; notify: boolean; @@ -9935,6 +9938,7 @@ export type operations = { users?: string[]; caseSensitive?: boolean; localOnly?: boolean; + excludeBots?: boolean; withReplies?: boolean; withFile?: boolean; notify?: boolean; From eb884721bb35dc19d0f2e8edc0033621493b752f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E3=81=BE=E3=81=A3=E3=81=A1=E3=82=83=E3=81=A8=E3=83=BC?= =?UTF-8?q?=E3=81=AB=E3=82=85?= <17376330+u1-liquid@users.noreply.github.com> Date: Thu, 21 Mar 2024 08:08:28 +0900 Subject: [PATCH 2/9] =?UTF-8?q?enhance(SSO):=20Attribute=E3=81=AE=E8=AA=BF?= =?UTF-8?q?=E6=95=B4=20(MisskeyIO#555)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../server/sso/JWTIdentifyProviderService.ts | 4 +++- .../server/sso/SAMLIdentifyProviderService.ts | 18 +++++++++++++----- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/packages/backend/src/server/sso/JWTIdentifyProviderService.ts b/packages/backend/src/server/sso/JWTIdentifyProviderService.ts index 182b6af04..2671a7fcb 100644 --- a/packages/backend/src/server/sso/JWTIdentifyProviderService.ts +++ b/packages/backend/src/server/sso/JWTIdentifyProviderService.ts @@ -172,7 +172,9 @@ export class JWTIdentifyProviderService { const roles = await this.roleService.getUserRoles(user.id); const payload: JWTPayload = { - name: user.name ?? user.username, + name: user.name ? `${user.name} (@${user.username})` : `@${user.username}`, + given_name: user.name ?? undefined, + family_name: `@${user.username}`, preferred_username: user.username, profile: `${this.config.url}/@${user.username}`, picture: user.avatarUrl ?? undefined, diff --git a/packages/backend/src/server/sso/SAMLIdentifyProviderService.ts b/packages/backend/src/server/sso/SAMLIdentifyProviderService.ts index 96c32291f..15ca1eecc 100644 --- a/packages/backend/src/server/sso/SAMLIdentifyProviderService.ts +++ b/packages/backend/src/server/sso/SAMLIdentifyProviderService.ts @@ -492,20 +492,28 @@ export class SAMLIdentifyProviderService { '#text': user.id, }, }, - { - '@Name': 'displayname', + ...(user.name ? [{ + '@Name': 'firstName', '@NameFormat': 'urn:oasis:names:tc:SAML:2.0:attrname-format:basic', 'saml:AttributeValue': { '@xsi:type': 'xs:string', - '#text': user.name ?? user.username, + '#text': user.name, + }, + }] : []), + { + '@Name': 'lastName', + '@NameFormat': 'urn:oasis:names:tc:SAML:2.0:attrname-format:basic', + 'saml:AttributeValue': { + '@xsi:type': 'xs:string', + '#text': `@${user.username}`, }, }, { - '@Name': 'name', + '@Name': 'displayName', '@NameFormat': 'urn:oasis:names:tc:SAML:2.0:attrname-format:basic', 'saml:AttributeValue': { '@xsi:type': 'xs:string', - '#text': user.username, + '#text': user.name ? `${user.name} (@${user.username})` : `@${user.username}`, }, }, { From 9a65c8ca618995f56683792f703c7354e176f07c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E3=81=BE=E3=81=A3=E3=81=A1=E3=82=83=E3=81=A8=E3=83=BC?= =?UTF-8?q?=E3=81=AB=E3=82=85?= <17376330+u1-liquid@users.noreply.github.com> Date: Thu, 21 Mar 2024 12:41:42 +0900 Subject: [PATCH 3/9] =?UTF-8?q?fix(frontend):=20=E5=90=8C=E3=81=98?= =?UTF-8?q?=E9=9F=B3=E6=BA=90=E3=81=AF=E7=9F=AD=E6=99=82=E9=96=93=E3=81=A7?= =?UTF-8?q?=E9=87=8D=E8=A4=87=E3=81=97=E3=81=A6=E6=B5=81=E3=82=8C=E3=81=AA?= =?UTF-8?q?=E3=81=84=E3=82=88=E3=81=86=E3=81=AB=20(MisskeyIO#556)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/frontend/src/scripts/sound.ts | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/packages/frontend/src/scripts/sound.ts b/packages/frontend/src/scripts/sound.ts index b6927e0d6..783baddc9 100644 --- a/packages/frontend/src/scripts/sound.ts +++ b/packages/frontend/src/scripts/sound.ts @@ -6,10 +6,10 @@ import type { SoundStore } from '@/store.js'; import { defaultStore } from '@/store.js'; import { $i } from '@/account.js'; +import { RateLimiter } from '@/scripts/rate-limiter.js'; let ctx: AudioContext; const cache = new Map(); -let canPlay = true; export const soundsTypes = [ // 音声なし @@ -188,17 +188,13 @@ export async function loadAudio(url: string, options?: { useCache?: boolean; }) */ export function playMisskeySfx(operationType: OperationType) { const sound = defaultStore.state[`sound_${operationType}`]; - if (sound.type == null || !canPlay || ('userActivation' in navigator && !navigator.userActivation.hasBeenActive)) return; + if (sound.type == null || ('userActivation' in navigator && !navigator.userActivation.hasBeenActive)) return; - canPlay = false; - playMisskeySfxFile(sound).finally(() => { - // ごく短時間に音が重複しないように - setTimeout(() => { - canPlay = true; - }, 25); - }); + playMisskeySfxFile(sound); } +const rateLimiter = new RateLimiter({ duration: 50, max: 1 }); + /** * サウンド設定形式で指定された音声を再生する * @param soundStore サウンド設定 @@ -213,7 +209,7 @@ export async function playMisskeySfxFile(soundStore: SoundStore) { } const url = soundStore.type === '_driveFile_' ? soundStore.fileUrl : `/client-assets/sounds/${soundStore.type}.mp3`; const buffer = await loadAudio(url); - if (!buffer) return; + if (!buffer || !rateLimiter.hit(url)) return; const volume = soundStore.volume * masterVolume; createSourceNode(buffer, { volume }).soundSource.start(); } From ee78828ddab3094f83289eda08e97e9132090964 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E3=81=BE=E3=81=A3=E3=81=A1=E3=82=83=E3=81=A8=E3=83=BC?= =?UTF-8?q?=E3=81=AB=E3=82=85?= <17376330+u1-liquid@users.noreply.github.com> Date: Thu, 21 Mar 2024 16:01:58 +0900 Subject: [PATCH 4/9] =?UTF-8?q?spec(backend):=20API=E3=81=AE=E3=83=AC?= =?UTF-8?q?=E3=83=BC=E3=83=88=E3=83=AA=E3=83=9F=E3=83=83=E3=83=88=E3=82=92?= =?UTF-8?q?=E3=83=90=E3=82=A4=E3=83=91=E3=82=B9=E3=81=A7=E3=81=8D=E3=82=8B?= =?UTF-8?q?=E3=82=88=E3=81=86=E3=81=AB=20(MisskeyIO#557)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/backend/src/config.ts | 4 ++++ packages/backend/src/server/api/ApiCallService.ts | 6 +++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/packages/backend/src/config.ts b/packages/backend/src/config.ts index 17c76b226..9b053a4b9 100644 --- a/packages/backend/src/config.ts +++ b/packages/backend/src/config.ts @@ -98,6 +98,8 @@ type Source = { proxyRemoteFiles?: boolean; videoThumbnailGenerator?: string; + bypassRateLimit?: { header: string; value: string }[]; + signToActivityPubGet?: boolean; perChannelMaxNoteCacheCount?: number; @@ -175,6 +177,7 @@ export type Config = { mediaProxy: string; externalMediaProxyEnabled: boolean; videoThumbnailGenerator: string | null; + bypassRateLimit: { header: string; value: string }[] | undefined; redis: RedisOptions & RedisOptionsSource; redisForPubsub: RedisOptions & RedisOptionsSource; redisForSystemQueue: RedisOptions & RedisOptionsSource; @@ -288,6 +291,7 @@ export function loadConfig(): Config { videoThumbnailGenerator: config.videoThumbnailGenerator ? config.videoThumbnailGenerator.endsWith('/') ? config.videoThumbnailGenerator.substring(0, config.videoThumbnailGenerator.length - 1) : config.videoThumbnailGenerator : null, + bypassRateLimit: config.bypassRateLimit, userAgent: `Misskey/${version} (${config.url})`, clientEntry: clientManifest['src/_boot_.ts'], clientManifestExists: clientManifestExists, diff --git a/packages/backend/src/server/api/ApiCallService.ts b/packages/backend/src/server/api/ApiCallService.ts index d8e792051..cd15721b0 100644 --- a/packages/backend/src/server/api/ApiCallService.ts +++ b/packages/backend/src/server/api/ApiCallService.ts @@ -18,6 +18,7 @@ import { createTemp } from '@/misc/create-temp.js'; import { bindThis } from '@/decorators.js'; import { RoleService } from '@/core/RoleService.js'; import { IdentifiableError } from '@/misc/identifiable-error.js'; +import type { Config } from '@/config.js'; import { ApiError } from './error.js'; import { RateLimiterService } from './RateLimiterService.js'; import { ApiLoggerService } from './ApiLoggerService.js'; @@ -39,6 +40,8 @@ export class ApiCallService implements OnApplicationShutdown { private userIpHistoriesClearIntervalId: NodeJS.Timeout; constructor( + @Inject(DI.config) + private config: Config, @Inject(DI.userIpsRepository) private userIpsRepository: UserIpsRepository, @@ -243,7 +246,8 @@ export class ApiCallService implements OnApplicationShutdown { throw new ApiError(accessDenied); } - if (ep.meta.limit) { + const bypassRateLimit = this.config.bypassRateLimit?.some(({ header, value }) => request.headers[header] === value) ?? false; + if (ep.meta.limit && !bypassRateLimit) { // koa will automatically load the `X-Forwarded-For` header if `proxy: true` is configured in the app. let limitActor: string; if (user) { From 831c74a25b2db0ba3f6d43a9a1a9072d342b2822 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E3=81=8A=E3=81=95=E3=82=80=E3=81=AE=E3=81=B2=E3=81=A8?= <46447427+samunohito@users.noreply.github.com> Date: Thu, 21 Mar 2024 18:46:42 +0900 Subject: [PATCH 5/9] =?UTF-8?q?fix:=20URL=E3=83=97=E3=83=AC=E3=83=93?= =?UTF-8?q?=E3=83=A5=E3=83=BC=E3=81=AE=E5=8B=95=E4=BD=9C=E6=94=B9=E5=96=84?= =?UTF-8?q?=EF=BC=8B=E5=8B=95=E4=BD=9C=E8=A8=AD=E5=AE=9A=E3=82=92=E5=8F=AF?= =?UTF-8?q?=E8=83=BD=E3=81=AB=E3=81=99=E3=82=8B=20(#13579)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * wip * support new version * URLプレビュー無効化時、フロント側も非表示にしてリクエストをしないようにする * fix lint * fix lint * tweak preview request error handles * fix: CHANGELOG.md * fix * fix --------- Co-authored-by: syuilo <4439005+syuilo@users.noreply.github.com> --- CHANGELOG.md | 5 ++ locales/index.d.ts | 58 +++++++++++++++ locales/ja-JP.yml | 15 ++++ .../1710512074000-url-preview-meta.js | 42 +++++++++++ packages/backend/package.json | 2 +- .../src/core/entities/MetaEntityService.ts | 1 + packages/backend/src/models/Meta.ts | 38 ++++++++-- .../backend/src/models/json-schema/meta.ts | 4 + .../src/server/api/endpoints/admin/meta.ts | 34 ++++++++- .../server/api/endpoints/admin/update-meta.ts | 41 ++++++++-- .../src/server/web/UrlPreviewService.ts | 67 +++++++++++++---- packages/frontend/src/components/MkLink.vue | 17 +++-- packages/frontend/src/components/MkNote.vue | 7 +- .../src/components/MkNoteDetailed.vue | 5 +- .../frontend/src/components/MkUrlPreview.vue | 12 +-- .../frontend/src/components/global/MkUrl.vue | 3 +- .../src/components/page/page.text.vue | 5 +- packages/frontend/src/instance.ts | 2 + .../frontend/src/pages/admin/security.vue | 16 ---- .../frontend/src/pages/admin/settings.vue | 74 ++++++++++++++++++- packages/misskey-js/src/autogen/types.ts | 20 ++++- pnpm-lock.yaml | 18 ++++- 22 files changed, 420 insertions(+), 66 deletions(-) create mode 100644 packages/backend/migration/1710512074000-url-preview-meta.js diff --git a/CHANGELOG.md b/CHANGELOG.md index 0dce1a049..188e146cd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,10 @@ ## Unreleased +### Note +- コントロールパネル内にあるサマリープロキシの設定個所がセキュリティから全般へ変更となります。 + ### General +- Enhance: URLプレビューの有効化・無効化を設定できるように #13569 - Enhance: アンテナでBotによるノートを除外できるように (Cherry-picked from https://github.com/MisskeyIO/misskey/pull/545) - Fix: Play作成時に設定した公開範囲が機能していない問題を修正 @@ -25,6 +29,7 @@ ### Server - Enhance: エンドポイント`antennas/update`の必須項目を`antennaId`のみに +- Enhance: misskey-dev/summaly@5.1.0の取り込み（プレビュー生成処理の効率化） - Fix: フォローリクエストを作成する際に既存のものは削除するように (Cherry-picked from https://activitypub.software/TransFem-org/Sharkey/-/merge_requests/440) diff --git a/locales/index.d.ts b/locales/index.d.ts index afb4adac6..70586d7a8 100644 --- a/locales/index.d.ts +++ b/locales/index.d.ts @@ -4916,6 +4916,10 @@ export interface Locale extends ILocale { * リトライ */ "gameRetry": string; + /** + * 使用しない場合は空欄にしてください + */ + "notUsePleaseLeaveBlank": string; "_bubbleGame": { /** * 遊び方 @@ -9768,6 +9772,60 @@ export interface Locale extends ILocale { */ "header": string; }; + "_urlPreviewSetting": { + /** + * URLプレビューの設定 + */ + "title": string; + /** + * URLプレビューを有効にする + */ + "enable": string; + /** + * プレビュー取得時のタイムアウト(ms) + */ + "timeout": string; + /** + * プレビュー取得の所要時間がこの値を超えた場合、プレビューは生成されません。 + */ + "timeoutDescription": string; + /** + * Content-Lengthの最大値(byte) + */ + "maximumContentLength": string; + /** + * Content-Lengthがこの値を超えた場合、プレビューは生成されません。 + */ + "maximumContentLengthDescription": string; + /** + * Content-Lengthが取得できた場合のみプレビューを生成 + */ + "requireContentLength": string; + /** + * 相手サーバがContent-Lengthを返さない場合、プレビューは生成されません。 + */ + "requireContentLengthDescription": string; + /** + * User-Agent + */ + "userAgent": string; + /** + * プレビュー取得時に使用されるUser-Agentを設定します。空欄の場合、デフォルトのUser-Agentが使用されます。 + */ + "userAgentDescription": string; + /** + * プレビューを生成するプロキシのエンドポイント + */ + "summaryProxy": string; + /** + * Misskey本体ではなく、サマリープロキシを使用してプレビューを生成します。 + */ + "summaryProxyDescription": string; + /** + * プロキシには下記パラメータがクエリ文字列として連携されます。プロキシ側がこれらをサポートしない場合、設定値は無視されます。 + */ + "summaryProxyDescription2": string; + }; } declare const locales: { [lang: string]: Locale; diff --git a/locales/ja-JP.yml b/locales/ja-JP.yml index a64c83b10..cada6d855 100644 --- a/locales/ja-JP.yml +++ b/locales/ja-JP.yml @@ -1225,6 +1225,7 @@ enableHorizontalSwipe: "スワイプしてタブを切り替える" loading: "読み込み中" surrender: "やめる" gameRetry: "リトライ" +notUsePleaseLeaveBlank: "使用しない場合は空欄にしてください" _bubbleGame: howToPlay: "遊び方" @@ -2602,3 +2603,17 @@ _offlineScreen: title: "オフライン - サーバーに接続できません" header: "サーバーに接続できません" +_urlPreviewSetting: + title: "URLプレビューの設定" + enable: "URLプレビューを有効にする" + timeout: "プレビュー取得時のタイムアウト(ms)" + timeoutDescription: "プレビュー取得の所要時間がこの値を超えた場合、プレビューは生成されません。" + maximumContentLength: "Content-Lengthの最大値(byte)" + maximumContentLengthDescription: "Content-Lengthがこの値を超えた場合、プレビューは生成されません。" + requireContentLength: "Content-Lengthが取得できた場合のみプレビューを生成" + requireContentLengthDescription: "相手サーバがContent-Lengthを返さない場合、プレビューは生成されません。" + userAgent: "User-Agent" + userAgentDescription: "プレビュー取得時に使用されるUser-Agentを設定します。空欄の場合、デフォルトのUser-Agentが使用されます。" + summaryProxy: "プレビューを生成するプロキシのエンドポイント" + summaryProxyDescription: "Misskey本体ではなく、サマリープロキシを使用してプレビューを生成します。" + summaryProxyDescription2: "プロキシには下記パラメータがクエリ文字列として連携されます。プロキシ側がこれらをサポートしない場合、設定値は無視されます。" diff --git a/packages/backend/migration/1710512074000-url-preview-meta.js b/packages/backend/migration/1710512074000-url-preview-meta.js new file mode 100644 index 000000000..8af521bbf --- /dev/null +++ b/packages/backend/migration/1710512074000-url-preview-meta.js @@ -0,0 +1,42 @@ +/* + * SPDX-FileCopyrightText: syuilo and misskey-project + * SPDX-License-Identifier: AGPL-3.0-only + */ + +export class UrlPreviewMeta1710512074000 { + name = 'UrlPreviewMeta1710512074000' + + async up(queryRunner) { + await queryRunner.query(` + alter table meta + rename column "summalyProxy" to "urlPreviewSummaryProxyUrl"; + alter table meta + add "urlPreviewEnabled" boolean default true not null; + alter table meta + add "urlPreviewTimeout" integer default 10000 not null; + alter table meta + add "urlPreviewMaximumContentLength" bigint default 10485760 not null; + alter table meta + add "urlPreviewRequireContentLength" boolean default false not null; + alter table meta + add "urlPreviewUserAgent" varchar(1024) default null; + `); + } + + async down(queryRunner) { + await queryRunner.query(` + alter table meta + rename column "urlPreviewSummaryProxyUrl" to "summalyProxy"; + alter table meta + drop column "urlPreviewEnabled"; + alter table meta + drop column "urlPreviewTimeout"; + alter table meta + drop column "urlPreviewMaximumContentLength"; + alter table meta + drop column "urlPreviewRequireContentLength"; + alter table meta + drop column "urlPreviewUserAgent"; + `); + } +} diff --git a/packages/backend/package.json b/packages/backend/package.json index eaad96d5f..d64fcc3d2 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -80,7 +80,7 @@ "@fastify/static": "6.12.0", "@fastify/view": "8.2.0", "@misskey-dev/sharp-read-bmp": "1.2.0", - "@misskey-dev/summaly": "5.0.3", + "@misskey-dev/summaly": "5.1.0", "@nestjs/common": "10.3.3", "@nestjs/core": "10.3.3", "@nestjs/testing": "10.3.3", diff --git a/packages/backend/src/core/entities/MetaEntityService.ts b/packages/backend/src/core/entities/MetaEntityService.ts index b50d76288..9d054ab6a 100644 --- a/packages/backend/src/core/entities/MetaEntityService.ts +++ b/packages/backend/src/core/entities/MetaEntityService.ts @@ -111,6 +111,7 @@ export class MetaEntityService { policies: { ...DEFAULT_POLICIES, ...instance.policies }, mediaProxy: this.config.mediaProxy, + enableUrlPreview: instance.urlPreviewEnabled, }; return packed; diff --git a/packages/backend/src/models/Meta.ts b/packages/backend/src/models/Meta.ts index 66f19ce19..04a34bbbb 100644 --- a/packages/backend/src/models/Meta.ts +++ b/packages/backend/src/models/Meta.ts @@ -277,12 +277,6 @@ export class MiMeta { }) public enableSensitiveMediaDetectionForVideos: boolean; - @Column('varchar', { - length: 1024, - nullable: true, - }) - public summalyProxy: string | null; - @Column('boolean', { default: false, }) @@ -588,4 +582,36 @@ export class MiMeta { default: 0, }) public notesPerOneAd: number; + + @Column('boolean', { + default: true, + }) + public urlPreviewEnabled: boolean; + + @Column('integer', { + default: 10000, + }) + public urlPreviewTimeout: number; + + @Column('bigint', { + default: 1024 * 1024 * 10, + }) + public urlPreviewMaximumContentLength: number; + + @Column('boolean', { + default: true, + }) + public urlPreviewRequireContentLength: boolean; + + @Column('varchar', { + length: 1024, + nullable: true, + }) + public urlPreviewSummaryProxyUrl: string | null; + + @Column('varchar', { + length: 1024, + nullable: true, + }) + public urlPreviewUserAgent: string | null; } diff --git a/packages/backend/src/models/json-schema/meta.ts b/packages/backend/src/models/json-schema/meta.ts index 17789f3b4..473339a1a 100644 --- a/packages/backend/src/models/json-schema/meta.ts +++ b/packages/backend/src/models/json-schema/meta.ts @@ -207,6 +207,10 @@ export const packedMetaLiteSchema = { type: 'string', optional: false, nullable: false, }, + enableUrlPreview: { + type: 'boolean', + optional: false, nullable: false, + }, backgroundImageUrl: { type: 'string', optional: false, nullable: true, diff --git a/packages/backend/src/server/api/endpoints/admin/meta.ts b/packages/backend/src/server/api/endpoints/admin/meta.ts index 88c5907bc..f4ff57327 100644 --- a/packages/backend/src/server/api/endpoints/admin/meta.ts +++ b/packages/backend/src/server/api/endpoints/admin/meta.ts @@ -434,6 +434,8 @@ export const meta = { summalyProxy: { type: 'string', optional: false, nullable: true, + deprecated: true, + description: '[Deprecated] Use "urlPreviewSummaryProxyUrl" instead.', }, themeColor: { type: 'string', @@ -451,6 +453,30 @@ export const meta = { type: 'string', optional: false, nullable: false, }, + urlPreviewEnabled: { + type: 'boolean', + optional: false, nullable: false, + }, + urlPreviewTimeout: { + type: 'number', + optional: false, nullable: false, + }, + urlPreviewMaximumContentLength: { + type: 'number', + optional: false, nullable: false, + }, + urlPreviewRequireContentLength: { + type: 'boolean', + optional: false, nullable: false, + }, + urlPreviewUserAgent: { + type: 'string', + optional: false, nullable: true, + }, + urlPreviewSummaryProxyUrl: { + type: 'string', + optional: false, nullable: true, + }, }, }, } as const; @@ -533,7 +559,6 @@ export default class extends Endpoint { // eslint- setSensitiveFlagAutomatically: instance.setSensitiveFlagAutomatically, enableSensitiveMediaDetectionForVideos: instance.enableSensitiveMediaDetectionForVideos, proxyAccountId: instance.proxyAccountId, - summalyProxy: instance.summalyProxy, email: instance.email, smtpSecure: instance.smtpSecure, smtpHost: instance.smtpHost, @@ -577,6 +602,13 @@ export default class extends Endpoint { // eslint- perUserHomeTimelineCacheMax: instance.perUserHomeTimelineCacheMax, perUserListTimelineCacheMax: instance.perUserListTimelineCacheMax, notesPerOneAd: instance.notesPerOneAd, + summalyProxy: instance.urlPreviewSummaryProxyUrl, + urlPreviewEnabled: instance.urlPreviewEnabled, + urlPreviewTimeout: instance.urlPreviewTimeout, + urlPreviewMaximumContentLength: instance.urlPreviewMaximumContentLength, + urlPreviewRequireContentLength: instance.urlPreviewRequireContentLength, + urlPreviewUserAgent: instance.urlPreviewUserAgent, + urlPreviewSummaryProxyUrl: instance.urlPreviewSummaryProxyUrl, }; }); } diff --git a/packages/backend/src/server/api/endpoints/admin/update-meta.ts b/packages/backend/src/server/api/endpoints/admin/update-meta.ts index bffceef81..2f62d30ad 100644 --- a/packages/backend/src/server/api/endpoints/admin/update-meta.ts +++ b/packages/backend/src/server/api/endpoints/admin/update-meta.ts @@ -90,7 +90,6 @@ export const paramDef = { type: 'string', }, }, - summalyProxy: { type: 'string', nullable: true }, deeplAuthKey: { type: 'string', nullable: true }, deeplIsPro: { type: 'boolean' }, enableEmail: { type: 'boolean' }, @@ -150,6 +149,16 @@ export const paramDef = { type: 'string', }, }, + summalyProxy: { + type: 'string', nullable: true, + description: '[Deprecated] Use "urlPreviewSummaryProxyUrl" instead.', + }, + urlPreviewEnabled: { type: 'boolean' }, + urlPreviewTimeout: { type: 'integer' }, + urlPreviewMaximumContentLength: { type: 'integer' }, + urlPreviewRequireContentLength: { type: 'boolean' }, + urlPreviewUserAgent: { type: 'string', nullable: true }, + urlPreviewSummaryProxyUrl: { type: 'string', nullable: true }, }, required: [], } as const; @@ -353,10 +362,6 @@ export default class extends Endpoint { // eslint- set.langs = ps.langs.filter(Boolean); } - if (ps.summalyProxy !== undefined) { - set.summalyProxy = ps.summalyProxy; - } - if (ps.enableEmail !== undefined) { set.enableEmail = ps.enableEmail; } @@ -581,6 +586,32 @@ export default class extends Endpoint { // eslint- set.bannedEmailDomains = ps.bannedEmailDomains; } + if (ps.urlPreviewEnabled !== undefined) { + set.urlPreviewEnabled = ps.urlPreviewEnabled; + } + + if (ps.urlPreviewTimeout !== undefined) { + set.urlPreviewTimeout = ps.urlPreviewTimeout; + } + + if (ps.urlPreviewMaximumContentLength !== undefined) { + set.urlPreviewMaximumContentLength = ps.urlPreviewMaximumContentLength; + } + + if (ps.urlPreviewRequireContentLength !== undefined) { + set.urlPreviewRequireContentLength = ps.urlPreviewRequireContentLength; + } + + if (ps.urlPreviewUserAgent !== undefined) { + const value = (ps.urlPreviewUserAgent ?? '').trim(); + set.urlPreviewUserAgent = value === '' ? null : ps.urlPreviewUserAgent; + } + + if (ps.summalyProxy !== undefined || ps.urlPreviewSummaryProxyUrl !== undefined) { + const value = ((ps.urlPreviewSummaryProxyUrl ?? ps.summalyProxy) ?? '').trim(); + set.urlPreviewSummaryProxyUrl = value === '' ? null : value; + } + const before = await this.metaService.fetch(true); await this.metaService.update(set); diff --git a/packages/backend/src/server/web/UrlPreviewService.ts b/packages/backend/src/server/web/UrlPreviewService.ts index c6a96e94c..8f8f08a30 100644 --- a/packages/backend/src/server/web/UrlPreviewService.ts +++ b/packages/backend/src/server/web/UrlPreviewService.ts @@ -5,6 +5,7 @@ import { Inject, Injectable } from '@nestjs/common'; import { summaly } from '@misskey-dev/summaly'; +import { SummalyResult } from '@misskey-dev/summaly/built/summary.js'; import { DI } from '@/di-symbols.js'; import type { Config } from '@/config.js'; import { MetaService } from '@/core/MetaService.js'; @@ -14,6 +15,7 @@ import { query } from '@/misc/prelude/url.js'; import { LoggerService } from '@/core/LoggerService.js'; import { bindThis } from '@/decorators.js'; import { ApiError } from '@/server/api/error.js'; +import { MiMeta } from '@/models/Meta.js'; import type { FastifyRequest, FastifyReply } from 'fastify'; @Injectable() @@ -62,24 +64,25 @@ export class UrlPreviewService { const meta = await this.metaService.fetch(); - this.logger.info(meta.summalyProxy + if (!meta.urlPreviewEnabled) { + reply.code(403); + return { + error: new ApiError({ + message: 'URL preview is disabled', + code: 'URL_PREVIEW_DISABLED', + id: '58b36e13-d2f5-0323-b0c6-76aa9dabefb8', + }), + }; + } + + this.logger.info(meta.urlPreviewSummaryProxyUrl ? `(Proxy) Getting preview of ${url}@${lang} ...` : `Getting preview of ${url}@${lang} ...`); + try { - const summary = meta.summalyProxy ? - await this.httpRequestService.getJson>(`${meta.summalyProxy}?${query({ - url: url, - lang: lang ?? 'ja-JP', - })}`) - : - await summaly(url, { - followRedirects: false, - lang: lang ?? 'ja-JP', - agent: this.config.proxy ? { - http: this.httpRequestService.httpAgent, - https: this.httpRequestService.httpsAgent, - } : undefined, - }); + const summary = meta.urlPreviewSummaryProxyUrl + ? await this.fetchSummaryFromProxy(url, meta, lang) + : await this.fetchSummary(url, meta, lang); this.logger.succ(`Got preview of ${url}: ${summary.title}`); @@ -100,6 +103,7 @@ export class UrlPreviewService { return summary; } catch (err) { this.logger.warn(`Failed to get preview of ${url}: ${err}`); + reply.code(422); reply.header('Cache-Control', 'max-age=86400, immutable'); return { @@ -111,4 +115,37 @@ export class UrlPreviewService { }; } } + + private fetchSummary(url: string, meta: MiMeta, lang?: string): Promise { + const agent = this.config.proxy + ? { + http: this.httpRequestService.httpAgent, + https: this.httpRequestService.httpsAgent, + } + : undefined; + + return summaly(url, { + followRedirects: false, + lang: lang ?? 'ja-JP', + agent: agent, + userAgent: meta.urlPreviewUserAgent ?? undefined, + operationTimeout: meta.urlPreviewTimeout, + contentLengthLimit: meta.urlPreviewMaximumContentLength, + contentLengthRequired: meta.urlPreviewRequireContentLength, + }); + } + + private fetchSummaryFromProxy(url: string, meta: MiMeta, lang?: string): Promise { + const proxy = meta.urlPreviewSummaryProxyUrl!; + const queryStr = query({ + url: url, + lang: lang ?? 'ja-JP', + userAgent: meta.urlPreviewUserAgent ?? undefined, + operationTimeout: meta.urlPreviewTimeout, + contentLengthLimit: meta.urlPreviewMaximumContentLength, + contentLengthRequired: meta.urlPreviewRequireContentLength, + }); + + return this.httpRequestService.getJson(`${proxy}?${queryStr}`); + } } diff --git a/packages/frontend/src/components/MkLink.vue b/packages/frontend/src/components/MkLink.vue index 3f7aba2fe..ca875242b 100644 --- a/packages/frontend/src/components/MkLink.vue +++ b/packages/frontend/src/components/MkLink.vue @@ -18,6 +18,7 @@ import { defineAsyncComponent, ref } from 'vue'; import { url as local } from '@/config.js'; import { useTooltip } from '@/scripts/use-tooltip.js'; import * as os from '@/os.js'; +import { isEnabledUrlPreview } from '@/instance.js'; const props = withDefaults(defineProps<{ url: string; @@ -31,13 +32,15 @@ const target = self ? null : '_blank'; const el = ref(); -useTooltip(el, (showing) => { - os.popup(defineAsyncComponent(() => import('@/components/MkUrlPreviewPopup.vue')), { - showing, - url: props.url, - source: el.value instanceof HTMLElement ? el.value : el.value?.$el, - }, {}, 'closed'); -}); +if (isEnabledUrlPreview.value) { + useTooltip(el, (showing) => { + os.popup(defineAsyncComponent(() => import('@/components/MkUrlPreviewPopup.vue')), { + showing, + url: props.url, + source: el.value instanceof HTMLElement ? el.value : el.value?.$el, + }, {}, 'closed'); + }); +} diff --git a/packages/misskey-js/src/autogen/types.ts b/packages/misskey-js/src/autogen/types.ts index 636bc62aa..c6e36d80a 100644 --- a/packages/misskey-js/src/autogen/types.ts +++ b/packages/misskey-js/src/autogen/types.ts @@ -4810,6 +4810,7 @@ export type components = { enableServiceWorker: boolean; translatorAvailable: boolean; mediaProxy: string; + enableUrlPreview: boolean; backgroundImageUrl: string | null; impressumUrl: string | null; logoImageUrl: string | null; @@ -4962,11 +4963,21 @@ export type operations = { objectStorageS3ForcePathStyle: boolean; privacyPolicyUrl: string | null; repositoryUrl: string | null; + /** + * @deprecated + * @description [Deprecated] Use "urlPreviewSummaryProxyUrl" instead. + */ summalyProxy: string | null; themeColor: string | null; tosUrl: string | null; uri: string; version: string; + urlPreviewEnabled: boolean; + urlPreviewTimeout: number; + urlPreviewMaximumContentLength: number; + urlPreviewRequireContentLength: boolean; + urlPreviewUserAgent: string | null; + urlPreviewSummaryProxyUrl: string | null; }; }; }; @@ -8862,7 +8873,6 @@ export type operations = { maintainerName?: string | null; maintainerEmail?: string | null; langs?: string[]; - summalyProxy?: string | null; deeplAuthKey?: string | null; deeplIsPro?: boolean; enableEmail?: boolean; @@ -8916,6 +8926,14 @@ export type operations = { perUserListTimelineCacheMax?: number; notesPerOneAd?: number; silencedHosts?: string[] | null; + /** @description [Deprecated] Use "urlPreviewSummaryProxyUrl" instead. */ + summalyProxy?: string | null; + urlPreviewEnabled?: boolean; + urlPreviewTimeout?: number; + urlPreviewMaximumContentLength?: number; + urlPreviewRequireContentLength?: boolean; + urlPreviewUserAgent?: string | null; + urlPreviewSummaryProxyUrl?: string | null; }; }; }; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 4c1e228a9..383b31b1f 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -117,8 +117,8 @@ importers: specifier: 1.2.0 version: 1.2.0 '@misskey-dev/summaly': - specifier: 5.0.3 - version: 5.0.3 + specifier: 5.1.0 + version: 5.1.0 '@nestjs/common': specifier: 10.3.3 version: 10.3.3(reflect-metadata@0.2.1)(rxjs@7.8.1) @@ -4772,6 +4772,20 @@ packages: jschardet: 3.0.0 private-ip: 2.3.3 trace-redirect: 1.0.6 + dev: true + + /@misskey-dev/summaly@5.1.0: + resolution: {integrity: sha512-WAUrgX3/z4h4aI8Y/WVwmJcJ6Fa1Zf2LJCSS651t9MHoWVGABLsQ2KCXRGmlpk4i+cMDNIwweObUroosE7j8rg==} + dependencies: + cheerio: 1.0.0-rc.12 + escape-regexp: 0.0.1 + got: 12.6.1 + html-entities: 2.3.2 + iconv-lite: 0.6.3 + jschardet: 3.0.0 + private-ip: 2.3.3 + trace-redirect: 1.0.6 + dev: false /@mole-inc/bin-wrapper@8.0.1: resolution: {integrity: sha512-sTGoeZnjI8N4KS+sW2AN95gDBErhAguvkw/tWdCjeM8bvxpz5lqrnd0vOJABA1A+Ic3zED7PYoLP/RANLgVotA==} From 722f01c4e70cc512283791f841fedd2673dc2ac8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E3=81=BE=E3=81=A3=E3=81=A1=E3=82=83=E3=81=A8=E3=83=BC?= =?UTF-8?q?=E3=81=AB=E3=82=85?= <17376330+u1-liquid@users.noreply.github.com> Date: Fri, 22 Mar 2024 07:43:41 +0900 Subject: [PATCH 6/9] =?UTF-8?q?fix(frontend):=20=E7=AE=A1=E7=90=86?= =?UTF-8?q?=E7=94=A8=E3=81=AE=E3=83=A6=E3=83=BC=E3=82=B6=E3=83=BC=E8=A9=B3?= =?UTF-8?q?=E7=B4=B0=E3=83=9A=E3=83=BC=E3=82=B8=E3=81=AE=E3=82=B3=E3=83=94?= =?UTF-8?q?=E3=83=BC=E3=83=9C=E3=82=BF=E3=83=B3=E3=81=AE=E3=83=87=E3=83=BC?= =?UTF-8?q?=E3=82=BF=E3=81=8C=E6=9C=9F=E5=BE=85=E3=81=A8=E9=81=95=E3=81=86?= =?UTF-8?q?=E5=95=8F=E9=A1=8C=E3=82=92=E4=BF=AE=E6=AD=A3=20(MisskeyIO#559)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/frontend/src/pages/admin-user.vue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/frontend/src/pages/admin-user.vue b/packages/frontend/src/pages/admin-user.vue index 403c6619b..59dc0ba0e 100644 --- a/packages/frontend/src/pages/admin-user.vue +++ b/packages/frontend/src/pages/admin-user.vue @@ -46,7 +46,7 @@ SPDX-License-Identifier: AGPL-3.0-only - + From 01ec286f3f479d030bd4a59ab666b513b1175bca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E3=81=BE=E3=81=A3=E3=81=A1=E3=82=83=E3=81=A8=E3=83=BC?= =?UTF-8?q?=E3=81=AB=E3=82=85?= <17376330+u1-liquid@users.noreply.github.com> Date: Fri, 22 Mar 2024 07:43:59 +0900 Subject: [PATCH 7/9] =?UTF-8?q?enhance(frontend):=20=E5=A4=96=E9=83=A8?= =?UTF-8?q?=E3=82=B5=E3=82=A4=E3=83=88=E3=81=B8=E3=81=AE=E3=83=AA=E3=83=B3?= =?UTF-8?q?=E3=82=AF=E3=81=AF=E7=A7=BB=E5=8B=95=E3=81=AE=E5=89=8D=E3=81=AB?= =?UTF-8?q?=E8=AD=A6=E5=91=8A=E3=82=92=E8=A1=A8=E7=A4=BA=E3=81=99=E3=82=8B?= =?UTF-8?q?=E3=82=88=E3=81=86=E3=81=AB=20(MisskeyIO#558)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- locales/en-US.yml | 4 +++ locales/index.d.ts | 19 +++++++++++ locales/ja-JP.yml | 4 +++ locales/ko-KR.yml | 4 +++ .../1711008460816-external-website-warn.js | 11 +++++++ .../src/core/entities/MetaEntityService.ts | 1 + packages/backend/src/models/Meta.ts | 5 +++ .../backend/src/models/json-schema/meta.ts | 8 +++++ .../src/server/api/endpoints/admin/meta.ts | 11 ++++++- .../server/api/endpoints/admin/update-meta.ts | 9 +++++ packages/frontend/src/components/MkLink.vue | 10 +++++- .../frontend/src/components/global/MkUrl.vue | 10 +++++- .../frontend/src/pages/admin/moderation.vue | 14 ++++++-- .../src/scripts/warning-external-website.ts | 33 +++++++++++++++++++ packages/misskey-js/src/autogen/types.ts | 5 ++- 15 files changed, 141 insertions(+), 7 deletions(-) create mode 100644 packages/backend/migration/1711008460816-external-website-warn.js create mode 100644 packages/frontend/src/scripts/warning-external-website.ts diff --git a/locales/en-US.yml b/locales/en-US.yml index 1fb581d8d..527beb471 100644 --- a/locales/en-US.yml +++ b/locales/en-US.yml @@ -1212,6 +1212,10 @@ useGroupedNotifications: "Display grouped notifications" signupPendingError: "There was a problem verifying the email address. The link may have expired." cwNotationRequired: "If \"Hide content\" is enabled, a description must be provided." doReaction: "Add reaction" +wellKnownWebsites: "Well-known websites" +wellKnownWebsitesDescription: "Separate with spaces for AND, new lines for OR. Surround with slashes for regular expressions. Matching will allow redirection to external sites without a warning." +warningRedirectingExternalWebsiteTitle: "You are leaving our site!" +warningRedirectingExternalWebsiteDescription: "You are about to jump to another site.\nPlease make sure this link is reliable before proceeding.\n\n{url}" code: "Code" reloadRequiredToApplySettings: "Reloading is required to apply the settings." remainingN: "Remaining: {n}" diff --git a/locales/index.d.ts b/locales/index.d.ts index 44b24d83c..bcca401da 100644 --- a/locales/index.d.ts +++ b/locales/index.d.ts @@ -4860,6 +4860,25 @@ export interface Locale extends ILocale { * リアクションする */ "doReaction": string; + /** + * よく知られたウェブサイト + */ + "wellKnownWebsites": string; + /** + * スペースで区切るとAND指定になり、改行で区切るとOR指定になります。スラッシュで囲むと正規表現になります。一致した場合、外部サイトへのリダイレクトの警告を省略させることができます。 + */ + "wellKnownWebsitesDescription": string; + /** + * 外部サイトへ移動します + */ + "warningRedirectingExternalWebsiteTitle": string; + /** + * 別のサイトにジャンプしようとしています。 + * リンク先の安全性を十分に確認した上で進んでください。 + * + * {url} + */ + "warningRedirectingExternalWebsiteDescription": ParameterizedString<"url">; /** * サムネイルの表示を制限するURL */ diff --git a/locales/ja-JP.yml b/locales/ja-JP.yml index fd84e0318..d30ded3a5 100644 --- a/locales/ja-JP.yml +++ b/locales/ja-JP.yml @@ -1211,6 +1211,10 @@ useGroupedNotifications: "通知をグルーピングして表示する" signupPendingError: "メールアドレスの確認中に問題が発生しました。リンクの有効期限が切れている可能性があります。" cwNotationRequired: "「内容を隠す」がオンの場合は注釈の記述が必要です。" doReaction: "リアクションする" +wellKnownWebsites: "よく知られたウェブサイト" +wellKnownWebsitesDescription: "スペースで区切るとAND指定になり、改行で区切るとOR指定になります。スラッシュで囲むと正規表現になります。一致した場合、外部サイトへのリダイレクトの警告を省略させることができます。" +warningRedirectingExternalWebsiteTitle: "外部サイトへ移動します" +warningRedirectingExternalWebsiteDescription: "別のサイトにジャンプしようとしています。\nリンク先の安全性を十分に確認した上で進んでください。\n\n{url}" urlPreviewDenyList: "サムネイルの表示を制限するURL" urlPreviewDenyListDescription: "スペースで区切るとAND指定になり、改行で区切るとOR指定になります。スラッシュで囲むと正規表現になります。一致した場合、サムネイルがぼかされて表示されます。" code: "コード" diff --git a/locales/ko-KR.yml b/locales/ko-KR.yml index 5b3d5538a..e105a0df1 100644 --- a/locales/ko-KR.yml +++ b/locales/ko-KR.yml @@ -1209,6 +1209,10 @@ useGroupedNotifications: "알림을 그룹화하고 표시" signupPendingError: "메일 주소 확인중에 문제가 발생했습니다. 링크의 유효기간이 지났을 가능성이 있습니다." cwNotationRequired: "'내용을 숨기기'를 체크한 경우 주석을 써야 합니다." doReaction: "리액션 추가" +wellKnownWebsites: "잘 알려진 웹사이트" +wellKnownWebsitesDescription: "공백으로 구분하면 AND 지정이 되며, 개행으로 구분하면 OR 지정이 됩니다. 슬래시로 둘러싸면 정규 표현식이 됩니다. 일치하는 경우, 외부 사이트로의 리다이렉트 경고를 생략할 수 있습니다." +warningRedirectingExternalWebsiteTitle: "외부 사이트로 이동합니다" +warningRedirectingExternalWebsiteDescription: "다른 사이트로 이동하려고 합니다.\n링크가 안전한지 충분히 확인한 후 이동해주세요.\n\n{url}" code: "문자열" reloadRequiredToApplySettings: "설정을 적용하려면 새로고침을 해야 합니다." remainingN: "나머지: {n}" diff --git a/packages/backend/migration/1711008460816-external-website-warn.js b/packages/backend/migration/1711008460816-external-website-warn.js new file mode 100644 index 000000000..6e6ae75ef --- /dev/null +++ b/packages/backend/migration/1711008460816-external-website-warn.js @@ -0,0 +1,11 @@ +export class ExternalWebsiteWarn1711008460816 { + name = 'ExternalWebsiteWarn1711008460816' + + async up(queryRunner) { + await queryRunner.query(`ALTER TABLE "meta" ADD "wellKnownWebsites" character varying(3072) array NOT NULL DEFAULT '{}'`); + } + + async down(queryRunner) { + await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "wellKnownWebsites"`); + } +} diff --git a/packages/backend/src/core/entities/MetaEntityService.ts b/packages/backend/src/core/entities/MetaEntityService.ts index b21185183..ad35a440d 100644 --- a/packages/backend/src/core/entities/MetaEntityService.ts +++ b/packages/backend/src/core/entities/MetaEntityService.ts @@ -99,6 +99,7 @@ export class MetaEntityService { imageUrl: ad.imageUrl, dayOfWeek: ad.dayOfWeek, })), + wellKnownWebsites: instance.wellKnownWebsites, notesPerOneAd: instance.notesPerOneAd, enableEmail: instance.enableEmail, enableServiceWorker: instance.enableServiceWorker, diff --git a/packages/backend/src/models/Meta.ts b/packages/backend/src/models/Meta.ts index 3255117d8..08451cff3 100644 --- a/packages/backend/src/models/Meta.ts +++ b/packages/backend/src/models/Meta.ts @@ -594,6 +594,11 @@ export class MiMeta { }) public notesPerOneAd: number; + @Column('varchar', { + length: 3072, array: true, default: '{}', + }) + public wellKnownWebsites: string[]; + @Column('varchar', { length: 3072, array: true, default: '{}', }) diff --git a/packages/backend/src/models/json-schema/meta.ts b/packages/backend/src/models/json-schema/meta.ts index 150b94a18..dd440b4b1 100644 --- a/packages/backend/src/models/json-schema/meta.ts +++ b/packages/backend/src/models/json-schema/meta.ts @@ -183,6 +183,14 @@ export const packedMetaLiteSchema = { }, }, }, + wellKnownWebsites: { + type: 'array', + optional: false, nullable: false, + items: { + type: 'string', + optional: false, nullable: false, + }, + }, notesPerOneAd: { type: 'number', optional: false, nullable: false, diff --git a/packages/backend/src/server/api/endpoints/admin/meta.ts b/packages/backend/src/server/api/endpoints/admin/meta.ts index 107608d79..2d533b255 100644 --- a/packages/backend/src/server/api/endpoints/admin/meta.ts +++ b/packages/backend/src/server/api/endpoints/admin/meta.ts @@ -381,9 +381,17 @@ export const meta = { type: 'number', optional: false, nullable: false, }, + wellKnownWebsites: { + type: 'array', + optional: false, nullable: false, + items: { + type: 'string', + optional: false, nullable: false, + }, + }, urlPreviewDenyList: { type: 'array', - optional: true, nullable: false, + optional: false, nullable: false, items: { type: 'string', optional: false, nullable: false, @@ -602,6 +610,7 @@ export default class extends Endpoint { // eslint- perRemoteUserUserTimelineCacheMax: instance.perRemoteUserUserTimelineCacheMax, perUserHomeTimelineCacheMax: instance.perUserHomeTimelineCacheMax, perUserListTimelineCacheMax: instance.perUserListTimelineCacheMax, + wellKnownWebsites: instance.wellKnownWebsites, notesPerOneAd: instance.notesPerOneAd, urlPreviewDenyList: instance.urlPreviewDenyList, featuredGameChannels: instance.featuredGameChannels, diff --git a/packages/backend/src/server/api/endpoints/admin/update-meta.ts b/packages/backend/src/server/api/endpoints/admin/update-meta.ts index 439116a1a..baad5eff7 100644 --- a/packages/backend/src/server/api/endpoints/admin/update-meta.ts +++ b/packages/backend/src/server/api/endpoints/admin/update-meta.ts @@ -155,6 +155,11 @@ export const paramDef = { type: 'string', }, }, + wellKnownWebsites: { + type: 'array', nullable: true, items: { + type: 'string', + }, + }, urlPreviewDenyList: { type: 'array', nullable: true, items: { type: 'string', @@ -220,6 +225,10 @@ export default class extends Endpoint { // eslint- }).map(x => x.toLowerCase()); } + if (Array.isArray(ps.wellKnownWebsites)) { + set.wellKnownWebsites = ps.wellKnownWebsites.filter(Boolean); + } + if (Array.isArray(ps.urlPreviewDenyList)) { set.urlPreviewDenyList = ps.urlPreviewDenyList.filter(Boolean); } diff --git a/packages/frontend/src/components/MkLink.vue b/packages/frontend/src/components/MkLink.vue index 3f7aba2fe..3ff5fcc75 100644 --- a/packages/frontend/src/components/MkLink.vue +++ b/packages/frontend/src/components/MkLink.vue @@ -5,8 +5,15 @@ SPDX-License-Identifier: AGPL-3.0-only