From 4ecfae0d85bbf35f0a10a83857d6ca2d9161264e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E3=81=9F=E3=83=BC=E3=81=B3=E3=82=93?= Date: Sun, 22 Dec 2024 04:01:53 +0900 Subject: [PATCH] perf(timeline): Optimizing for CDN Caching (MisskeyIO#834) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: あわわわとーにゅ <17376330+u1-liquid@users.noreply.github.com> --- .../src/server/api/stream/channels/antenna.ts | 12 ++- .../src/server/api/stream/channels/channel.ts | 12 ++- .../api/stream/channels/global-timeline.ts | 12 ++- .../api/stream/channels/home-timeline.ts | 12 ++- .../api/stream/channels/hybrid-timeline.ts | 12 ++- .../api/stream/channels/local-timeline.ts | 12 ++- .../api/stream/channels/role-timeline.ts | 12 ++- .../server/api/stream/channels/user-list.ts | 12 ++- .../src/server/web/ClientServerService.ts | 75 +++++++++++++++++++ packages/backend/test/utils.ts | 2 +- .../frontend/src/components/MkTimeline.vue | 43 ++++++++++- packages/misskey-js/etc/misskey-js.api.md | 8 ++ packages/misskey-js/src/streaming.types.ts | 8 ++ 13 files changed, 205 insertions(+), 27 deletions(-) diff --git a/packages/backend/src/server/api/stream/channels/antenna.ts b/packages/backend/src/server/api/stream/channels/antenna.ts index 0ef12ae660..5c16cea607 100644 --- a/packages/backend/src/server/api/stream/channels/antenna.ts +++ b/packages/backend/src/server/api/stream/channels/antenna.ts @@ -15,6 +15,7 @@ class AntennaChannel extends Channel { public static readonly requireCredential = true as const; public static readonly kind = 'read:account'; private antennaId: string; + private idOnly: boolean; constructor( private noteEntityService: NoteEntityService, @@ -29,6 +30,7 @@ class AntennaChannel extends Channel { @bindThis public async init(params: any) { this.antennaId = params.antennaId as string; + this.idOnly = params.idOnly ?? false; // Subscribe stream this.subscriber.on(`antennaStream:${this.antennaId}`, this.onEvent); @@ -49,9 +51,13 @@ class AntennaChannel extends Channel { if (this.isNoteMutedOrBlocked(note)) return; - this.connection.cacheNote(note); - - this.send('note', note); + if (this.idOnly && ['public', 'home'].includes(note.visibility)) { + const idOnlyNote = { id: note.id }; + this.send('note', idOnlyNote); + } else { + this.connection.cacheNote(note); + this.send('note', note); + } } else { this.send(data.type, data.body); } diff --git a/packages/backend/src/server/api/stream/channels/channel.ts b/packages/backend/src/server/api/stream/channels/channel.ts index 98db02fd07..94ffcd5d62 100644 --- a/packages/backend/src/server/api/stream/channels/channel.ts +++ b/packages/backend/src/server/api/stream/channels/channel.ts @@ -15,6 +15,7 @@ class ChannelChannel extends Channel { public static readonly shouldShare = false; public static readonly requireCredential = false as const; private channelId: string; + private idOnly: boolean; constructor( private noteEntityService: NoteEntityService, @@ -29,6 +30,7 @@ class ChannelChannel extends Channel { @bindThis public async init(params: any) { this.channelId = params.channelId as string; + this.idOnly = params.idOnly ?? false; // Subscribe stream this.subscriber.on('notesStream', this.onNote); @@ -55,9 +57,13 @@ class ChannelChannel extends Channel { } } - this.connection.cacheNote(note); - - this.send('note', note); + if (this.idOnly && ['public', 'home'].includes(note.visibility)) { + const idOnlyNote = { id: note.id }; + this.send('note', idOnlyNote); + } else { + this.connection.cacheNote(note); + this.send('note', note); + } } @bindThis diff --git a/packages/backend/src/server/api/stream/channels/global-timeline.ts b/packages/backend/src/server/api/stream/channels/global-timeline.ts index fee5972e85..64f1941603 100644 --- a/packages/backend/src/server/api/stream/channels/global-timeline.ts +++ b/packages/backend/src/server/api/stream/channels/global-timeline.ts @@ -18,6 +18,7 @@ class GlobalTimelineChannel extends Channel { public static readonly requireCredential = false as const; private withRenotes: boolean; private withFiles: boolean; + private idOnly: boolean; constructor( private metaService: MetaService, @@ -38,6 +39,7 @@ class GlobalTimelineChannel extends Channel { this.withRenotes = params.withRenotes ?? true; this.withFiles = params.withFiles ?? false; + this.idOnly = params.idOnly ?? false; // Subscribe events this.subscriber.on('notesStream', this.onNote); @@ -85,9 +87,13 @@ class GlobalTimelineChannel extends Channel { } } - this.connection.cacheNote(note); - - this.send('note', note); + if (this.idOnly && ['public', 'home'].includes(note.visibility)) { + const idOnlyNote = { id: note.id }; + this.send('note', idOnlyNote); + } else { + this.connection.cacheNote(note); + this.send('note', note); + } } @bindThis diff --git a/packages/backend/src/server/api/stream/channels/home-timeline.ts b/packages/backend/src/server/api/stream/channels/home-timeline.ts index 68822aa5d4..3ee1935227 100644 --- a/packages/backend/src/server/api/stream/channels/home-timeline.ts +++ b/packages/backend/src/server/api/stream/channels/home-timeline.ts @@ -17,6 +17,7 @@ class HomeTimelineChannel extends Channel { public static readonly kind = 'read:account'; private withRenotes: boolean; private withFiles: boolean; + private idOnly: boolean; constructor( private noteEntityService: NoteEntityService, @@ -32,6 +33,7 @@ class HomeTimelineChannel extends Channel { public async init(params: any) { this.withRenotes = params.withRenotes ?? true; this.withFiles = params.withFiles ?? false; + this.idOnly = params.idOnly ?? false; this.subscriber.on('notesStream', this.onNote); } @@ -89,9 +91,13 @@ class HomeTimelineChannel extends Channel { } } - this.connection.cacheNote(note); - - this.send('note', note); + if (this.idOnly && ['public', 'home'].includes(note.visibility)) { + const idOnlyNote = { id: note.id }; + this.send('note', idOnlyNote); + } else { + this.connection.cacheNote(note); + this.send('note', note); + } } @bindThis diff --git a/packages/backend/src/server/api/stream/channels/hybrid-timeline.ts b/packages/backend/src/server/api/stream/channels/hybrid-timeline.ts index 39b7b8b641..20337f85b9 100644 --- a/packages/backend/src/server/api/stream/channels/hybrid-timeline.ts +++ b/packages/backend/src/server/api/stream/channels/hybrid-timeline.ts @@ -20,6 +20,7 @@ class HybridTimelineChannel extends Channel { private withRenotes: boolean; private withReplies: boolean; private withFiles: boolean; + private idOnly: boolean; constructor( private metaService: MetaService, @@ -41,6 +42,7 @@ class HybridTimelineChannel extends Channel { this.withRenotes = params.withRenotes ?? true; this.withReplies = params.withReplies ?? false; this.withFiles = params.withFiles ?? false; + this.idOnly = params.idOnly ?? false; // Subscribe events this.subscriber.on('notesStream', this.onNote); @@ -103,9 +105,13 @@ class HybridTimelineChannel extends Channel { } } - this.connection.cacheNote(note); - - this.send('note', note); + if (this.idOnly && ['public', 'home'].includes(note.visibility)) { + const idOnlyNote = { id: note.id }; + this.send('note', idOnlyNote); + } else { + this.connection.cacheNote(note); + this.send('note', note); + } } @bindThis diff --git a/packages/backend/src/server/api/stream/channels/local-timeline.ts b/packages/backend/src/server/api/stream/channels/local-timeline.ts index c191be84ee..441b0132a5 100644 --- a/packages/backend/src/server/api/stream/channels/local-timeline.ts +++ b/packages/backend/src/server/api/stream/channels/local-timeline.ts @@ -19,6 +19,7 @@ class LocalTimelineChannel extends Channel { private withRenotes: boolean; private withReplies: boolean; private withFiles: boolean; + private idOnly: boolean; constructor( private metaService: MetaService, @@ -40,6 +41,7 @@ class LocalTimelineChannel extends Channel { this.withRenotes = params.withRenotes ?? true; this.withReplies = params.withReplies ?? false; this.withFiles = params.withFiles ?? false; + this.idOnly = params.idOnly ?? false; // Subscribe events this.subscriber.on('notesStream', this.onNote); @@ -88,9 +90,13 @@ class LocalTimelineChannel extends Channel { } } - this.connection.cacheNote(note); - - this.send('note', note); + if (this.idOnly && ['public', 'home'].includes(note.visibility)) { + const idOnlyNote = { id: note.id }; + this.send('note', idOnlyNote); + } else { + this.connection.cacheNote(note); + this.send('note', note); + } } @bindThis diff --git a/packages/backend/src/server/api/stream/channels/role-timeline.ts b/packages/backend/src/server/api/stream/channels/role-timeline.ts index 6fabdcb75a..3fb8885717 100644 --- a/packages/backend/src/server/api/stream/channels/role-timeline.ts +++ b/packages/backend/src/server/api/stream/channels/role-timeline.ts @@ -16,6 +16,7 @@ class RoleTimelineChannel extends Channel { public static readonly shouldShare = false; public static readonly requireCredential = false as const; private roleId: string; + private idOnly: boolean; constructor( private noteEntityService: NoteEntityService, @@ -31,6 +32,7 @@ class RoleTimelineChannel extends Channel { @bindThis public async init(params: any) { this.roleId = params.roleId as string; + this.idOnly = params.idOnly ?? false; this.subscriber.on(`roleTimelineStream:${this.roleId}`, this.onEvent); } @@ -71,9 +73,13 @@ class RoleTimelineChannel extends Channel { } } - this.connection.cacheNote(note); - - this.send('note', note); + if (this.idOnly && ['public', 'home'].includes(note.visibility)) { + const idOnlyNote = { id: note.id }; + this.send('note', idOnlyNote); + } else { + this.connection.cacheNote(note); + this.send('note', note); + } } else { this.send(data.type, data.body); } diff --git a/packages/backend/src/server/api/stream/channels/user-list.ts b/packages/backend/src/server/api/stream/channels/user-list.ts index 297c253c4a..651ab05f34 100644 --- a/packages/backend/src/server/api/stream/channels/user-list.ts +++ b/packages/backend/src/server/api/stream/channels/user-list.ts @@ -21,6 +21,7 @@ class UserListChannel extends Channel { private listUsersClock: NodeJS.Timeout; private withFiles: boolean; private withRenotes: boolean; + private idOnly: boolean; constructor( private userListsRepository: UserListsRepository, @@ -40,6 +41,7 @@ class UserListChannel extends Channel { this.listId = params.listId as string; this.withFiles = params.withFiles ?? false; this.withRenotes = params.withRenotes ?? true; + this.idOnly = params.idOnly ?? false; // Check existence and owner const listExist = await this.userListsRepository.exists({ @@ -128,9 +130,13 @@ class UserListChannel extends Channel { } } - this.connection.cacheNote(note); - - this.send('note', note); + if (this.idOnly && ['public', 'home'].includes(note.visibility)) { + const idOnlyNote = { id: note.id }; + this.send('note', idOnlyNote); + } else { + this.connection.cacheNote(note); + this.send('note', note); + } } @bindThis diff --git a/packages/backend/src/server/web/ClientServerService.ts b/packages/backend/src/server/web/ClientServerService.ts index d5c3ef3b01..9c8ee94d6a 100644 --- a/packages/backend/src/server/web/ClientServerService.ts +++ b/packages/backend/src/server/web/ClientServerService.ts @@ -616,6 +616,81 @@ export class ClientServerService { } }); + fastify.get<{ Params: { note: string; } }>('/notes/:note.json', async (request, reply) => { + const note = await this.notesRepository.findOneBy({ + id: request.params.note, + visibility: In(['public', 'home']), + }); + + if (note) { + try { + const _note = await this.noteEntityService.pack(note, null); + reply.header('Content-Type', 'application/json; charset=utf-8'); + reply.header('Cache-Control', 'public, max-age=600'); + return reply.send(_note); + } catch (err) { + reply.header('Cache-Control', 'max-age=10, must-revalidate'); + if (err instanceof IdentifiableError) { + this.clientLoggerService.logger.error(`Internal error occurred in ${request.routeOptions.url}: ${err.message}`, { + path: request.routeOptions.url, + params: request.params, + query: request.query, + id: err.id, + error: { + message: err.message, + code: 'INTERNAL_ERROR', + stack: err.stack, + }, + }); + const httpStatusCode = err.id === '85ab9bd7-3a41-4530-959d-f07073900109' ? 403 : 500; + reply.code(httpStatusCode); + return reply.send({ + message: err.message, + code: 'INTERNAL_ERROR', + id: err.id, + kind: 'server', + httpStatusCode, + info: { + message: err.message, + code: err.name, + id: err.id, + }, + }); + } else { + const error = err as Error; + const errId = randomUUID(); + this.clientLoggerService.logger.error(`Internal error occurred in ${request.routeOptions.url}: ${error.message}`, { + path: request.routeOptions.url, + params: request.params, + query: request.query, + id: errId, + error: { + message: error.message, + code: error.name, + stack: error.stack, + }, + }); + reply.code(500); + return reply.send({ + message: 'Internal error occurred. Please contact us if the error persists.', + code: 'INTERNAL_ERROR', + id: 'b9f2a7f9-fe64-434b-9484-cb1f804d1a80', + kind: 'server', + httpStatusCode: 500, + info: { + message: error.message, + code: error.name, + id: errId, + }, + }); + } + } + } else { + reply.code(404); + return; + } + }); + // Page fastify.get<{ Params: { user: string; page: string; } }>('/@:user/pages/:page', async (request, reply) => { const { username, host } = Acct.parse(request.params.user); diff --git a/packages/backend/test/utils.ts b/packages/backend/test/utils.ts index 8c0a0354a5..6ef69f4d3c 100644 --- a/packages/backend/test/utils.ts +++ b/packages/backend/test/utils.ts @@ -400,7 +400,7 @@ export const waitFire = async (user: UserToken if (timer) clearTimeout(timer); res(true); } - }, params); + }, { ...params, idOnly: false }); } catch (e) { rej(e); } diff --git a/packages/frontend/src/components/MkTimeline.vue b/packages/frontend/src/components/MkTimeline.vue index 06c3b609b1..94d89f8ce6 100644 --- a/packages/frontend/src/components/MkTimeline.vue +++ b/packages/frontend/src/components/MkTimeline.vue @@ -18,15 +18,17 @@ SPDX-License-Identifier: AGPL-3.0-only