mirror/iceshrimp
1
0
Fork 0
mirror of https://iceshrimp.dev/iceshrimp/iceshrimp synced 2024-11-23 22:56:08 +09:00
Code Issues Actions Packages Projects Releases Wiki Activity

[mastodon-client] Add html cache for user profiles and note contents

Browse Source
This commit is contained in:
Laura Hausmann 2023-11-25 22:25:24 +01:00
parent 6832347b6c
commit 61c532a854
No known key found for this signature in database
GPG Key ID: D044E84C5BE01605
18 changed files with 373 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
.config/example-docker.yml
View File

@ -184,6 +184,25 @@ reservedUsernames: [
# Upload or download file size limits (bytes)
#maxFileSize: 262144000
# ┌────────────────────────────────┐
#───┘ Mastodon client API HTML Cache └──────────────────────────
# Caution: rendered post html content is stored in redis (in-memory cache)
# for the duration of ttl, so don't set it too high if you have little system memory.
#
# The prewarm option causes every incoming user/note create/update event to
# be rendered so the cache is always "warm". This trades background cpu load for
# better request response time and better scaling, as posts won't have to be rendered
# on request.
#
# The dbFallback option stores html data that expires into postgres,
# which is more expensive than fetching it from redis,
# but cheaper than re-rendering the HTML.
#htmlCache:
# ttl: 1h
# prewarm: false
# dbFallback: false
#━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
# Congrats, you've reached the end of the config file needed for most deployments!
# Enjoy your Iceshrimp server!

19
.config/example.yml
View File

@ -184,6 +184,25 @@ reservedUsernames: [
# Upload or download file size limits (bytes)
#maxFileSize: 262144000
# ┌────────────────────────────────┐
#───┘ Mastodon client API HTML Cache └──────────────────────────
# Caution: rendered post html content is stored in redis (in-memory cache)
# for the duration of ttl, so don't set it too high if you have little system memory.
#
# The prewarm option causes every incoming user/note create/update event to
# be rendered so the cache is always "warm". This trades background cpu load for
# better request response time and better scaling, as posts won't have to be rendered
# on request.
#
# The dbFallback option stores html data that expires into postgres,
# which is more expensive than fetching it from redis,
# but cheaper than re-rendering the HTML.
#htmlCache:
# ttl: 1h
# prewarm: false
# dbFallback: false
#━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
# Congrats, you've reached the end of the config file needed for most deployments!
# Enjoy your Iceshrimp server!

10
.pnp.cjs generated
View File

@ -7275,6 +7275,7 @@ const RAW_RUNTIME_STATE =
["oauth", "npm:0.10.0"],\
["os-utils", "npm:0.0.14"],\
["otpauth", "npm:9.1.4"],\
["parse-duration", "npm:1.1.0"],\
["parse5", "npm:7.1.2"],\
["pg", "virtual:aa59773ac87791c4813d53447077fcf8a847d6de5a301d34dc31286584b1dbb26d30d3adb5b4c41c1e8aea04371e926fda05c09c6253647c432e11d872a304ba#npm:8.11.1"],\
["private-ip", "npm:2.3.4"],\
@ -19221,6 +19222,15 @@ const RAW_RUNTIME_STATE =
"linkType": "HARD"\
}]\
]],\
["parse-duration", [\
["npm:1.1.0", {\
"packageLocation": "./.yarn/cache/parse-duration-npm-1.1.0-cb12528e2a-c26ab1e3fd.zip/node_modules/parse-duration/",\
"packageDependencies": [\
["parse-duration", "npm:1.1.0"]\
],\
"linkType": "HARD"\
}]\
]],\
["parse-entities", [\
["npm:2.0.0", {\
"packageLocation": "./.yarn/cache/parse-entities-npm-2.0.0-b7b4f46ff6-feb46b5167.zip/node_modules/parse-entities/",\

BIN
.yarn/cache/parse-duration-npm-1.1.0-cb12528e2a-c26ab1e3fd.zip (Stored with Git LFS) vendored Normal file
View File

Binary file not shown.

1
packages/backend/package.json
View File

@ -98,6 +98,7 @@
"oauth": "^0.10.0",
"os-utils": "0.0.14",
"otpauth": "^9.1.3",
"parse-duration": "^1.1.0",
"parse5": "7.1.2",
"pg": "8.11.1",
"private-ip": "2.3.4",

10
packages/backend/src/config/load.ts
View File

@ -8,6 +8,7 @@ import { dirname } from "node:path";
import * as yaml from "js-yaml";
import type { Source, Mixin } from "./types.js";
import Path from "node:path";
import parseDuration from 'parse-duration'
export default function load() {
const _filename = fileURLToPath(import.meta.url);
@ -53,6 +54,15 @@ export default function load() {
...config.images,
};
config.htmlCache = {
ttlSeconds: parseDuration(config.htmlCache?.ttl ?? '1h', 's')!,
prewarm: false,
dbFallback: false,
...config.htmlCache,
}
if (config.htmlCache.ttlSeconds == null) throw new Error('Failed to parse config.ttl');
config.searchEngine = config.searchEngine ?? 'https://duckduckgo.com/?q=';
mixin.version = meta.version;

7
packages/backend/src/config/types.ts
View File

@ -41,6 +41,13 @@ export type Source = {
info?: string;
};
htmlCache?: {
ttl?: string;
ttlSeconds?: number;
prewarm?: boolean;
dbFallback?: boolean;
}
searchEngine?: string;
proxy?: string;

6
packages/backend/src/db/postgre.ts
View File

@ -71,12 +71,12 @@ import { UserPending } from "@/models/entities/user-pending.js";
import { Webhook } from "@/models/entities/webhook.js";
import { UserIp } from "@/models/entities/user-ip.js";
import { NoteEdit } from "@/models/entities/note-edit.js";
import { entities as charts } from "@/services/chart/entities.js";
import { envOption } from "../env.js";
import { dbLogger } from "./logger.js";
import { OAuthApp } from "@/models/entities/oauth-app.js";
import { OAuthToken } from "@/models/entities/oauth-token.js";
import { HtmlNoteCacheEntry } from "@/models/entities/html-note-cache-entry.js";
import { HtmlUserCacheEntry } from "@/models/entities/html-user-cache-entry.js";
const sqlLogger = dbLogger.createSubLogger("sql", "gray", false);
class MyCustomLogger implements Logger {
@ -179,6 +179,8 @@ export const entities = [
UserIp,
OAuthApp,
OAuthToken,
HtmlNoteCacheEntry,
HtmlUserCacheEntry,
...charts,
];

20
packages/backend/src/migration/1700962939886-add-html-cache.ts Normal file
View File

@ -0,0 +1,20 @@
import { MigrationInterface, QueryRunner } from "typeorm";
export class AddHtmlCache1700962939886 implements MigrationInterface {
name = 'AddHtmlCache1700962939886'
public async up(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(`CREATE TABLE "html_note_cache_entry" ("noteId" character varying(32) NOT NULL, "updatedAt" TIMESTAMP WITH TIME ZONE, "content" text, CONSTRAINT "PK_6ef86ec901b2017cbe82d3a8286" PRIMARY KEY ("noteId"))`);
await queryRunner.query(`CREATE TABLE "html_user_cache_entry" ("userId" character varying(32) NOT NULL, "updatedAt" TIMESTAMP WITH TIME ZONE, "bio" text, "fields" jsonb NOT NULL DEFAULT '[]', CONSTRAINT "PK_920b9474e3c9cae3f3c37c057e1" PRIMARY KEY ("userId"))`);
await queryRunner.query(`ALTER TABLE "html_note_cache_entry" ADD CONSTRAINT "FK_6ef86ec901b2017cbe82d3a8286" FOREIGN KEY ("noteId") REFERENCES "note"("id") ON DELETE CASCADE ON UPDATE NO ACTION`);
await queryRunner.query(`ALTER TABLE "html_user_cache_entry" ADD CONSTRAINT "FK_920b9474e3c9cae3f3c37c057e1" FOREIGN KEY ("userId") REFERENCES "user"("id") ON DELETE CASCADE ON UPDATE NO ACTION`);
}
public async down(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(`ALTER TABLE "html_user_cache_entry" DROP CONSTRAINT "FK_920b9474e3c9cae3f3c37c057e1"`);
await queryRunner.query(`ALTER TABLE "html_note_cache_entry" DROP CONSTRAINT "FK_6ef86ec901b2017cbe82d3a8286"`);
await queryRunner.query(`DROP TABLE "html_user_cache_entry"`);
await queryRunner.query(`DROP TABLE "html_note_cache_entry"`);
}
}

21
packages/backend/src/models/entities/html-note-cache-entry.ts Normal file
View File

@ -0,0 +1,21 @@
import { Entity, PrimaryColumn, Column, ManyToOne, JoinColumn } from "typeorm";
import { id } from "../id.js";
import { Note } from "./note.js";
@Entity()
export class HtmlNoteCacheEntry {
@PrimaryColumn(id())
public noteId: Note["id"];
@ManyToOne((type) => Note, {
onDelete: "CASCADE",
})
@JoinColumn()
public note: Note | null;
@Column("timestamp with time zone", { nullable: true })
public updatedAt: Date;
@Column("text", { nullable: true })
public content: string | null;
}

26
packages/backend/src/models/entities/html-user-cache-entry.ts Normal file
View File

@ -0,0 +1,26 @@
import { Entity, PrimaryColumn, Column, ManyToOne, JoinColumn } from "typeorm";
import { User } from "@/models/entities/user.js";
import { id } from "../id.js";
@Entity()
export class HtmlUserCacheEntry {
@PrimaryColumn(id())
public userId: User["id"];
@ManyToOne((type) => User, {
onDelete: "CASCADE",
})
@JoinColumn()
public user: User | null;
@Column("timestamp with time zone", { nullable: true })
public updatedAt: Date;
@Column("text", { nullable: true })
public bio: string | null;
@Column("jsonb", {
default: [],
})
public fields: MastodonEntity.Field[];
}

4
packages/backend/src/models/index.ts
View File

@ -69,6 +69,8 @@ import { NoteEdit } from "./entities/note-edit.js";
import { OAuthApp } from "@/models/entities/oauth-app.js";
import { OAuthToken } from "@/models/entities/oauth-token.js";
import { UserProfileRepository } from "@/models/repositories/user-profile.js";
import { HtmlNoteCacheEntry } from "@/models/entities/html-note-cache-entry.js";
import { HtmlUserCacheEntry } from "@/models/entities/html-user-cache-entry.js";
export const Announcements = db.getRepository(Announcement);
export const AnnouncementReads = db.getRepository(AnnouncementRead);
@ -136,3 +138,5 @@ export const Webhooks = db.getRepository(Webhook);
export const PasswordResetRequests = db.getRepository(PasswordResetRequest);
export const OAuthApps = db.getRepository(OAuthApp);
export const OAuthTokens = db.getRepository(OAuthToken);
export const HtmlUserCacheEntries = db.getRepository(HtmlUserCacheEntry);
export const HtmlNoteCacheEntries = db.getRepository(HtmlNoteCacheEntry);

11
packages/backend/src/remote/activitypub/models/person.ts
View File

@ -54,6 +54,7 @@ import {
getSubjectHostFromAcctParts
} from "@/remote/resolve-user.js"
import { RecursionLimiter } from "@/models/repositories/user-profile.js";
import { UserConverter } from "@/server/api/mastodon/converters/user.js";
const logger = apLogger;
@ -397,8 +398,9 @@ export async function createPerson(
// Hashtag update
updateUsertags(user!, tags);
// Mentions update
if (await limiter.shouldContinue()) UserProfiles.updateMentions(user!.id, limiter);
// Mentions update, then prewarm html cache
if (await limiter.shouldContinue()) UserProfiles.updateMentions(user!.id, limiter)
.then(_ => UserConverter.prewarmCacheById(user!.id));
//#region Fetch avatar and header image
const [avatar, banner] = await Promise.all(
@ -635,8 +637,9 @@ export async function updatePerson(
// Hashtag Update
updateUsertags(user, tags);
// Mentions update
UserProfiles.updateMentions(user!.id);
// Mentions update, then prewarm html cache
UserProfiles.updateMentions(user!.id)
.then(_ => UserConverter.prewarmCacheById(user!.id));
// If the user in question is a follower, followers will also be updated.
await Followings.update(

99
packages/backend/src/server/api/mastodon/converters/note.ts
View File

@ -8,7 +8,15 @@ import { VisibilityConverter } from "@/server/api/mastodon/converters/visibility
import { escapeMFM } from "@/server/api/mastodon/converters/mfm.js";
import { aggregateNoteEmojis, PopulatedEmoji, populateEmojis, prefetchEmojis } from "@/misc/populate-emojis.js";
import { EmojiConverter } from "@/server/api/mastodon/converters/emoji.js";
import { DriveFiles, NoteFavorites, NoteReactions, Notes, NoteThreadMutings, UserNotePinings } from "@/models/index.js";
import {
DriveFiles,
HtmlNoteCacheEntries,
NoteFavorites,
NoteReactions,
Notes,
NoteThreadMutings,
UserNotePinings
} from "@/models/index.js";
import { decodeReaction } from "@/misc/reaction-lib.js";
import { MentionConverter } from "@/server/api/mastodon/converters/mention.js";
import { PollConverter } from "@/server/api/mastodon/converters/poll.js";
@ -23,8 +31,11 @@ import { NoteHelpers } from "@/server/api/mastodon/helpers/note.js";
import isQuote from "@/misc/is-quote.js";
import { unique } from "@/prelude/array.js";
import { NoteReaction } from "@/models/entities/note-reaction.js";
import { Cache } from "@/misc/cache.js";
import { HtmlNoteCacheEntry } from "@/models/entities/html-note-cache-entry.js";
export class NoteConverter {
private static noteContentHtmlCache = new Cache<string | null>('html:note:content', config.htmlCache?.ttlSeconds ?? 60 * 60);
public static async encode(note: Note, ctx: MastoContext, recurseCounter: number = 2): Promise<MastodonEntity.Status> {
const user = ctx.user as ILocalUser | null;
const noteUser = note.user ?? UserHelpers.getUserCached(note.userId, ctx);
@ -102,12 +113,16 @@ export class NoteConverter {
return renote.url ?? renote.uri ?? `${config.url}/notes/${renote.id}`;
});
const identifier = `${note.id}:${(note.updatedAt ?? note.createdAt).getTime()}`;
const text = quoteUri.then(quoteUri => note.text !== null ? quoteUri !== null ? note.text.replaceAll(`RE: ${quoteUri}`, '').replaceAll(quoteUri, '').trimEnd() : note.text : null);
const content = text.then(text => text !== null
? quoteUri.then(quoteUri => MfmHelpers.toHtml(mfm.parse(text), JSON.parse(note.mentionedRemoteUsers), note.userHost, false, quoteUri))
.then(p => p ?? escapeMFM(text))
: "");
const content = this.noteContentHtmlCache.fetch(identifier, async () =>
Promise.resolve(await this.fetchFromCacheWithFallback(note, ctx) ?? text.then(text => text !== null
? quoteUri.then(quoteUri => MfmHelpers.toHtml(mfm.parse(text), JSON.parse(note.mentionedRemoteUsers), note.userHost, false, quoteUri))
.then(p => p ?? escapeMFM(text))
: "")), true)
.then(p => p ?? '');
const isPinned = (ctx.pinAggregate as Map<string, boolean>)?.get(note.id)
?? (user && note.userId === user.id
@ -174,16 +189,28 @@ export class NoteConverter {
const reactionAggregate = new Map<Note["id"], NoteReaction | null>();
const renoteAggregate = new Map<Note["id"], boolean>();
const mutingAggregate = new Map<Note["id"], boolean>();
const bookmarkAggregate = new Map<Note["id"], boolean>();;
const bookmarkAggregate = new Map<Note["id"], boolean>();
const pinAggregate = new Map<Note["id"], boolean>();
const htmlNoteCacheAggregate = new Map<Note["id"], HtmlNoteCacheEntry | null>();
const renoteIds = notes
.filter((n) => n.renoteId != null)
.map((n) => n.renoteId!);
const noteIds = unique(notes.map((n) => n.id));
const targets = unique([...noteIds, ...renoteIds]);
if (config.htmlCache?.dbFallback) {
const htmlNoteCacheEntries = await HtmlNoteCacheEntries.findBy({
noteId: In(targets)
});
for (const target of targets) {
htmlNoteCacheAggregate.set(target, htmlNoteCacheEntries.find(n => n.noteId === target) ?? null);
}
}
if (user?.id != null) {
const renoteIds = notes
.filter((n) => n.renoteId != null)
.map((n) => n.renoteId!);
const noteIds = unique(notes.map((n) => n.id));
const targets = unique([...noteIds, ...renoteIds]);
const mutingTargets = unique([...notes.map(n => n.threadId ?? n.id)]);
const pinTargets = unique([...notes.filter(n => n.userId === user.id).map(n => n.id)]);
@ -239,9 +266,12 @@ export class NoteConverter {
ctx.mutingAggregate = mutingAggregate;
ctx.bookmarkAggregate = bookmarkAggregate;
ctx.pinAggregate = pinAggregate;
ctx.htmlNoteCacheAggregate = htmlNoteCacheAggregate;
const users = notes.filter(p => !!p.user).map(p => p.user as User);
const renoteUserIds = notes.filter(p => p.renoteUserId !== null).map(p => p.renoteUserId as string);
await UserConverter.aggregateData([...users], ctx)
await UserConverter.aggregateDataByIds(renoteUserIds, ctx);
await prefetchEmojis(aggregateNoteEmojis(notes));
}
@ -268,4 +298,49 @@ export class NoteConverter {
NoteHelpers.fixupEventNote(note);
return NoteConverter.encode(note, ctx);
}
private static async fetchFromCacheWithFallback(note: Note, ctx: MastoContext): Promise<string | null> {
if (!config.htmlCache?.dbFallback) return null;
let dbHit: HtmlNoteCacheEntry | Promise<HtmlNoteCacheEntry | null> | null | undefined = (ctx.htmlNoteCacheAggregate as Map<string, HtmlNoteCacheEntry | null> | undefined)?.get(note.id);
if (dbHit === undefined) dbHit = HtmlNoteCacheEntries.findOneBy({ noteId: note.id });
return Promise.resolve(dbHit)
.then(res => {
if (res === null || (res.updatedAt !== note.updatedAt)) {
this.prewarmCache(note);
return null;
}
return res;
})
.then(hit => hit?.updatedAt === note.updatedAt ? hit?.content ?? null : null);
}
public static async prewarmCache(note: Note): Promise<void> {
if (!config.htmlCache?.prewarm) return;
const identifier = `${note.id}:${(note.updatedAt ?? note.createdAt).getTime()}`;
if (await this.noteContentHtmlCache.get(identifier) !== undefined) return;
const quoteUri = note.renote
? isQuote(note)
? (note.renote.url ?? note.renote.uri ?? `${config.url}/notes/${note.renote.id}`)
: null
: null;
const text = note.text !== null ? quoteUri !== null ? note.text.replaceAll(`RE: ${quoteUri}`, '').replaceAll(quoteUri, '').trimEnd() : note.text : null;
const content = text !== null
? MfmHelpers.toHtml(mfm.parse(text), JSON.parse(note.mentionedRemoteUsers), note.userHost, false, quoteUri)
.then(p => p ?? escapeMFM(text))
: null;
if (note.user) UserConverter.prewarmCache(note.user);
else if (note.userId) UserConverter.prewarmCacheById(note.userId);
if (note.replyUserId) UserConverter.prewarmCacheById(note.replyUserId);
if (note.renoteUserId) UserConverter.prewarmCacheById(note.renoteUserId);
this.noteContentHtmlCache.set(identifier, await content);
if (config.htmlCache?.dbFallback)
HtmlNoteCacheEntries.upsert({ noteId: note.id, updatedAt: note.updatedAt, content: await content }, ["noteId"]);
}
}

124
packages/backend/src/server/api/mastodon/converters/user.ts
View File

@ -1,6 +1,6 @@
import { ILocalUser, User } from "@/models/entities/user.js";
import config from "@/config/index.js";
import { DriveFiles, Followings, UserProfiles, Users } from "@/models/index.js";
import {DriveFiles, Followings, HtmlUserCacheEntries, UserProfiles, Users} from "@/models/index.js";
import { EmojiConverter } from "@/server/api/mastodon/converters/emoji.js";
import { populateEmojis } from "@/misc/populate-emojis.js";
import { escapeMFM } from "@/server/api/mastodon/converters/mfm.js";
@ -9,10 +9,14 @@ import { awaitAll } from "@/prelude/await-all.js";
import { AccountCache, UserHelpers } from "@/server/api/mastodon/helpers/user.js";
import { MfmHelpers } from "@/server/api/mastodon/helpers/mfm.js";
import { MastoContext } from "@/server/api/mastodon/index.js";
import { IMentionedRemoteUsers } from "@/models/entities/note.js";
import {IMentionedRemoteUsers, Note} from "@/models/entities/note.js";
import { UserProfile } from "@/models/entities/user-profile.js";
import { In } from "typeorm";
import { unique } from "@/prelude/array.js";
import { Cache } from "@/misc/cache.js";
import { getUser } from "../../common/getters.js";
import { HtmlUserCacheEntry } from "@/models/entities/html-user-cache-entry.js";
import AsyncLock from "async-lock";
type Field = {
name: string;
@ -21,6 +25,9 @@ type Field = {
};
export class UserConverter {
private static userBioHtmlCache = new Cache<string | null>('html:user:bio', config.htmlCache?.ttlSeconds ?? 60 * 60);
private static userFieldsHtmlCache = new Cache<MastodonEntity.Field[]>('html:user:fields', config.htmlCache?.ttlSeconds ?? 60 * 60);
public static async encode(u: User, ctx: MastoContext): Promise<MastodonEntity.Account> {
const localUser = ctx.user as ILocalUser | null;
const cache = ctx.cache as AccountCache;
@ -28,6 +35,7 @@ export class UserConverter {
const cacheHit = cache.accounts.find(p => p.id == u.id);
if (cacheHit) return cacheHit;
const identifier = `${u.id}:${(u.updatedAt ?? u.createdAt).getTime()}`;
let fqn = `${u.username}@${u.host ?? config.domain}`;
let acct = u.username;
let acctUrl = `https://${u.host || config.host}/@${u.username}`;
@ -38,15 +46,33 @@ export class UserConverter {
const aggregateProfile = (ctx.userProfileAggregate as Map<string, UserProfile | null>)?.get(u.id);
let htmlCacheEntry: HtmlUserCacheEntry | null | undefined = undefined;
const htmlCacheEntryLock = new AsyncLock();
const profile = aggregateProfile !== undefined
? aggregateProfile
: UserProfiles.findOneBy({ userId: u.id });
const bio = Promise.resolve(profile).then(profile => MfmHelpers.toHtml(mfm.parse(profile?.description ?? ""), profile?.mentions, u.host).then(p => p ?? escapeMFM(profile?.description ?? "")));
const bio = this.userBioHtmlCache.fetch(identifier, async () => {
return htmlCacheEntryLock.acquire(u.id, async () => {
if (htmlCacheEntry === undefined) await this.fetchFromCacheWithFallback(u, await profile, ctx);
if (htmlCacheEntry === null) {
return Promise.resolve(profile).then(async profile => {
return MfmHelpers.toHtml(mfm.parse(profile?.description ?? ""), profile?.mentions, u.host)
.then(p => p ?? escapeMFM(profile?.description ?? ""))
.then(p => p !== '<p></p>' ? p : null)
});
}
return htmlCacheEntry?.bio ?? null;
});
}, true)
.then(p => p ?? '<p></p>');
const avatar = u.avatarId
? DriveFiles.getFinalUrlMaybe(u.avatarUrl) ?? (DriveFiles.findOneBy({ id: u.avatarId }))
.then(p => p?.url ?? Users.getIdenticonUrl(u.id))
.then(p => DriveFiles.getFinalUrl(p))
: Users.getIdenticonUrl(u.id);
const banner = u.bannerId
? DriveFiles.getFinalUrlMaybe(u.bannerUrl) ?? (DriveFiles.findOneBy({ id: u.bannerId }))
.then(p => p?.url ?? `${config.url}/static-assets/transparent.png`)
@ -75,6 +101,7 @@ export class UserConverter {
return localUser?.id === profile.userId ? u.followersCount : 0;
}
});
const followingCount = Promise.resolve(profile).then(async profile => {
if (profile === null) return u.followingCount;
switch (profile.ffVisibility) {
@ -87,6 +114,17 @@ export class UserConverter {
}
});
const fields =
this.userFieldsHtmlCache.fetch(identifier, async () => {
return htmlCacheEntryLock.acquire(u.id, async () => {
if (htmlCacheEntry === undefined) await this.fetchFromCacheWithFallback(u, await profile, ctx);
if (htmlCacheEntry === null) {
return Promise.resolve(profile).then(profile => Promise.all(profile?.fields.map(async p => this.encodeField(p, u.host, profile?.mentions)) ?? []));
}
return htmlCacheEntry?.fields ?? [];
});
}, true);
return awaitAll({
id: u.id,
username: u.username,
@ -106,7 +144,7 @@ export class UserConverter {
header_static: banner,
emojis: populateEmojis(u.emojis, u.host).then(emoji => emoji.map((e) => EmojiConverter.encode(e))),
moved: null, //FIXME
fields: Promise.resolve(profile).then(profile => Promise.all(profile?.fields.map(async p => this.encodeField(p, u.host, profile?.mentions)) ?? [])),
fields: fields,
bot: u.isBot,
discoverable: u.isExplorable
}).then(p => {
@ -124,6 +162,17 @@ export class UserConverter {
const followedOrSelfAggregate = new Map<User["id"], boolean>();
const userProfileAggregate = new Map<User["id"], UserProfile | null>();
const htmlUserCacheAggregate = ctx.htmlUserCacheAggregate ?? new Map<Note["id"], HtmlUserCacheEntry | null>();
if (config.htmlCache?.dbFallback) {
const htmlUserCacheEntries = await HtmlUserCacheEntries.findBy({
userId: In(targets)
});
for (const target of targets) {
htmlUserCacheAggregate.set(target, htmlUserCacheEntries.find(n => n.userId === target) ?? null);
}
}
if (user) {
const targetsWithoutSelf = targets.filter(u => u !== user.id);
@ -152,6 +201,24 @@ export class UserConverter {
}
ctx.followedOrSelfAggregate = followedOrSelfAggregate;
ctx.htmlUserCacheAggregate = htmlUserCacheAggregate;
}
public static async aggregateDataByIds(userIds: User["id"][], ctx: MastoContext): Promise<void> {
const targets = unique(userIds);
const htmlUserCacheAggregate = ctx.htmlUserCacheAggregate ?? new Map<Note["id"], HtmlUserCacheEntry | null>();
if (config.htmlCache?.dbFallback) {
const htmlUserCacheEntries = await HtmlUserCacheEntries.findBy({
userId: In(targets)
});
for (const target of targets) {
htmlUserCacheAggregate.set(target, htmlUserCacheEntries.find(n => n.userId === target) ?? null);
}
}
ctx.htmlUserCacheAggregate = htmlUserCacheAggregate;
}
public static async encodeMany(users: User[], ctx: MastoContext): Promise<MastodonEntity.Account[]> {
@ -167,4 +234,53 @@ export class UserConverter {
verified_at: f.verified ? (new Date()).toISOString() : null,
}
}
private static async fetchFromCacheWithFallback(user: User, profile: UserProfile | null, ctx: MastoContext): Promise<HtmlUserCacheEntry | null> {
if (!config.htmlCache?.dbFallback) return null;
let dbHit: HtmlUserCacheEntry | Promise<HtmlUserCacheEntry | null> | null | undefined = (ctx.htmlUserCacheAggregate as Map<string, HtmlUserCacheEntry | null> | undefined)?.get(user.id);
if (dbHit === undefined) dbHit = HtmlUserCacheEntries.findOneBy({ userId: user.id });
return Promise.resolve(dbHit)
.then(res => {
if (res === null || (res.updatedAt !== user.updatedAt ?? user.createdAt)) {
this.prewarmCache(user, profile);
return null;
}
return res;
});
}
public static async prewarmCache(user: User, profile?: UserProfile | null): Promise<void> {
if (!config.htmlCache?.prewarm) return;
const identifier = `${user.id}:${(user.updatedAt ?? user.createdAt).getTime()}`;
if (profile !== null) {
if (profile === undefined) {
profile = await UserProfiles.findOneBy({userId: user.id});
}
if (await this.userBioHtmlCache.get(identifier) === undefined) {
const bio = MfmHelpers.toHtml(mfm.parse(profile?.description ?? ""), profile?.mentions, user.host)
.then(p => p ?? escapeMFM(profile?.description ?? ""))
.then(p => p !== '<p></p>' ? p : null);
this.userBioHtmlCache.set(identifier, await bio);
if (config.htmlCache?.dbFallback)
HtmlUserCacheEntries.upsert({ userId: user.id, bio: await bio }, ["userId"]);
}
if (await this.userFieldsHtmlCache.get(identifier) === undefined) {
const fields = await Promise.all(profile!.fields.map(async p => this.encodeField(p, user.host, profile!.mentions)) ?? []);
this.userFieldsHtmlCache.set(identifier, fields);
if (config.htmlCache?.dbFallback)
HtmlUserCacheEntries.upsert({ userId: user.id, updatedAt: user.updatedAt ?? user.createdAt, fields: fields }, ["userId"]);
}
}
}
public static async prewarmCacheById(userId: string): Promise<void> {
await this.prewarmCache(await getUser(userId));
}
}

5
packages/backend/src/services/note/create.ts
View File

@ -65,6 +65,7 @@ import { shouldSilenceInstance } from "@/misc/should-block-instance.js";
import { redisClient } from "@/db/redis.js";
import { Mutex } from "redis-semaphore";
import { RecursionLimiter } from "@/models/repositories/user-profile.js";
import { NoteConverter } from "@/server/api/mastodon/converters/note.js";
const mutedWordsCache = new Cache<
{ userId: UserProfile["userId"]; mutedWords: UserProfile["mutedWords"] }[]
@ -341,9 +342,11 @@ export default async (
) {
await incRenoteCount(data.renote);
}
res(note);
// Prewarm html cache
NoteConverter.prewarmCache(note);
// 統計を更新
notesChart.update(note, true);
perUserNotesChart.update(user, note, true);

3
packages/backend/src/services/note/edit.ts
View File

@ -26,6 +26,7 @@ import { deliverToRelays } from "../relay.js";
import renderUpdate from "@/remote/activitypub/renderer/update.js";
import { extractMentionedUsers } from "@/services/note/create.js";
import { normalizeForSearch } from "@/misc/normalize-for-search.js";
import { NoteConverter } from "@/server/api/mastodon/converters/note.js";
type Option = {
text?: string | null;
@ -182,6 +183,8 @@ export default async function (
note = await Notes.findOneByOrFail({ id: note.id });
if (publishing) {
NoteConverter.prewarmCache(note);
// Publish update event for the updated note details
publishNoteStream(note.id, "updated", {
updatedAt: update.updatedAt,

8
yarn.lock
View File

@ -5575,6 +5575,7 @@ __metadata:
oauth: "npm:^0.10.0"
os-utils: "npm:0.0.14"
otpauth: "npm:^9.1.3"
parse-duration: "npm:^1.1.0"
parse5: "npm:7.1.2"
pg: "npm:8.11.1"
private-ip: "npm:2.3.4"
@ -15999,6 +16000,13 @@ __metadata:
languageName: node
linkType: hard
"parse-duration@npm:^1.1.0":
version: 1.1.0
resolution: "parse-duration@npm:1.1.0"
checksum: c26ab1e3fdf1dc4b7006e87a82fd33c7dbee3116413a59369bbc3b160a8e7ed88616852c4c3dde23b7a857e270cb18fccf629ff52220803194239f8e092774a9
languageName: node
linkType: hard
"parse-entities@npm:^2.0.0":
version: 2.0.0
resolution: "parse-entities@npm:2.0.0"