From 74c7b5fe70541f1819c0fb3c5686b429f04bf81c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EC=95=84=EB=A5=B4=ED=8E=98?= Date: Fri, 16 Feb 2024 06:41:21 +0900 Subject: [PATCH] wip: auto note removal --- packages/backend/src/core/QueueService.ts | 6 + .../AutoRemovalConditionEntityService.ts | 41 +++++++ packages/backend/src/di-symbols.ts | 1 + packages/backend/src/misc/json-schema.ts | 2 + .../src/models/AutoRemovalCondition.ts | 40 +++++++ .../backend/src/models/RepositoryModule.ts | 9 ++ packages/backend/src/models/User.ts | 20 ++++ packages/backend/src/models/_.ts | 3 + .../json-schema/auto-removal-condition.ts | 28 +++++ packages/backend/src/postgres.ts | 2 + .../AutoNoteRemovalProcessorService.ts | 106 ++++++++++++++++++ .../src/server/api/endpoints/i/update.ts | 14 ++- 12 files changed, 270 insertions(+), 2 deletions(-) create mode 100644 packages/backend/src/core/entities/AutoRemovalConditionEntityService.ts create mode 100644 packages/backend/src/models/AutoRemovalCondition.ts create mode 100644 packages/backend/src/models/json-schema/auto-removal-condition.ts create mode 100644 packages/backend/src/queue/processors/AutoNoteRemovalProcessorService.ts diff --git a/packages/backend/src/core/QueueService.ts b/packages/backend/src/core/QueueService.ts index 3a4724a25..d59cc2b15 100644 --- a/packages/backend/src/core/QueueService.ts +++ b/packages/backend/src/core/QueueService.ts @@ -69,6 +69,12 @@ export class QueueService { repeat: { pattern: '*/5 * * * *' }, removeOnComplete: true, }); + + this.systemQueue.add('autoNoteRemoval', { + }, { + repeat: { pattern: '0 0 * * *' }, + removeOnComplete: true, + }); } @bindThis diff --git a/packages/backend/src/core/entities/AutoRemovalConditionEntityService.ts b/packages/backend/src/core/entities/AutoRemovalConditionEntityService.ts new file mode 100644 index 000000000..87daed5fb --- /dev/null +++ b/packages/backend/src/core/entities/AutoRemovalConditionEntityService.ts @@ -0,0 +1,41 @@ +/* + * SPDX-FileCopyrightText: syuilo and other misskey, cherrypick contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import { Inject, Injectable } from '@nestjs/common'; +import { DI } from '@/di-symbols.js'; +import type { AutoRemovalConditionRepository, MiAutoRemovalCondition } from '@/models/_.js'; +import type { Packed } from '@/misc/json-schema.js'; +import { bindThis } from '@/decorators.js'; + +@Injectable() +export class AutoRemovalConditionEntityService { + constructor( + @Inject(DI.autoRemovalConditionRepository) + private autoRemovalConditionRepository: AutoRemovalConditionRepository, + ) { + } + + @bindThis + public async pack( + src: MiAutoRemovalCondition['id'] | MiAutoRemovalCondition, + ): Promise> { + const condition = typeof src === 'object' ? src : await this.autoRemovalConditionRepository.findOneByOrFail({ id: src }); + + return { + id: condition.id, + deleteAfter: condition.deleteAfter, + noPiningNotes: condition.noPiningNotes, + noSpecifiedNotes: condition.noSpecifiedNotes, + }; + } + + @bindThis + public packMany( + conditions: any[], + ) { + return Promise.all(conditions.map(x => this.pack(x))); + } +} + diff --git a/packages/backend/src/di-symbols.ts b/packages/backend/src/di-symbols.ts index fb570d0b4..ba281b968 100644 --- a/packages/backend/src/di-symbols.ts +++ b/packages/backend/src/di-symbols.ts @@ -84,5 +84,6 @@ export const DI = { userMemosRepository: Symbol('userMemosRepository'), bubbleGameRecordsRepository: Symbol('bubbleGameRecordsRepository'), reversiGamesRepository: Symbol('reversiGamesRepository'), + autoRemovalConditionRepository: Symbol('autoRemovalConditionRepository'), //#endregion }; diff --git a/packages/backend/src/misc/json-schema.ts b/packages/backend/src/misc/json-schema.ts index 27fd280a8..d86016f68 100644 --- a/packages/backend/src/misc/json-schema.ts +++ b/packages/backend/src/misc/json-schema.ts @@ -15,6 +15,7 @@ import { import { packedAbuseUserReportSchema } from '@/models/json-schema/abuse-user-report.js'; import { packedAntennaSchema } from '@/models/json-schema/antenna.js'; import { packedAppSchema } from '@/models/json-schema/app.js'; +import { packedAutoRemovalConditionSchema } from '@/models/json-schema/auto-removal-condition.js'; import { packedBlockingSchema } from '@/models/json-schema/blocking.js'; import { packedChannelSchema } from '@/models/json-schema/channel.js'; import { packedClipSchema } from '@/models/json-schema/clip.js'; @@ -101,6 +102,7 @@ export const refs = { EmojiDetailed: packedEmojiDetailedSchema, Flash: packedFlashSchema, FlashLike: packedFlashLikeSchema, + AutoRemovalCondition: packedAutoRemovalConditionSchema, Signin: packedSigninSchema, RoleCondFormulaLogics: packedRoleCondFormulaLogicsSchema, diff --git a/packages/backend/src/models/AutoRemovalCondition.ts b/packages/backend/src/models/AutoRemovalCondition.ts new file mode 100644 index 000000000..e01d019ac --- /dev/null +++ b/packages/backend/src/models/AutoRemovalCondition.ts @@ -0,0 +1,40 @@ +/* + * SPDX-FileCopyrightText: syuilo and noridev and other misskey, cherrypick contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import { Entity, Column, PrimaryColumn, Index } from 'typeorm'; +import { id } from './util/id.js'; + +@Entity('auto_removal_condition') +// @Index(['userId'], { unique: true }) +export class MiAutoRemovalCondition { + @PrimaryColumn(id()) + public id: string; + + @Column('bigint', { + default: 7, + nullable: false, + }) + public deleteAfter: number; + + @Column('boolean', { + default: true, + nullable: false, + }) + public noPiningNotes: boolean; + + @Column('boolean', { + default: true, + nullable: false, + }) + public noSpecifiedNotes: boolean; + + constructor(data: Partial) { + if (data == null) return; + + for (const [k, v] of Object.entries(data)) { + (this as any)[k] = v; + } + } +} diff --git a/packages/backend/src/models/RepositoryModule.ts b/packages/backend/src/models/RepositoryModule.ts index a61094500..f3b265ef8 100644 --- a/packages/backend/src/models/RepositoryModule.ts +++ b/packages/backend/src/models/RepositoryModule.ts @@ -16,6 +16,7 @@ import { MiApp, MiAuthSession, MiAvatarDecoration, + MiAutoRemovalCondition, MiBlocking, MiChannel, MiChannelFavorite, @@ -314,6 +315,12 @@ const $authSessionsRepository: Provider = { inject: [DI.db], }; +const $autoRemovalConditionRepository: Provider = { + provide: DI.autoRemovalConditionRepository, + useFactory: (db: DataSource) => db.getRepository(MiAutoRemovalCondition), + inject: [DI.db], +}; + const $accessTokensRepository: Provider = { provide: DI.accessTokensRepository, useFactory: (db: DataSource) => db.getRepository(MiAccessToken), @@ -574,6 +581,7 @@ const $abuseReportResolversRepository: Provider = { $bubbleGameRecordsRepository, $reversiGamesRepository, $abuseReportResolversRepository, + $autoRemovalConditionRepository, ], exports: [ $usersRepository, @@ -646,6 +654,7 @@ const $abuseReportResolversRepository: Provider = { $bubbleGameRecordsRepository, $reversiGamesRepository, $abuseReportResolversRepository, + $autoRemovalConditionRepository, ], }) export class RepositoryModule {} diff --git a/packages/backend/src/models/User.ts b/packages/backend/src/models/User.ts index c52dc73f0..6630a0f81 100644 --- a/packages/backend/src/models/User.ts +++ b/packages/backend/src/models/User.ts @@ -6,6 +6,7 @@ import { Entity, Column, Index, OneToOne, JoinColumn, PrimaryColumn } from 'typeorm'; import { id } from './util/id.js'; import { MiDriveFile } from './DriveFile.js'; +import { MiAutoRemovalCondition } from './AutoRemovalCondition.js'; @Entity('user') @Index(['usernameLower', 'host'], { unique: true }) @@ -217,6 +218,12 @@ export class MiUser { }) public isDeleted: boolean; + @Column('boolean', { + default: false, + comment: 'Whether the User is using note auto removal.', + }) + public autoRemoval: boolean; + @Column('varchar', { length: 128, array: true, default: '{}', }) @@ -267,6 +274,18 @@ export class MiUser { }) public token: string | null; + @Column({ + ...id(), + nullable: false, + }) + public autoRemovalConditionId: MiAutoRemovalCondition['id']; + + @OneToOne(type => MiAutoRemovalCondition, { + onDelete: 'CASCADE', + }) + @JoinColumn() + public autoRemovalCondition: MiAutoRemovalCondition; + constructor(data: Partial) { if (data == null) return; @@ -304,3 +323,4 @@ export const nameSchema = { type: 'string', minLength: 1, maxLength: 50 } as con export const descriptionSchema = { type: 'string', minLength: 1, maxLength: 1500 } as const; export const locationSchema = { type: 'string', minLength: 1, maxLength: 50 } as const; export const birthdaySchema = { type: 'string', pattern: /^([0-9]{4})-([0-9]{2})-([0-9]{2})$/.toString().slice(1, -1) } as const; + diff --git a/packages/backend/src/models/_.ts b/packages/backend/src/models/_.ts index ca1410c24..3f3c27514 100644 --- a/packages/backend/src/models/_.ts +++ b/packages/backend/src/models/_.ts @@ -13,6 +13,7 @@ import { MiAntenna } from '@/models/Antenna.js'; import { MiApp } from '@/models/App.js'; import { MiAvatarDecoration } from '@/models/AvatarDecoration.js'; import { MiAuthSession } from '@/models/AuthSession.js'; +import { MiAutoRemovalCondition } from '@/models/AutoRemovalCondition.js'; import { MiBlocking } from '@/models/Blocking.js'; import { MiChannelFollowing } from '@/models/ChannelFollowing.js'; import { MiChannelFavorite } from '@/models/ChannelFavorite.js'; @@ -86,6 +87,7 @@ export { MiApp, MiAvatarDecoration, MiAuthSession, + MiAutoRemovalCondition, MiBlocking, MiChannelFollowing, MiChannelFavorite, @@ -158,6 +160,7 @@ export type AntennasRepository = Repository; export type AppsRepository = Repository; export type AvatarDecorationsRepository = Repository; export type AuthSessionsRepository = Repository; +export type AutoRemovalConditionRepository = Repository; export type BlockingsRepository = Repository; export type ChannelFollowingsRepository = Repository; export type ChannelFavoritesRepository = Repository; diff --git a/packages/backend/src/models/json-schema/auto-removal-condition.ts b/packages/backend/src/models/json-schema/auto-removal-condition.ts new file mode 100644 index 000000000..3db62e226 --- /dev/null +++ b/packages/backend/src/models/json-schema/auto-removal-condition.ts @@ -0,0 +1,28 @@ +/* + * SPDX-FileCopyrightText: syuilo and other misskey, cherrypick contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +export const packedAutoRemovalConditionSchema = { + type: 'object', + properties: { + id: { + type: 'string', + optional: false, nullable: false, + format: 'id', + example: 'xxxxxxxxxx', + }, + deleteAfter: { + type: 'number', + optional: true, nullable: false, + }, + noPiningNotes: { + type: 'boolean', + optional: true, nullable: false, + }, + noSpecifiedNotes: { + type: 'boolean', + optional: true, nullable: false, + }, + }, +} as const; diff --git a/packages/backend/src/postgres.ts b/packages/backend/src/postgres.ts index dfa5d86e1..68eaa96c7 100644 --- a/packages/backend/src/postgres.ts +++ b/packages/backend/src/postgres.ts @@ -83,6 +83,7 @@ import { MiFlashLike } from '@/models/FlashLike.js'; import { MiUserMemo } from '@/models/UserMemo.js'; import { MiBubbleGameRecord } from '@/models/BubbleGameRecord.js'; import { MiReversiGame } from '@/models/ReversiGame.js'; +import { MiAutoRemovalCondition } from '@/models/AutoRemovalCondition.js'; import { Config } from '@/config.js'; import { bindThis } from '@/decorators.js'; @@ -205,6 +206,7 @@ export const entities = [ MiUserMemo, MiBubbleGameRecord, MiReversiGame, + MiAutoRemovalCondition, ...charts, ]; diff --git a/packages/backend/src/queue/processors/AutoNoteRemovalProcessorService.ts b/packages/backend/src/queue/processors/AutoNoteRemovalProcessorService.ts new file mode 100644 index 000000000..898a1ce60 --- /dev/null +++ b/packages/backend/src/queue/processors/AutoNoteRemovalProcessorService.ts @@ -0,0 +1,106 @@ +/* + * SPDX-FileCopyrightText: syuilo and other misskey, cherrypick contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import { Inject, Injectable } from '@nestjs/common'; +import { And, In, MoreThan, Not } from 'typeorm'; +import { DI } from '@/di-symbols.js'; +import type { MiUserNotePining, NotesRepository, UserNotePiningsRepository, UsersRepository } from '@/models/_.js'; +import type Logger from '@/logger.js'; +import type { MiNote } from '@/models/Note.js'; +import { bindThis } from '@/decorators.js'; +import { NoteDeleteService } from '@/core/NoteDeleteService.js'; +import { IdService } from '@/core/IdService.js'; +import { QueueLoggerService } from '../QueueLoggerService.js'; + +@Injectable() +export class AutoNoteRemovalProcessorService { + private logger: Logger; + + constructor( + @Inject(DI.notesRepository) + private notesRepository: NotesRepository, + private usersRepository: UsersRepository, + private userNotePiningsRepository: UserNotePiningsRepository, + + private idService: IdService, + private noteDeleteService: NoteDeleteService, + private queueLoggerService: QueueLoggerService, + ) { + this.logger = this.queueLoggerService.logger.createSubLogger('auto-note-removal'); + } + + @bindThis + public async process(): Promise { + this.logger.info('Checking notes that to remove automatically...'); + this.logger.info('Checking users that enabled note auto-removal'); + const users = await this.usersRepository.find({ where: { autoRemoval: true } }); + if (users.length < 1) { + this.logger.info('Does not have any user that enabled autoRemoval'); + return; + } + const now = Date.now(); + + for (const user of users) { + if (user.autoRemovalCondition === null) { + continue; + } + const pinings: MiUserNotePining[] = await this.userNotePiningsRepository.findBy({ userId: user.id }); + const piningNoteIds: string[] = pinings.map(pining => pining.noteId); // pining.note always undefined (bug?) + + const specifiedNotes: MiNote[] = await this.notesRepository.findBy({ + userId: user.id, + visibility: Not(In(['public', 'home', 'followers'])), + }); + const specifiedNoteIds: string[] = specifiedNotes.map(note => note.id); + const deleteAfter: number = user.autoRemovalCondition.deleteAfter * 86400000; + + // Delete notes + let cursor: MiNote['id'] | null = null; + let condition: string[] = []; + if (user.autoRemovalCondition.noSpecifiedNotes === true) { + condition = [...condition, ...specifiedNoteIds]; + } + + if (user.autoRemovalCondition.noPiningNotes === true) { + condition = [...condition, ...piningNoteIds]; + } + + while (true) { + const notes = await this.notesRepository.find({ + where: { + userId: user.id, + ...(cursor ? { + id: And(Not(In(condition)), MoreThan(cursor)), + } : { + id: Not(In(condition)), + }), + }, + take: 100, + order: { + id: 1, + }, + }) as MiNote[]; + + if (notes.length === 0) { + break; + } + + cursor = notes.at(-1)?.id ?? null; + + for (const note of notes) { + const createdAt: number = this.idService.parse(note.id).date.getTime(); + const delta: number = now - createdAt; + if (delta > deleteAfter) { + await Promise.bind(this.noteDeleteService.delete(user, note, false, user)); + } + } + } + + this.logger.succ('All of auto-removable notes deleted'); + } + + this.logger.succ('All notes to auto-remove has beed removed.'); + } +} diff --git a/packages/backend/src/server/api/endpoints/i/update.ts b/packages/backend/src/server/api/endpoints/i/update.ts index c0a354982..f04fcf46c 100644 --- a/packages/backend/src/server/api/endpoints/i/update.ts +++ b/packages/backend/src/server/api/endpoints/i/update.ts @@ -11,7 +11,7 @@ import { JSDOM } from 'jsdom'; import { extractCustomEmojisFromMfm } from '@/misc/extract-custom-emojis-from-mfm.js'; import { extractHashtags } from '@/misc/extract-hashtags.js'; import * as Acct from '@/misc/acct.js'; -import type { UsersRepository, DriveFilesRepository, UserProfilesRepository, PagesRepository } from '@/models/_.js'; +import type { UsersRepository, DriveFilesRepository, UserProfilesRepository, PagesRepository, AutoRemovalConditionRepository } from '@/models/_.js'; import type { MiLocalUser, MiUser } from '@/models/User.js'; import { birthdaySchema, descriptionSchema, locationSchema, nameSchema } from '@/models/User.js'; import type { MiUserProfile } from '@/models/UserProfile.js'; @@ -112,7 +112,7 @@ export const meta = { }, uriNull: { - message: 'User ActivityPup URI is null.', + message: 'User ActivityPub URI is null.', code: 'URI_NULL', id: 'bf326f31-d430-4f97-9933-5d61e4d48a23', }, @@ -257,6 +257,8 @@ export const paramDef = { required: ['mutualLinks'], }, }, + autoRemoval: { type: 'boolean' }, + autoRemovalCondition: { type: 'object' }, }, } as const; @@ -278,6 +280,9 @@ export default class extends Endpoint { // eslint- @Inject(DI.pagesRepository) private pagesRepository: PagesRepository, + @Inject(DI.autoRemovalConditionRepository) + private autoRemovalConditionRepository: AutoRemovalConditionRepository, + private idService: IdService, private userEntityService: UserEntityService, private driveFileEntityService: DriveFileEntityService, @@ -345,6 +350,7 @@ export default class extends Endpoint { // eslint- if (typeof ps.isVacation === 'boolean') updates.isVacation = ps.isVacation; if (typeof ps.injectFeaturedNote === 'boolean') profileUpdates.injectFeaturedNote = ps.injectFeaturedNote; if (typeof ps.receiveAnnouncementEmail === 'boolean') profileUpdates.receiveAnnouncementEmail = ps.receiveAnnouncementEmail; + if (typeof ps.autoRemoval === 'boolean') updates.autoRemoval = ps.autoRemoval; if (typeof ps.alwaysMarkNsfw === 'boolean') { if (policy.alwaysMarkNsfw) throw new ApiError(meta.errors.restrictedByRole); profileUpdates.alwaysMarkNsfw = ps.alwaysMarkNsfw; @@ -521,6 +527,10 @@ export default class extends Endpoint { // eslint- this.globalEventService.publishInternalEvent('localUserUpdated', { id: user.id }); } + if (ps.autoRemovalCondition !== undefined) { + await this.autoRemovalConditionRepository.update(user.autoRemovalConditionId, ps.autoRemovalCondition); + } + await this.userProfilesRepository.update(user.id, { ...profileUpdates, verifiedLinks: [],