mirror/iceshrimp
1
0
Fork 0
mirror of https://iceshrimp.dev/iceshrimp/iceshrimp synced 2024-11-24 07:06:07 +09:00
Code Issues Actions Packages Projects Releases Wiki Activity

feat: Log user ips (#8872)

Browse Source 
* wip

* store ip and headers

* Update admin-file.vue

* require admin for view ip/headers

* IP (recent) 消した

* admin必須

* opt in

* clean ips periodically

* respect logging setting in drive/files/create
This commit is contained in:
syuilo 2022-07-02 15:12:11 +09:00 committed by GitHub
parent c8c7d10348
commit e23e7de453
29 changed files with 371 additions and 73 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
locales/ja-JP.yml
View File

@ -854,6 +854,7 @@ noEmailServerWarning: "メールサーバーの設定がされていません。
thereIsUnresolvedAbuseReportWarning: "未対応の通報があります。"
recommended: "推奨"
check: "チェック"
requireAdminForView: "閲覧するには管理者アカウントでログインしている必要があります。"
isSystemAccount: "システムにより自動で作成・管理されているアカウントです。"
typeToConfirm: "この操作を行うには {x} と入力してください"
deleteAccount: "アカウント削除"

17
packages/backend/migration/1655918165614-user-ip.js Normal file
View File

@ -0,0 +1,17 @@
export class userIp1655918165614 {
name = 'userIp1655918165614'
async up(queryRunner) {
await queryRunner.query(`CREATE TABLE "user_ip" ("id" SERIAL NOT NULL, "createdAt" TIMESTAMP WITH TIME ZONE NOT NULL, "userId" character varying(32) NOT NULL, "ip" character varying(128) NOT NULL, CONSTRAINT "PK_2c44ddfbf7c0464d028dcef325e" PRIMARY KEY ("id"))`);
await queryRunner.query(`CREATE INDEX "IDX_7f7f1c66f48e9a8e18a33bc515" ON "user_ip" ("userId") `);
await queryRunner.query(`CREATE UNIQUE INDEX "IDX_361b500e06721013c124b7b6c5" ON "user_ip" ("userId", "ip") `);
await queryRunner.query(`ALTER TABLE "user_ip" ADD CONSTRAINT "FK_7f7f1c66f48e9a8e18a33bc5150" FOREIGN KEY ("userId") REFERENCES "user"("id") ON DELETE NO ACTION ON UPDATE NO ACTION`);
}
async down(queryRunner) {
await queryRunner.query(`ALTER TABLE "user_ip" DROP CONSTRAINT "FK_7f7f1c66f48e9a8e18a33bc5150"`);
await queryRunner.query(`DROP INDEX "public"."IDX_361b500e06721013c124b7b6c5"`);
await queryRunner.query(`DROP INDEX "public"."IDX_7f7f1c66f48e9a8e18a33bc515"`);
await queryRunner.query(`DROP TABLE "user_ip"`);
}
}

13
packages/backend/migration/1656122560740-file-ip.js Normal file
View File

@ -0,0 +1,13 @@
export class fileIp1656122560740 {
name = 'fileIp1656122560740'
async up(queryRunner) {
await queryRunner.query(`ALTER TABLE "drive_file" ADD "requestHeaders" jsonb DEFAULT '{}'`);
await queryRunner.query(`ALTER TABLE "drive_file" ADD "requestIp" character varying(128)`);
}
async down(queryRunner) {
await queryRunner.query(`ALTER TABLE "drive_file" DROP COLUMN "requestIp"`);
await queryRunner.query(`ALTER TABLE "drive_file" DROP COLUMN "requestHeaders"`);
}
}

13
packages/backend/migration/1656328812281-ip-2.js Normal file
View File

@ -0,0 +1,13 @@
export class ip21656328812281 {
name = 'ip21656328812281'
async up(queryRunner) {
await queryRunner.query(`ALTER TABLE "user_ip" DROP CONSTRAINT "FK_7f7f1c66f48e9a8e18a33bc5150"`);
await queryRunner.query(`ALTER TABLE "meta" ADD "enableIpLogging" boolean NOT NULL DEFAULT false`);
}
async down(queryRunner) {
await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "enableIpLogging"`);
await queryRunner.query(`ALTER TABLE "user_ip" ADD CONSTRAINT "FK_7f7f1c66f48e9a8e18a33bc5150" FOREIGN KEY ("userId") REFERENCES "user"("id") ON DELETE NO ACTION ON UPDATE NO ACTION`);
}
}

4
packages/backend/src/db/postgre.ts
View File

@ -68,9 +68,10 @@ import { RegistryItem } from '@/models/entities/registry-item.js';
import { Ad } from '@/models/entities/ad.js';
import { PasswordResetRequest } from '@/models/entities/password-reset-request.js';
import { UserPending } from '@/models/entities/user-pending.js';
import { Webhook } from '@/models/entities/webhook.js';
import { UserIp } from '@/models/entities/user-ip.js';
import { entities as charts } from '@/services/chart/entities.js';
import { Webhook } from '@/models/entities/webhook.js';
import { envOption } from '../env.js';
import { dbLogger } from './logger.js';
import { redisClient } from './redis.js';
@ -173,6 +174,7 @@ export const entities = [
PasswordResetRequest,
UserPending,
Webhook,
UserIp,
...charts,
];

13
packages/backend/src/models/entities/drive-file.ts
View File

@ -1,7 +1,7 @@
import { PrimaryColumn, Entity, Index, JoinColumn, Column, ManyToOne } from 'typeorm';
import { id } from '../id.js';
import { User } from './user.js';
import { DriveFolder } from './drive-folder.js';
import { id } from '../id.js';
@Entity()
@Index(['userId', 'folderId', 'id'])
@ -165,4 +165,15 @@ export class DriveFile {
comment: 'Whether the DriveFile is direct link to remote server.',
})
public isLink: boolean;
@Column('jsonb', {
default: {},
nullable: true,
})
public requestHeaders: Record<string, string> | null;
@Column('varchar', {
length: 128, nullable: true,
})
public requestIp: string | null;
}

7
packages/backend/src/models/entities/meta.ts
View File

@ -1,6 +1,6 @@
import { Entity, Column, PrimaryColumn, ManyToOne, JoinColumn } from 'typeorm';
import { User } from './user.js';
import { id } from '../id.js';
import { User } from './user.js';
import { Clip } from './clip.js';
@Entity()
@ -427,4 +427,9 @@ export class Meta {
default: true,
})
public objectStorageS3ForcePathStyle: boolean;
@Column('boolean', {
default: false,
})
public enableIpLogging: boolean;
}

24
packages/backend/src/models/entities/user-ip.ts Normal file
View File

@ -0,0 +1,24 @@
import { PrimaryColumn, Entity, Index, JoinColumn, Column, ManyToOne, PrimaryGeneratedColumn } from 'typeorm';
import { id } from '../id.js';
import { Note } from './note.js';
import { User } from './user.js';
@Entity()
@Index(['userId', 'ip'], { unique: true })
export class UserIp {
@PrimaryGeneratedColumn()
public id: string;
@Column('timestamp with time zone', {
})
public createdAt: Date;
@Index()
@Column(id())
public userId: User['id'];
@Column('varchar', {
length: 128,
})
public ip: string;
}

2
packages/backend/src/models/index.ts
View File

@ -65,6 +65,7 @@ import { PasswordResetRequest } from './entities/password-reset-request.js';
import { UserPending } from './entities/user-pending.js';
import { InstanceRepository } from './repositories/instance.js';
import { Webhook } from './entities/webhook.js';
import { UserIp } from './entities/user-ip.js';
export const Announcements = db.getRepository(Announcement);
export const AnnouncementReads = db.getRepository(AnnouncementRead);
@ -90,6 +91,7 @@ export const UserGroups = (UserGroupRepository);
export const UserGroupJoinings = db.getRepository(UserGroupJoining);
export const UserGroupInvitations = (UserGroupInvitationRepository);
export const UserNotePinings = db.getRepository(UserNotePining);
export const UserIps = db.getRepository(UserIp);
export const UsedUsernames = db.getRepository(UsedUsername);
export const Followings = (FollowingRepository);
export const FollowRequests = (FollowRequestRepository);

18
packages/backend/src/queue/index.ts
View File

@ -2,6 +2,9 @@ import httpSignature from '@peertube/http-signature';
import { v4 as uuid } from 'uuid';
import config from '@/config/index.js';
import { DriveFile } from '@/models/entities/drive-file.js';
import { IActivity } from '@/remote/activitypub/type.js';
import { Webhook, webhookEventTypes } from '@/models/entities/webhook.js';
import { envOption } from '../env.js';
import processDeliver from './processors/deliver.js';
@ -12,18 +15,15 @@ import processSystemQueue from './processors/system/index.js';
import processWebhookDeliver from './processors/webhook-deliver.js';
import { endedPollNotification } from './processors/ended-poll-notification.js';
import { queueLogger } from './logger.js';
import { DriveFile } from '@/models/entities/drive-file.js';
import { getJobInfo } from './get-job-info.js';
import { systemQueue, dbQueue, deliverQueue, inboxQueue, objectStorageQueue, endedPollNotificationQueue, webhookDeliverQueue } from './queues.js';
import { ThinUser } from './types.js';
import { IActivity } from '@/remote/activitypub/type.js';
import { Webhook, webhookEventTypes } from '@/models/entities/webhook.js';
function renderError(e: Error): any {
return {
stack: e?.stack,
message: e?.message,
name: e?.name,
stack: e.stack,
message: e.message,
name: e.name,
};
}
@ -314,6 +314,12 @@ export default function() {
removeOnComplete: true,
});
systemQueue.add('clean', {
}, {
repeat: { cron: '0 0 * * *' },
removeOnComplete: true,
});
systemQueue.add('checkExpiredMutings', {
}, {
repeat: { cron: '*/5 * * * *' },

18
packages/backend/src/queue/processors/system/clean.ts Normal file
View File

@ -0,0 +1,18 @@
import Bull from 'bull';
import { LessThan } from 'typeorm';
import { UserIps } from '@/models/index.js';
import { queueLogger } from '../../logger.js';
const logger = queueLogger.createSubLogger('clean');
export async function clean(job: Bull.Job<Record<string, unknown>>, done: any): Promise<void> {
logger.info('Cleaning...');
UserIps.delete({
createdAt: LessThan(new Date(Date.now() - (1000 * 60 * 60 * 24 * 90))),
});
logger.succ('Cleaned.');
done();
}

2
packages/backend/src/queue/processors/system/index.ts
View File

@ -3,12 +3,14 @@ import { tickCharts } from './tick-charts.js';
import { resyncCharts } from './resync-charts.js';
import { cleanCharts } from './clean-charts.js';
import { checkExpiredMutings } from './check-expired-mutings.js';
import { clean } from './clean.js';
const jobs = {
tickCharts,
resyncCharts,
cleanCharts,
checkExpiredMutings,
clean,
} as Record<string, Bull.ProcessCallbackFunction<Record<string, unknown>> | Bull.ProcessPromiseFunction<Record<string, unknown>>>;
export default function(dbQueue: Bull.Queue<Record<string, unknown>>) {

34
packages/backend/src/server/api/api-handler.ts
View File

@ -1,10 +1,19 @@
import Koa from 'koa';
import { User } from '@/models/entities/user.js';
import { UserIps } from '@/models/index.js';
import { fetchMeta } from '@/misc/fetch-meta.js';
import { IEndpoint } from './endpoints.js';
import authenticate, { AuthenticationError } from './authenticate.js';
import call from './call.js';
import { ApiError } from './error.js';
const userIpHistories = new Map<User['id'], Set<string>>();
setInterval(() => {
userIpHistories.clear();
}, 1000 * 60 * 60);
export default (endpoint: IEndpoint, ctx: Koa.Context) => new Promise<void>((res) => {
const body = ctx.is('multipart/form-data')
? (ctx.request as any).body
@ -44,6 +53,31 @@ export default (endpoint: IEndpoint, ctx: Koa.Context) => new Promise<void>((res
}).catch((e: ApiError) => {
reply(e.httpStatusCode ? e.httpStatusCode : e.kind === 'client' ? 400 : 500, e);
});
// Log IP
if (user) {
fetchMeta().then(meta => {
if (!meta.enableIpLogging) return;
const ip = ctx.ip;
const ips = userIpHistories.get(user.id);
if (ips == null || !ips.has(ip)) {
if (ips == null) {
userIpHistories.set(user.id, new Set([ip]));
} else {
ips.add(ip);
}
try {
UserIps.insert({
createdAt: new Date(),
userId: user.id,
ip: ip,
});
} catch {
}
}
});
}
}).catch(e => {
if (e instanceof AuthenticationError) {
reply(403, new ApiError({

2
packages/backend/src/server/api/call.ts
View File

@ -116,7 +116,7 @@ export default async (endpoint: string, user: CacheableLocalUser | null | undefi
// API invoking
const before = performance.now();
return await ep.exec(data, user, token, ctx?.file).catch((e: Error) => {
return await ep.exec(data, user, token, ctx?.file, ctx?.ip, ctx?.headers).catch((e: Error) => {
if (e instanceof ApiError) {
throw e;
} else {

34
packages/backend/src/server/api/define.ts
View File

@ -1,16 +1,16 @@
import * as fs from 'node:fs';
import Ajv from 'ajv';
import { CacheableLocalUser, ILocalUser } from '@/models/entities/user.js';
import { IEndpointMeta } from './endpoints.js';
import { ApiError } from './error.js';
import { Schema, SchemaType } from '@/misc/schema.js';
import { AccessToken } from '@/models/entities/access-token.js';
import { IEndpointMeta } from './endpoints.js';
import { ApiError } from './error.js';
export type Response = Record<string, any> | void;
// TODO: paramsの型をT['params']のスキーマ定義から推論する
type executor<T extends IEndpointMeta, Ps extends Schema> =
(params: SchemaType<Ps>, user: T['requireCredential'] extends true ? CacheableLocalUser : CacheableLocalUser | null, token: AccessToken | null, file?: any, cleanup?: () => any) =>
(params: SchemaType<Ps>, user: T['requireCredential'] extends true ? CacheableLocalUser : CacheableLocalUser | null, token: AccessToken | null, file?: any, cleanup?: () => any, ip?: string | null, headers?: Record<string, string> | null) =>
Promise<T['res'] extends undefined ? Response : SchemaType<NonNullable<T['res']>>>;
const ajv = new Ajv({
@ -20,23 +20,27 @@ const ajv = new Ajv({
ajv.addFormat('misskey:id', /^[a-zA-Z0-9]+$/);
export default function <T extends IEndpointMeta, Ps extends Schema>(meta: T, paramDef: Ps, cb: executor<T, Ps>)
: (params: any, user: T['requireCredential'] extends true ? CacheableLocalUser : CacheableLocalUser | null, token: AccessToken | null, file?: any) => Promise<any> {
: (params: any, user: T['requireCredential'] extends true ? CacheableLocalUser : CacheableLocalUser | null, token: AccessToken | null, file?: any, ip?: string | null, headers?: Record<string, string> | null) => Promise<any> {
const validate = ajv.compile(paramDef);
return (params: any, user: T['requireCredential'] extends true ? CacheableLocalUser : CacheableLocalUser | null, token: AccessToken | null, file?: any) => {
function cleanup() {
fs.unlink(file.path, () => {});
}
return (params: any, user: T['requireCredential'] extends true ? CacheableLocalUser : CacheableLocalUser | null, token: AccessToken | null, file?: any, ip?: string | null, headers?: Record<string, string> | null) => {
let cleanup: undefined | (() => void) = undefined;
if (meta.requireFile && file == null) return Promise.reject(new ApiError({
message: 'File required.',
code: 'FILE_REQUIRED',
id: '4267801e-70d1-416a-b011-4ee502885d8b',
}));
if (meta.requireFile) {
cleanup = () => {
fs.unlink(file.path, () => {});
};
if (file == null) return Promise.reject(new ApiError({
message: 'File required.',
code: 'FILE_REQUIRED',
id: '4267801e-70d1-416a-b011-4ee502885d8b',
}));
}
const valid = validate(params);
if (!valid) {
if (file) cleanup();
if (file) cleanup!();
const errors = validate.errors!;
const err = new ApiError({
@ -50,6 +54,6 @@ export default function <T extends IEndpointMeta, Ps extends Schema>(meta: T, pa
return Promise.reject(err);
}
return cb(params as SchemaType<Ps>, user, token, file, cleanup);
return cb(params as SchemaType<Ps>, user, token, file, cleanup, ip, headers);
};
}

2
packages/backend/src/server/api/endpoints.ts
View File

@ -35,6 +35,7 @@ import * as ep___admin_federation_removeAllFollowing from './endpoints/admin/fed
import * as ep___admin_federation_updateInstance from './endpoints/admin/federation/update-instance.js';
import * as ep___admin_getIndexStats from './endpoints/admin/get-index-stats.js';
import * as ep___admin_getTableStats from './endpoints/admin/get-table-stats.js';
import * as ep___admin_getUserIps from './endpoints/admin/get-user-ips.js';
import * as ep___admin_invite from './endpoints/admin/invite.js';
import * as ep___admin_moderators_add from './endpoints/admin/moderators/add.js';
import * as ep___admin_moderators_remove from './endpoints/admin/moderators/remove.js';
@ -348,6 +349,7 @@ const eps = [
['admin/federation/update-instance', ep___admin_federation_updateInstance],
['admin/get-index-stats', ep___admin_getIndexStats],
['admin/get-table-stats', ep___admin_getTableStats],
['admin/get-user-ips', ep___admin_getUserIps],
['admin/invite', ep___admin_invite],
['admin/moderators/add', ep___admin_moderators_add],
['admin/moderators/remove', ep___admin_moderators_remove],

7
packages/backend/src/server/api/endpoints/admin/drive/show-file.ts
View File

@ -1,6 +1,6 @@
import { DriveFiles } from '@/models/index.js';
import define from '../../../define.js';
import { ApiError } from '../../../error.js';
import { DriveFiles } from '@/models/index.js';
export const meta = {
tags: ['admin'],
@ -184,5 +184,10 @@ export default define(meta, paramDef, async (ps, me) => {
throw new ApiError(meta.errors.noSuchFile);
}
if (!me.isAdmin) {
delete file.requestIp;
delete file.requestHeaders;
}
return file;
});

31
packages/backend/src/server/api/endpoints/admin/get-user-ips.ts Normal file
View File

@ -0,0 +1,31 @@
import { UserIps } from '@/models/index.js';
import define from '../../define.js';
export const meta = {
tags: ['admin'],
requireCredential: true,
requireAdmin: true,
} as const;
export const paramDef = {
type: 'object',
properties: {
userId: { type: 'string', format: 'misskey:id' },
},
required: ['userId'],
} as const;
// eslint-disable-next-line import/no-default-export
export default define(meta, paramDef, async (ps, me) => {
const ips = await UserIps.find({
where: { userId: ps.userId },
order: { createdAt: 'DESC' },
take: 30,
});
return ips.map(x => ({
ip: x.ip,
createdAt: x.createdAt.toISOString(),
}));
});

8
packages/backend/src/server/api/endpoints/admin/meta.ts
View File

@ -1,7 +1,7 @@
import config from '@/config/index.js';
import define from '../../define.js';
import { fetchMeta } from '@/misc/fetch-meta.js';
import { MAX_NOTE_TEXT_LENGTH } from '@/const.js';
import define from '../../define.js';
export const meta = {
tags: ['meta'],
@ -304,6 +304,10 @@ export const meta = {
type: 'boolean',
optional: true, nullable: false,
},
enableIpLogging: {
type: 'boolean',
optional: true, nullable: false,
},
},
},
} as const;
@ -360,7 +364,6 @@ export default define(meta, paramDef, async (ps, me) => {
pinnedPages: instance.pinnedPages,
pinnedClipId: instance.pinnedClipId,
cacheRemoteFiles: instance.cacheRemoteFiles,
useStarForReactionFallback: instance.useStarForReactionFallback,
pinnedUsers: instance.pinnedUsers,
hiddenTags: instance.hiddenTags,
@ -397,5 +400,6 @@ export default define(meta, paramDef, async (ps, me) => {
objectStorageS3ForcePathStyle: instance.objectStorageS3ForcePathStyle,
deeplAuthKey: instance.deeplAuthKey,
deeplIsPro: instance.deeplIsPro,
enableIpLogging: instance.enableIpLogging,
};
});

7
packages/backend/src/server/api/endpoints/admin/update-meta.ts
View File

@ -1,8 +1,8 @@
import define from '../../define.js';
import { Meta } from '@/models/entities/meta.js';
import { insertModerationLog } from '@/services/insert-moderation-log.js';
import { DB_MAX_NOTE_TEXT_LENGTH } from '@/misc/hard-limits.js';
import { db } from '@/db/postgre.js';
import define from '../../define.js';
export const meta = {
tags: ['admin'],
@ -96,6 +96,7 @@ export const paramDef = {
objectStorageUseProxy: { type: 'boolean' },
objectStorageSetPublicRead: { type: 'boolean' },
objectStorageS3ForcePathStyle: { type: 'boolean' },
enableIpLogging: { type: 'boolean' },
},
required: [],
} as const;
@ -396,6 +397,10 @@ export default define(meta, paramDef, async (ps, me) => {
set.deeplIsPro = ps.deeplIsPro;
}
if (ps.enableIpLogging !== undefined) {
set.enableIpLogging = ps.enableIpLogging;
}
await db.transaction(async transactionalEntityManager => {
const metas = await transactionalEntityManager.find(Meta, {
order: {

21
packages/backend/src/server/api/endpoints/drive/files/create.ts
View File

@ -1,10 +1,11 @@
import ms from 'ms';
import { addFile } from '@/services/drive/add-file.js';
import { DriveFiles } from '@/models/index.js';
import { DB_MAX_IMAGE_COMMENT_LENGTH } from '@/misc/hard-limits.js';
import { fetchMeta } from '@/misc/fetch-meta.js';
import define from '../../../define.js';
import { apiLogger } from '../../../logger.js';
import { ApiError } from '../../../error.js';
import { DriveFiles } from '@/models/index.js';
import { DB_MAX_IMAGE_COMMENT_LENGTH } from '@/misc/hard-limits.js';
export const meta = {
tags: ['drive'],
@ -50,7 +51,7 @@ export const paramDef = {
} as const;
// eslint-disable-next-line import/no-default-export
export default define(meta, paramDef, async (ps, user, _, file, cleanup) => {
export default define(meta, paramDef, async (ps, user, _, file, cleanup, ip, headers) => {
// Get 'name' parameter
let name = ps.name || file.originalname;
if (name !== undefined && name !== null) {
@ -66,9 +67,21 @@ export default define(meta, paramDef, async (ps, user, _, file, cleanup) => {
name = null;
}
const meta = await fetchMeta();
try {
// Create file
const driveFile = await addFile({ user, path: file.path, name, comment: ps.comment, folderId: ps.folderId, force: ps.force, sensitive: ps.isSensitive });
const driveFile = await addFile({
user,
path: file.path,
name,
comment: ps.comment,
folderId: ps.folderId,
force: ps.force,
sensitive: ps.isSensitive,
requestIp: meta.enableIpLogging ? ip : null,
requestHeaders: meta.enableIpLogging ? headers : null,
});
return await DriveFiles.pack(driveFile, { self: true });
} catch (e) {
if (e instanceof Error || typeof e === 'string') {

6
packages/backend/src/server/api/endpoints/drive/files/upload-from-url.ts
View File

@ -1,9 +1,9 @@
import ms from 'ms';
import { uploadFromUrl } from '@/services/drive/upload-from-url.js';
import define from '../../../define.js';
import { DriveFiles } from '@/models/index.js';
import { publishMainStream } from '@/services/stream.js';
import { DB_MAX_IMAGE_COMMENT_LENGTH } from '@/misc/hard-limits.js';
import define from '../../../define.js';
export const meta = {
tags: ['drive'],
@ -34,8 +34,8 @@ export const paramDef = {
} as const;
// eslint-disable-next-line import/no-default-export
export default define(meta, paramDef, async (ps, user) => {
uploadFromUrl({ url: ps.url, user, folderId: ps.folderId, sensitive: ps.isSensitive, force: ps.force, comment: ps.comment }).then(file => {
export default define(meta, paramDef, async (ps, user, _1, _2, _3, ip, headers) => {
uploadFromUrl({ url: ps.url, user, folderId: ps.folderId, sensitive: ps.isSensitive, force: ps.force, comment: ps.comment, requestIp: ip, requestHeaders: headers }).then(file => {
DriveFiles.pack(file, { self: true }).then(packedFile => {
publishMainStream(user.id, 'urlUploadFinished', {
marker: ps.marker,

51
packages/backend/src/services/drive/add-file.ts
View File

@ -2,26 +2,26 @@ import * as fs from 'node:fs';
import { v4 as uuid } from 'uuid';
import S3 from 'aws-sdk/clients/s3.js';
import sharp from 'sharp';
import { IsNull } from 'typeorm';
import { publishMainStream, publishDriveStream } from '@/services/stream.js';
import { deleteFile } from './delete-file.js';
import { fetchMeta } from '@/misc/fetch-meta.js';
import { GenerateVideoThumbnail } from './generate-video-thumbnail.js';
import { driveLogger } from './logger.js';
import { IImage, convertSharpToJpeg, convertSharpToWebp, convertSharpToPng } from './image-processor.js';
import { contentDisposition } from '@/misc/content-disposition.js';
import { getFileInfo } from '@/misc/get-file-info.js';
import { DriveFiles, DriveFolders, Users, Instances, UserProfiles } from '@/models/index.js';
import { InternalStorage } from './internal-storage.js';
import { DriveFile } from '@/models/entities/drive-file.js';
import { IRemoteUser, User } from '@/models/entities/user.js';
import { driveChart, perUserDriveChart, instanceChart } from '@/services/chart/index.js';
import { genId } from '@/misc/gen-id.js';
import { isDuplicateKeyValueError } from '@/misc/is-duplicate-key-value-error.js';
import S3 from 'aws-sdk/clients/s3.js';
import { getS3 } from './s3.js';
import sharp from 'sharp';
import { FILE_TYPE_BROWSERSAFE } from '@/const.js';
import { IsNull } from 'typeorm';
import { getS3 } from './s3.js';
import { InternalStorage } from './internal-storage.js';
import { IImage, convertSharpToJpeg, convertSharpToWebp, convertSharpToPng } from './image-processor.js';
import { driveLogger } from './logger.js';
import { GenerateVideoThumbnail } from './generate-video-thumbnail.js';
import { deleteFile } from './delete-file.js';
const logger = driveLogger.createSubLogger('register', 'yellow');
@ -171,7 +171,7 @@ export async function generateAlts(path: string, type: string, generateWeb: bool
}
if (!['image/jpeg', 'image/png', 'image/webp', 'image/svg+xml'].includes(type)) {
logger.debug(`web image and thumbnail not created (not an required file)`);
logger.debug('web image and thumbnail not created (not an required file)');
return {
webpublic: null,
thumbnail: null,
@ -212,7 +212,7 @@ export async function generateAlts(path: string, type: string, generateWeb: bool
let webpublic: IImage | null = null;
if (generateWeb && !satisfyWebpublic) {
logger.info(`creating web image`);
logger.info('creating web image');
try {
if (['image/jpeg', 'image/webp'].includes(type)) {
@ -222,14 +222,14 @@ export async function generateAlts(path: string, type: string, generateWeb: bool
} else if (['image/svg+xml'].includes(type)) {
webpublic = await convertSharpToPng(img, 2048, 2048);
} else {
logger.debug(`web image not created (not an required image)`);
logger.debug('web image not created (not an required image)');
}
} catch (err) {
logger.warn(`web image not created (an error occured)`, err as Error);
logger.warn('web image not created (an error occured)', err as Error);
}
} else {
if (satisfyWebpublic) logger.info(`web image not created (original satisfies webpublic)`);
else logger.info(`web image not created (from remote)`);
if (satisfyWebpublic) logger.info('web image not created (original satisfies webpublic)');
else logger.info('web image not created (from remote)');
}
// #endregion webpublic
@ -240,10 +240,10 @@ export async function generateAlts(path: string, type: string, generateWeb: bool
if (['image/jpeg', 'image/webp', 'image/png', 'image/svg+xml'].includes(type)) {
thumbnail = await convertSharpToWebp(img, 498, 280);
} else {
logger.debug(`thumbnail not created (not an required file)`);
logger.debug('thumbnail not created (not an required file)');
}
} catch (err) {
logger.warn(`thumbnail not created (an error occured)`, err as Error);
logger.warn('thumbnail not created (an error occured)', err as Error);
}
// #endregion thumbnail
@ -276,7 +276,7 @@ async function upload(key: string, stream: fs.ReadStream | Buffer, type: string,
const s3 = getS3(meta);
const upload = s3.upload(params, {
partSize: s3.endpoint?.hostname === 'storage.googleapis.com' ? 500 * 1024 * 1024 : 8 * 1024 * 1024,
partSize: s3.endpoint.hostname === 'storage.googleapis.com' ? 500 * 1024 * 1024 : 8 * 1024 * 1024,
});
const result = await upload.promise();
@ -326,6 +326,9 @@ type AddFileArgs = {
uri?: string | null;
/** Mark file as sensitive */
sensitive?: boolean | null;
requestIp?: string | null;
requestHeaders?: Record<string, string> | null;
};
/**
@ -342,7 +345,9 @@ export async function addFile({
isLink = false,
url = null,
uri = null,
sensitive = null
sensitive = null,
requestIp = null,
requestHeaders = null,
}: AddFileArgs): Promise<DriveFile> {
const info = await getFileInfo(path);
logger.info(`${JSON.stringify(info)}`);
@ -427,11 +432,13 @@ export async function addFile({
file.properties = properties;
file.blurhash = info.blurhash || null;
file.isLink = isLink;
file.requestIp = requestIp;
file.requestHeaders = requestHeaders;
file.isSensitive = user
? Users.isLocalUser(user) && profile!.alwaysMarkNsfw ? true :
(sensitive !== null && sensitive !== undefined)
? sensitive
: false
(sensitive !== null && sensitive !== undefined)
? sensitive
: false
: false;
if (url !== null) {

10
packages/backend/src/services/drive/upload-from-url.ts
View File

@ -1,12 +1,12 @@
import { URL } from 'node:url';
import { addFile } from './add-file.js';
import { User } from '@/models/entities/user.js';
import { driveLogger } from './logger.js';
import { createTemp } from '@/misc/create-temp.js';
import { downloadUrl } from '@/misc/download-url.js';
import { DriveFolder } from '@/models/entities/drive-folder.js';
import { DriveFile } from '@/models/entities/drive-file.js';
import { DriveFiles } from '@/models/index.js';
import { driveLogger } from './logger.js';
import { addFile } from './add-file.js';
const logger = driveLogger.createSubLogger('downloader');
@ -19,6 +19,8 @@ type Args = {
force?: boolean;
isLink?: boolean;
comment?: string | null;
requestIp?: string | null;
requestHeaders?: Record<string, string> | null;
};
export async function uploadFromUrl({
@ -30,6 +32,8 @@ export async function uploadFromUrl({
force = false,
isLink = false,
comment = null,
requestIp = null,
requestHeaders = null,
}: Args): Promise<DriveFile> {
let name = new URL(url).pathname.split('/').pop() || null;
if (name == null || !DriveFiles.validateFileName(name)) {
@ -49,7 +53,7 @@ export async function uploadFromUrl({
// write content at URL to temp file
await downloadUrl(url, path);
const driveFile = await addFile({ user, path, name, comment, folderId, force, isLink, url, uri, sensitive });
const driveFile = await addFile({ user, path, name, comment, folderId, force, isLink, url, uri, sensitive, requestIp, requestHeaders });
logger.succ(`Got: ${driveFile.id}`);
return driveFile!;
} catch (e) {

1
packages/client/src/account.ts
View File

@ -17,6 +17,7 @@ const accountData = localStorage.getItem('account');
export const $i = accountData ? reactive(JSON.parse(accountData) as Account) : null;
export const iAmModerator = $i != null && ($i.isAdmin || $i.isModerator);
export const iAmAdmin = $i != null && $i.isAdmin;
export async function signout() {
waiting();

26
packages/client/src/pages/admin-file.vue
View File

@ -1,7 +1,7 @@
<template>
<MkStickyContainer>
<template #header><MkPageHeader v-model:tab="tab" :actions="headerActions" :tabs="headerTabs"/></template>
<MkSpacer v-if="file" :content-max="500" :margin-min="16" :margin-max="32">
<MkSpacer v-if="file" :content-max="600" :margin-min="16" :margin-max="32">
<div v-if="tab === 'overview'" class="cxqhhsmd _formRoot">
<a class="_formBlock thumbnail" :href="file.url" target="_blank">
<MkDriveFileThumbnail class="thumbnail" :file="file" fit="contain"/>
@ -39,6 +39,20 @@
<MkButton danger @click="del"><i class="fas fa-trash-alt"></i> {{ i18n.ts.delete }}</MkButton>
</div>
</div>
<div v-else-if="tab === 'ip' && info" class="_formRoot">
<MkInfo v-if="!iAmAdmin" warn>{{ i18n.ts.requireAdminForView }}</MkInfo>
<MkKeyValue v-if="info.requestIp" class="_formBlock _monospace" :copy="info.requestIp" oneline>
<template #key>IP</template>
<template #value>{{ info.requestIp }}</template>
</MkKeyValue>
<FormSection v-if="info.requestHeaders">
<template #label>Headers</template>
<MkKeyValue v-for="(v, k) in info.requestHeaders" :key="k" class="_formBlock _monospace">
<template #key>{{ k }}</template>
<template #value>{{ v }}</template>
</MkKeyValue>
</FormSection>
</div>
<div v-else-if="tab === 'raw'" class="_formRoot">
<MkObjectView v-if="info" tall :value="info">
</MkObjectView>
@ -54,13 +68,15 @@ import MkSwitch from '@/components/form/switch.vue';
import MkObjectView from '@/components/object-view.vue';
import MkDriveFileThumbnail from '@/components/drive-file-thumbnail.vue';
import MkKeyValue from '@/components/key-value.vue';
import FormLink from '@/components/form/link.vue';
import FormSection from '@/components/form/section.vue';
import MkUserCardMini from '@/components/user-card-mini.vue';
import MkInfo from '@/components/ui/info.vue';
import bytes from '@/filters/bytes';
import * as os from '@/os';
import { i18n } from '@/i18n';
import { definePageMetadata } from '@/scripts/page-metadata';
import { acct } from '@/filters/user';
import { iAmAdmin, iAmModerator } from '@/account';
let tab = $ref('overview');
let file: any = $ref(null);
@ -108,7 +124,11 @@ const headerTabs = $computed(() => [{
key: 'overview',
title: i18n.ts.overview,
icon: 'fas fa-info-circle',
}, {
}, iAmModerator ? {
key: 'ip',
title: 'IP',
icon: 'fas fa-bars-staggered',
} : null, {
key: 'raw',
title: 'Raw data',
icon: 'fas fa-code',

15
packages/client/src/pages/admin/security.vue
View File

@ -14,6 +14,18 @@
<XBotProtection/>
</FormFolder>
<FormFolder class="_formBlock">
<template #label>Log IP address</template>
<template v-if="enableIpLogging" #suffix>Enabled</template>
<template v-else #suffix>Disabled</template>
<div class="_formRoot">
<FormSwitch v-model="enableIpLogging" class="_formBlock" @update:modelValue="save">
<template #label>Enable</template>
</FormSwitch>
</div>
</FormFolder>
<FormFolder class="_formBlock">
<template #label>Summaly Proxy</template>
@ -51,17 +63,20 @@ import { definePageMetadata } from '@/scripts/page-metadata';
let summalyProxy: string = $ref('');
let enableHcaptcha: boolean = $ref(false);
let enableRecaptcha: boolean = $ref(false);
let enableIpLogging: boolean = $ref(false);
async function init() {
const meta = await os.api('admin/meta');
summalyProxy = meta.summalyProxy;
enableHcaptcha = meta.enableHcaptcha;
enableRecaptcha = meta.enableRecaptcha;
enableIpLogging = meta.enableIpLogging;
}
function save() {
os.apiWithDialog('admin/update-meta', {
summalyProxy,
enableIpLogging,
}).then(() => {
fetchInstance();
});

47
packages/client/src/pages/user-info.vue
View File

@ -27,6 +27,12 @@
<template #key>ID</template>
<template #value><span class="_monospace">{{ user.id }}</span></template>
</MkKeyValue>
<!-- 要る？
<MkKeyValue v-if="ips.length > 0" :copy="user.id" oneline style="margin: 1em 0;">
<template #key>IP (recent)</template>
<template #value><span class="_monospace">{{ ips[0].ip }}</span></template>
</MkKeyValue>
-->
<MkKeyValue oneline style="margin: 1em 0;">
<template #key>{{ i18n.ts.createdAt }}</template>
<template #value><span class="_monospace"><MkTime :time="user.createdAt" :mode="'detail'"/></span></template>
@ -92,8 +98,18 @@
<div v-else-if="tab === 'files'" class="_formRoot">
<MkFileListForAdmin :pagination="filesPagination" view-mode="grid"/>
</div>
<div v-else-if="tab === 'ip'" class="_formRoot">
<MkInfo v-if="!iAmAdmin" warn>{{ i18n.ts.requireAdminForView }}</MkInfo>
<MkInfo v-else>The date is the IP address was first acknowledged.</MkInfo>
<template v-if="iAmAdmin && ips">
<div v-for="record in ips" :key="record.ip" class="_monospace" :class="$style.ip" style="margin: 1em 0;">
<span class="date">{{ record.createdAt }}</span>
<span class="ip">{{ record.ip }}</span>
</div>
</template>
</div>
<div v-else-if="tab === 'ap'" class="_formRoot">
<MkObjectView v-if="ap" tall :value="user">
<MkObjectView v-if="ap" tall :value="ap">
</MkObjectView>
</div>
<div v-else-if="tab === 'raw'" class="_formRoot">
@ -122,6 +138,7 @@ import MkKeyValue from '@/components/key-value.vue';
import MkSelect from '@/components/form/select.vue';
import FormSuspense from '@/components/form/suspense.vue';
import MkFileListForAdmin from '@/components/file-list-for-admin.vue';
import MkInfo from '@/components/ui/info.vue';
import * as os from '@/os';
import number from '@/filters/number';
import bytes from '@/filters/bytes';
@ -129,7 +146,7 @@ import { url } from '@/config';
import { userPage, acct } from '@/filters/user';
import { definePageMetadata } from '@/scripts/page-metadata';
import { i18n } from '@/i18n';
import { iAmModerator } from '@/account';
import { iAmAdmin, iAmModerator } from '@/account';
const props = defineProps<{
userId: string;
@ -140,6 +157,7 @@ let chartSrc = $ref('per-user-notes');
let user = $ref<null | misskey.entities.UserDetailed>();
let init = $ref();
let info = $ref();
let ips = $ref(null);
let ap = $ref(null);
let moderator = $ref(false);
let silenced = $ref(false);
@ -158,9 +176,12 @@ function createFetcher() {
userId: props.userId,
}), os.api('admin/show-user', {
userId: props.userId,
})]).then(([_user, _info]) => {
}), iAmAdmin ? os.api('admin/get-user-ips', {
userId: props.userId,
}) : Promise.resolve(null)]).then(([_user, _info, _ips]) => {
user = _user;
info = _info;
ips = _ips;
moderator = info.isModerator;
silenced = info.isSilenced;
suspended = info.isSuspended;
@ -300,7 +321,11 @@ const headerTabs = $computed(() => [{
key: 'ap',
title: 'AP',
icon: 'fas fa-share-alt',
}, {
}, iAmModerator ? {
key: 'ip',
title: 'IP',
icon: 'fas fa-bars-staggered',
} : null, {
key: 'raw',
title: 'Raw',
icon: 'fas fa-code',
@ -362,3 +387,17 @@ definePageMetadata(computed(() => ({
}
}
</style>
<style lang="scss" module>
.ip {
display: flex;
> :global(.date) {
opacity: 0.7;
}
> :global(.ip) {
margin-left: auto;
}
}
</style>

10
packages/client/src/scripts/upload.ts
View File

@ -1,9 +1,9 @@
import { reactive, ref } from 'vue';
import * as Misskey from 'misskey-js';
import { readAndCompressImage } from 'browser-image-resizer';
import { defaultStore } from '@/store';
import { apiUrl } from '@/config';
import * as Misskey from 'misskey-js';
import { $i } from '@/account';
import { readAndCompressImage } from 'browser-image-resizer';
import { alert } from '@/os';
type Uploading = {
@ -31,7 +31,7 @@ export function uploadFile(
file: File,
folder?: any,
name?: string,
keepOriginal: boolean = defaultStore.state.keepOriginalUploading
keepOriginal: boolean = defaultStore.state.keepOriginalUploading,
): Promise<Misskey.entities.DriveFile> {
if (folder && typeof folder === 'object') folder = folder.id;
@ -45,7 +45,7 @@ export function uploadFile(
name: name || file.name || 'untitled',
progressMax: undefined,
progressValue: undefined,
img: window.URL.createObjectURL(file)
img: window.URL.createObjectURL(file),
});
uploads.value.push(ctx);
@ -86,7 +86,7 @@ export function uploadFile(
alert({
type: 'error',
title: 'Failed to upload',
text: `${JSON.stringify(ev.target?.response)}, ${JSON.stringify(xhr.response)}`
text: `${JSON.stringify(ev.target?.response)}, ${JSON.stringify(xhr.response)}`,
});
reject();