[backend] Add prometheus metrics endpoint

.pnp.cjs

@@ -3012,6 +3012,15 @@ const RAW_RUNTIME_STATE =
         "linkType": "HARD"\
       }]\
     ]],\
+    ["@opentelemetry/api", [\
+      ["npm:1.7.0", {\
+        "packageLocation": "./.yarn/cache/@opentelemetry-api-npm-1.7.0-6263fad98a-bcf7afa705.zip/node_modules/@opentelemetry/api/",\
+        "packageDependencies": [\
+          ["@opentelemetry/api", "npm:1.7.0"]\
+        ],\
+        "linkType": "HARD"\
+      }]\
+    ]],\
     ["@paralleldrive/cuid2", [\
       ["npm:2.2.2", {\
         "packageLocation": "./.yarn/cache/@paralleldrive-cuid2-npm-2.2.2-e6061749b2-40ee269d6e.zip/node_modules/@paralleldrive/cuid2/",\
@@ -7299,6 +7308,7 @@ const RAW_RUNTIME_STATE =
           ["pg", "virtual:aa59773ac87791c4813d53447077fcf8a847d6de5a301d34dc31286584b1dbb26d30d3adb5b4c41c1e8aea04371e926fda05c09c6253647c432e11d872a304ba#npm:8.11.1"],\
           ["private-ip", "npm:2.3.4"],\
           ["probe-image-size", "npm:7.2.3"],\
+          ["prom-client", "npm:15.1.0"],\
           ["promise-limit", "npm:2.7.0"],\
           ["pug", "npm:3.0.2"],\
           ["punycode", "npm:2.3.0"],\
@@ -7474,6 +7484,15 @@ const RAW_RUNTIME_STATE =
         "linkType": "HARD"\
       }]\
     ]],\
+    ["bintrees", [\
+      ["npm:1.0.2", {\
+        "packageLocation": "./.yarn/cache/bintrees-npm-1.0.2-b28feeda03-071896cea5.zip/node_modules/bintrees/",\
+        "packageDependencies": [\
+          ["bintrees", "npm:1.0.2"]\
+        ],\
+        "linkType": "HARD"\
+      }]\
+    ]],\
     ["bl", [\
       ["npm:1.2.3", {\
         "packageLocation": "./.yarn/cache/bl-npm-1.2.3-49c4213ca5-11d775b09e.zip/node_modules/bl/",\
@@ -20443,6 +20462,17 @@ const RAW_RUNTIME_STATE =
         "linkType": "HARD"\
       }]\
     ]],\
+    ["prom-client", [\
+      ["npm:15.1.0", {\
+        "packageLocation": "./.yarn/cache/prom-client-npm-15.1.0-0b2231d02c-ecb6f40de7.zip/node_modules/prom-client/",\
+        "packageDependencies": [\
+          ["prom-client", "npm:15.1.0"],\
+          ["@opentelemetry/api", "npm:1.7.0"],\
+          ["tdigest", "npm:0.1.2"]\
+        ],\
+        "linkType": "HARD"\
+      }]\
+    ]],\
     ["promise", [\
       ["npm:7.3.1", {\
         "packageLocation": "./.yarn/cache/promise-npm-7.3.1-5d81d474c0-37dbe58ca7.zip/node_modules/promise/",\
@@ -23175,6 +23205,16 @@ const RAW_RUNTIME_STATE =
         "linkType": "HARD"\
       }]\
     ]],\
+    ["tdigest", [\
+      ["npm:0.1.2", {\
+        "packageLocation": "./.yarn/cache/tdigest-npm-0.1.2-b73cfcf726-45be99fa52.zip/node_modules/tdigest/",\
+        "packageDependencies": [\
+          ["tdigest", "npm:0.1.2"],\
+          ["bintrees", "npm:1.0.2"]\
+        ],\
+        "linkType": "HARD"\
+      }]\
+    ]],\
     ["terminal-link", [\
       ["npm:2.1.1", {\
         "packageLocation": "./.yarn/cache/terminal-link-npm-2.1.1-de80341758-ce3d2cd3a4.zip/node_modules/terminal-link/",\

packages/backend/package.json

@@ -103,6 +103,7 @@
 		"pg": "8.11.1",
 		"private-ip": "2.3.4",
 		"probe-image-size": "7.2.3",
+		"prom-client": "^15.1.0",
 		"promise-limit": "2.7.0",
 		"punycode": "2.3.0",
 		"pureimage": "0.3.15",

packages/backend/src/config/load.ts

@@ -71,6 +71,9 @@ export default function load() {
 
 	config.searchEngine = config.searchEngine ?? 'https://duckduckgo.com/?q=';
 
+	config.metrics = config.metrics ?? {};
+	config.metrics.enable = config.metrics?.enable ?? false;
+
 	mixin.version = meta.version;
 	mixin.host = url.host;
 	mixin.hostname = url.hostname;

packages/backend/src/config/types.ts

@@ -150,6 +150,10 @@ export type Source = {
 		s3ForcePathStyle?: boolean;
 	};
 	summalyProxyUrl?: string;
+	metrics?: {
+		enable?: boolean;
+		token?: string;
+	};
 };
 
 /**

packages/backend/src/metrics.ts

@@ -0,0 +1,125 @@
+import Router from "@koa/router";
+import {
+	collectDefaultMetrics,
+	register,
+	Gauge,
+	Counter,
+	CounterConfiguration,
+} from "prom-client";
+import config from "./config/index.js";
+import { queues } from "./queue/queues.js";
+import cluster from "node:cluster";
+import Xev from "xev";
+
+const xev = new Xev();
+
+if (config.metrics?.enable) {
+	if (cluster.isPrimary) {
+		collectDefaultMetrics();
+
+		new Gauge({
+			name: "iceshrimp_queue_jobs",
+			help: "Amount of jobs in the bull queues",
+			labelNames: ["queue", "status"] as const,
+			async collect() {
+				for (const queue of queues) {
+					const counts = await queue.getJobCounts();
+					this.set({ queue: queue.name, status: "completed" }, counts.completed);
+					this.set({ queue: queue.name, status: "waiting" }, counts.waiting);
+					this.set({ queue: queue.name, status: "active" }, counts.active);
+					this.set({ queue: queue.name, status: "delayed" }, counts.delayed);
+					this.set({ queue: queue.name, status: "failed" }, counts.failed);
+				}
+			},
+		});
+	}
+}
+
+if (cluster.isPrimary) {
+	xev.on("registry-request", async () => {
+		try {
+			const metrics = await register.metrics();
+			xev.emit("registry-response", {
+				contentType: register.contentType,
+				body: metrics
+			});
+		} catch (error) {
+			xev.emit("registry-response", { error });
+		}
+	});
+}
+
+export const handleMetrics: Router.Middleware = async (ctx) => {
+	if (config.metrics?.token !== undefined) {
+		if (ctx.query.token === undefined) {
+			ctx.res.statusCode = 401;
+			ctx.body = "Missing token parameter";
+			return;
+		}
+		const correct = config.metrics.token === ctx.query.token;
+		if (!correct) {
+			ctx.res.statusCode = 403;
+			ctx.body = "Incorrect token";
+			return;
+		}
+	}
+	try {
+		if (cluster.isPrimary) {
+			ctx.set("content-type", register.contentType);
+			ctx.body = await register.metrics();
+		} else {
+			const wait = new Promise<void>((resolve, reject) => {
+				const timeout = setTimeout(
+					() => reject("Timeout while waiting for cluster master"),
+					1000 * 60
+				);
+				xev.once("registry-response", (response) => {
+					clearTimeout(timeout);
+					if (response.error) reject(response.error);
+					ctx.set("content-type", response.contentType);
+					ctx.body = response.body;
+					resolve();
+				});
+			});
+			xev.emit("registry-request");
+			await wait;
+		}
+	} catch (err) {
+		ctx.res.statusCode = 500;
+		ctx.body = err;
+	}
+};
+
+const counter = (configuration: CounterConfiguration<string>) => {
+	if (config.metrics?.enable) {
+		if (cluster.isPrimary) {
+			const counter = new Counter(configuration);
+			counter.reset(); // initialize internal hashmap
+			xev.on(`metrics-counter-${configuration.name}`, () => counter.inc());
+			return () => counter.inc();
+		} else {
+			return () => xev.emit(`metrics-counter-${configuration.name}`);
+		}
+	} else {
+		return () => { };
+	}
+};
+
+export const tickOutbox = counter({
+	name: "iceshrimp_outbox_total",
+	help: "Total AP outbox calls",
+});
+
+export const tickInbox = counter({
+	name: "iceshrimp_inbox_total",
+	help: "Total AP inbox calls",
+});
+
+export const tickFetch = counter({
+	name: "iceshrimp_fetch_total",
+	help: "Total AP fetch calls",
+});
+
+export const tickResolve = counter({
+	name: "iceshrimp_resolve_total",
+	help: "Total AP resolve calls",

packages/backend/src/queue/queues/deliver.ts

@@ -17,6 +17,7 @@ import config from "@/config/index.js";
 import { createQueue, processorTimeout } from "./index.js";
 import { ThinUser } from "../types.js";
 import { Job } from "bullmq";
+import { tickOutbox } from "@/metrics.js";
 
 export const deliverLogger = new Logger("deliver");
 
@@ -55,6 +56,8 @@ async function process(job: Job<DeliverJobData>) {
 			federationChart.deliverd(i.host, true);
 		});
 
+		tickOutbox();
+
 		return "Success";
 	} catch (res) {
 		// Update stats

packages/backend/src/queue/queues/inbox.ts

@@ -25,6 +25,7 @@ import { verifySignature } from "@/remote/activitypub/check-fetch.js";
 import { Job } from "bullmq";
 import { createQueue, processorTimeout } from "./index.js";
 import config from "@/config/index.js";
+import { tickInbox } from "@/metrics.js";
 
 export const inboxLogger = new Logger("inbox");
 
@@ -221,6 +222,8 @@ async function process(job: Job<InboxJobData>): Promise<string> {
 		}
 	}
 
+	tickInbox();
+
 	// アクティビティを処理
 	await perform(authUser.user, activity);
 	return "ok";

packages/backend/src/remote/activitypub/check-fetch.ts

@@ -12,6 +12,7 @@ import type { UserPublickey } from "@/models/entities/user-publickey.js";
 import { verify } from "node:crypto";
 import { toSingle } from "@/prelude/array.js";
 import { createHash } from "node:crypto";
+import { tickFetch } from "@/metrics.js";
 
 export async function hasSignature(req: IncomingMessage): Promise<string> {
 	const meta = await fetchMeta();
@@ -120,7 +121,12 @@ export async function checkFetch(req: IncomingMessage): Promise<number> {
 			return 403;
 		}
 
-		return verifySignature(signature, authUser.key) ? 200 : 401;
+		if (!verifySignature(signature, authUser.key)) {
+			return 401;
+		}
+
+		tickFetch();
+		return 200;
 	}
 	return 200;
 }

packages/backend/src/remote/activitypub/resolver.ts

@@ -25,6 +25,7 @@ import renderFollow from "@/remote/activitypub/renderer/follow.js";
 import { shouldBlockInstance } from "@/misc/should-block-instance.js";
 import { apLogger } from "@/remote/activitypub/logger.js";
 import { In, IsNull, Not } from "typeorm";
+import { tickResolve } from "@/metrics.js";
 
 export default class Resolver {
 	private history: Set<string>;
@@ -127,7 +128,10 @@ export default class Resolver {
 		if (object.id == null) throw new Error("Object has no ID");
 		const objectId = new URL(object.id);
 		const resFinalUrl = new URL(res.finalUrl);
-		if (resFinalUrl.toString() === objectId.toString()) return object;
+		if (resFinalUrl.toString() === objectId.toString()) {
+			tickResolve();
+			return object;
+		}
 
 		if (resFinalUrl.host !== objectId.host)
 			throw new Error("Object ID host doesn't match final url host");
@@ -141,6 +145,7 @@ export default class Resolver {
 		if (finalResFinalUrl.toString() !== finalObjectId.toString())
 			throw new Error("Object ID still doesn't match final URL after second fetch attempt")
 
+		tickResolve();
 		return finalObject;
 	}
 

packages/backend/src/server/activitypub.ts

@@ -35,6 +35,7 @@ import Outbox, { packActivity } from "./activitypub/outbox.js";
 import { serverLogger } from "./index.js";
 import config from "@/config/index.js";
 import Koa from "koa";
+import { tickFetch } from "@/metrics.js";
 
 // Init router
 const router = new Router();
@@ -223,6 +224,7 @@ router.get("/users/:user/collections/featured", Featured);
 router.get("/users/:user/publickey", async (ctx) => {
 	const instanceActor = await getInstanceActor();
 	if (ctx.params.user === instanceActor.id) {
+		tickFetch();
 		ctx.body = renderActivity(
 			renderKey(instanceActor, await getUserKeypair(instanceActor.id)),
 		);
@@ -287,6 +289,7 @@ router.get("/users/:user", async (ctx, next) => {
 
 	const instanceActor = await getInstanceActor();
 	if (ctx.params.user === instanceActor.id) {
+		tickFetch();
 		await userInfo(ctx, instanceActor);
 		return;
 	}
@@ -312,6 +315,7 @@ router.get("/@:user", async (ctx, next) => {
 	if (!isActivityPubReq(ctx)) return await next();
 
 	if (ctx.params.user === "instance.actor") {
+		tickFetch();
 		const instanceActor = await getInstanceActor();
 		await userInfo(ctx, instanceActor);
 		return;
@@ -333,6 +337,7 @@ router.get("/@:user", async (ctx, next) => {
 });
 
 router.get("/actor", async (ctx, next) => {
+	tickFetch();
 	const instanceActor = await getInstanceActor();
 	await userInfo(ctx, instanceActor);
 });

packages/backend/src/server/index.ts

@@ -33,6 +33,7 @@ import removeTrailingSlash from "koa-remove-trailing-slashes";
 import { koaBody } from "koa-body";
 import { setupEndpointsAuthRoot } from "@/server/api/mastodon/endpoints/auth.js";
 import { CatchErrorsMiddleware } from "@/server/api/mastodon/middleware/catch-errors.js";
+import { handleMetrics } from "@/metrics.js";
 export const serverLogger = new Logger("server", "gray", false);
 
 // Init app
@@ -117,6 +118,10 @@ router.get("/identicon/:x", async (ctx) => {
 	}
 });
 
+if (config.metrics?.enable) {
+	router.get("/metrics", handleMetrics);
+}
+
 mastoRouter.use(
 	koaBody({
 		urlencoded: true,

yarn.lock

@@ -2159,6 +2159,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@opentelemetry/api@npm:^1.4.0":
+  version: 1.7.0
+  resolution: "@opentelemetry/api@npm:1.7.0"
+  checksum: 10/bcf7afa7051dcd4583898a68f8a57fb4c85b5cedddf7b6eb3616595c0b3bcd7f5448143b8355b00935a755de004d6285489f8e132f34127efe7b1be404622a3e
+  languageName: node
+  linkType: hard
+
 "@paralleldrive/cuid2@npm:^2.2.2":
   version: 2.2.2
   resolution: "@paralleldrive/cuid2@npm:2.2.2"
@@ -5599,6 +5606,7 @@ __metadata:
     pg: "npm:8.11.1"
     private-ip: "npm:2.3.4"
     probe-image-size: "npm:7.2.3"
+    prom-client: "npm:^15.1.0"
     promise-limit: "npm:2.7.0"
     pug: "npm:3.0.2"
     punycode: "npm:2.3.0"
@@ -5759,6 +5767,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"bintrees@npm:1.0.2":
+  version: 1.0.2
+  resolution: "bintrees@npm:1.0.2"
+  checksum: 10/071896cea5ea5413316c8436e95799444c208630d5c539edd8a7089fc272fc5d3634aa4a2e4847b28350dda1796162e14a34a0eda53108cc5b3c2ff6a036c1fa
+  languageName: node
+  linkType: hard
+
 "bl@npm:^1.0.0":
   version: 1.2.3
   resolution: "bl@npm:1.2.3"
@@ -17072,6 +17087,16 @@ __metadata:
   languageName: node
   linkType: hard
 
+"prom-client@npm:^15.1.0":
+  version: 15.1.0
+  resolution: "prom-client@npm:15.1.0"
+  dependencies:
+    "@opentelemetry/api": "npm:^1.4.0"
+    tdigest: "npm:^0.1.1"
+  checksum: 10/ecb6f40de755ca9cc6dde758d195ed3e1d3b47a341d2092af8c18dbf7e6ef1079c8b8bb02496f2f430cf8bd9d391c1ea5bebbb85cdda95f67dad2dbfb90509aa
+  languageName: node
+  linkType: hard
+
 "promise-limit@npm:2.7.0":
   version: 2.7.0
   resolution: "promise-limit@npm:2.7.0"
@@ -19630,6 +19655,15 @@ __metadata:
   languageName: node
   linkType: hard
 
+"tdigest@npm:^0.1.1":
+  version: 0.1.2
+  resolution: "tdigest@npm:0.1.2"
+  dependencies:
+    bintrees: "npm:1.0.2"
+  checksum: 10/45be99fa52dab74b8edafe150e473cdc45aa1352c75ed516a39905f350a08c3175f6555598111042c3677ba042d7e3cae6b5ce4c663fe609bc634f326aabc9d6
+  languageName: node
+  linkType: hard
+
 "terminal-link@npm:^2.0.0":
   version: 2.1.1
   resolution: "terminal-link@npm:2.1.1"