From a5f4279d321ce3b3797098ac9b4e7ee2c9ae6b89 Mon Sep 17 00:00:00 2001 From: Kopper Date: Mon, 18 Nov 2024 23:36:47 +0300 Subject: [PATCH] [backend] Check target IP before sending HTTP request Backported upstream commit "fix(backend): check target IP before sending HTTP request" Co-authored-by: rectcoordsystem Co-authored-by: anatawa12 --- packages/backend/src/misc/checked-fetch.ts | 60 +++++++++++++++++++ packages/backend/src/misc/download-url.ts | 24 -------- packages/backend/src/misc/fetch.ts | 11 ++-- .../backend/src/server/proxy/proxy-media.ts | 34 ----------- 4 files changed, 66 insertions(+), 63 deletions(-) create mode 100644 packages/backend/src/misc/checked-fetch.ts diff --git a/packages/backend/src/misc/checked-fetch.ts b/packages/backend/src/misc/checked-fetch.ts new file mode 100644 index 000000000..bf2edb1f0 --- /dev/null +++ b/packages/backend/src/misc/checked-fetch.ts @@ -0,0 +1,60 @@ +import * as http from "node:http"; +import * as https from "node:https"; +import net from "node:net"; +import { HttpProxyAgent, HttpsProxyAgent } from "hpagent"; +import config from "@/config/index.js"; +import IPCIDR from "ip-cidr"; +import PrivateIp from "private-ip"; + +declare module 'node:http' { + interface Agent { + createConnection(options: net.NetConnectOpts, callback?: (err: unknown, stream: net.Socket) => void): net.Socket; + } +} + +function isPrivateIp(ip: string): boolean { + for (const net of config.allowedPrivateNetworks || []) { + const cidr = new IPCIDR(net); + if (cidr.contains(ip)) { + return false; + } + } + + return PrivateIp(ip); +} + +function checkConnection(socket: net.Socket) { + const address = socket.remoteAddress; + if (process.env.NODE_ENV === 'production') { + if (address && IPCIDR.isValidAddress(address) && isPrivateIp(address)) { + socket.destroy(new Error(`Blocked address: ${address}`)); + } + } +} + +export class CheckedHttpAgent extends http.Agent { + createConnection(options: net.NetConnectOpts, callback?: (err: unknown, stream: net.Socket) => void): net.Socket { + const socket = super.createConnection(options, callback).on('connect', () => { checkConnection(socket) }); + return socket; + } +} + +export class CheckedHttpsAgent extends https.Agent { + createConnection(options: net.NetConnectOpts, callback?: (err: unknown, stream: net.Socket) => void): net.Socket { + const socket = super.createConnection(options, callback).on('connect', () => { checkConnection(socket) }); + return socket; + } +} +export class CheckedHttpProxyAgent extends HttpProxyAgent { + createConnection(options: net.NetConnectOpts, callback?: (err: unknown, stream: net.Socket) => void): net.Socket { + const socket = super.createConnection(options, callback).on('connect', () => { checkConnection(socket) }); + return socket; + } +} + +export class CheckedHttpsProxyAgent extends HttpsProxyAgent { + createConnection(options: net.NetConnectOpts, callback?: (err: unknown, stream: net.Socket) => void): net.Socket { + const socket = super.createConnection(options, callback).on('connect', () => { checkConnection(socket) }); + return socket; + } +} diff --git a/packages/backend/src/misc/download-url.ts b/packages/backend/src/misc/download-url.ts index e9975f348..fca6dc36d 100644 --- a/packages/backend/src/misc/download-url.ts +++ b/packages/backend/src/misc/download-url.ts @@ -6,8 +6,6 @@ import { httpAgent, httpsAgent, StatusError } from "./fetch.js"; import config from "@/config/index.js"; import chalk from "chalk"; import Logger from "@/services/logger.js"; -import IPCIDR from "ip-cidr"; -import PrivateIp from "private-ip"; const pipeline = util.promisify(stream.pipeline); @@ -45,18 +43,6 @@ export async function downloadUrl(url: string, path: string): Promise { }, }) .on("response", (res: Got.Response) => { - if ( - (process.env.NODE_ENV === "production" || - process.env.NODE_ENV === "test") && - !config.proxy && - res.ip - ) { - if (isPrivateIp(res.ip)) { - logger.warn(`Blocked address: ${res.ip}`); - req.destroy(); - } - } - const contentLength = res.headers["content-length"]; if (contentLength != null) { const size = Number(contentLength); @@ -92,13 +78,3 @@ export async function downloadUrl(url: string, path: string): Promise { logger.succ(`Download finished: ${chalk.cyan(url)}`); } -function isPrivateIp(ip: string): boolean { - for (const net of config.allowedPrivateNetworks || []) { - const cidr = new IPCIDR(net); - if (cidr.contains(ip)) { - return false; - } - } - - return PrivateIp(ip); -} diff --git a/packages/backend/src/misc/fetch.ts b/packages/backend/src/misc/fetch.ts index 69f03ad05..bbf063d9b 100644 --- a/packages/backend/src/misc/fetch.ts +++ b/packages/backend/src/misc/fetch.ts @@ -3,8 +3,9 @@ import * as https from "node:https"; import type { URL } from "node:url"; import CacheableLookup from "cacheable-lookup"; import fetch from "node-fetch"; -import { HttpProxyAgent, HttpsProxyAgent } from "hpagent"; import config from "@/config/index.js"; +import net from "node:net"; +import {CheckedHttpAgent, CheckedHttpProxyAgent, CheckedHttpsAgent, CheckedHttpsProxyAgent} from "@/misc/checked-fetch.js"; export async function getJson( url: string, @@ -132,7 +133,7 @@ const cache = new CacheableLookup({ /** * Get http non-proxy agent */ -const _http = new http.Agent({ +const _http = new CheckedHttpAgent({ keepAlive: true, keepAliveMsecs: 30 * 1000, lookup: cache.lookup, @@ -141,7 +142,7 @@ const _http = new http.Agent({ /** * Get https non-proxy agent */ -const _https = new https.Agent({ +const _https = new CheckedHttpsAgent({ keepAlive: true, keepAliveMsecs: 30 * 1000, lookup: cache.lookup, @@ -153,7 +154,7 @@ const maxSockets = Math.max(256, config.deliverJobConcurrency || 128); * Get http proxy or non-proxy agent */ export const httpAgent = config.proxy - ? new HttpProxyAgent({ + ? new CheckedHttpProxyAgent({ keepAlive: true, keepAliveMsecs: 30 * 1000, maxSockets, @@ -167,7 +168,7 @@ export const httpAgent = config.proxy * Get https proxy or non-proxy agent */ export const httpsAgent = config.proxy - ? new HttpsProxyAgent({ + ? new CheckedHttpsProxyAgent({ keepAlive: true, keepAliveMsecs: 30 * 1000, maxSockets, diff --git a/packages/backend/src/server/proxy/proxy-media.ts b/packages/backend/src/server/proxy/proxy-media.ts index af802c939..818347482 100644 --- a/packages/backend/src/server/proxy/proxy-media.ts +++ b/packages/backend/src/server/proxy/proxy-media.ts @@ -51,40 +51,6 @@ export async function proxyMedia(ctx: Koa.Context) { if (ctx.status == 429) return; - const { hostname } = new URL(url); - let resolvedIps; - try { - resolvedIps = await promises.resolve(hostname); - } catch (error) { - ctx.status = 400; - ctx.body = { message: "Invalid URL" }; - return; - } - - const isSSRF = resolvedIps.some((ip) => { - if (net.isIPv4(ip)) { - const parts = ip.split(".").map(Number); - return ( - parts[0] === 10 || - (parts[0] === 172 && parts[1] >= 16 && parts[1] < 32) || - (parts[0] === 192 && parts[1] === 168) || - parts[0] === 127 || - parts[0] === 0 - ); - } else if (net.isIPv6(ip)) { - return ( - ip.startsWith("::") || ip.startsWith("fc00:") || ip.startsWith("fe80:") - ); - } - return false; - }); - - if (isSSRF) { - ctx.status = 400; - ctx.body = { message: "Access to this URL is not allowed" }; - return; - } - // Create temp file const [path, cleanup] = await createTemp();