プロキシの除外ホスト (#6244)

* プロキシの除外ホスト

* オブジェクトストレージとの通信にProxyを使うかを選択できるように

* fix lint

* コメント

Co-authored-by: rinsuki <428rinsuki+git@gmail.com>

.config/example.yml

@@ -142,6 +142,11 @@ id: 'aid'
 # Proxy for HTTP/HTTPS
 #proxy: http://127.0.0.1:3128
 
+#proxyBypassHosts: [
+#  'example.com',
+#  '192.0.2.8'
+#]
+
 # Proxy for SMTP/SMTPS
 #proxySmtp: http://127.0.0.1:3128   # use HTTP/1.1 CONNECT
 #proxySmtp: socks4://127.0.0.1:1080 # use SOCKS4

locales/ja-JP.yml

@@ -454,6 +454,8 @@ objectStorageRegion: "Region"
 objectStorageRegionDesc: "'xx-east-1'のようなregionを指定してください。使用サービスにregionの概念がない場合は、空または'us-east-1'にしてください。"
 objectStorageUseSSL: "SSLを使用する"
 objectStorageUseSSLDesc: "API接続にhttpsを使用しない場合はオフにしてください"
+objectStorageUseProxy: "Proxyを利用する"
+objectStorageUseProxyDesc: "API接続にproxyを利用しない場合はオフにしてください"
 serverLogs: "サーバーログ"
 deleteAll: "全て削除"
 showFixedPostForm: "タイムライン上部に投稿フォームを表示する"

migration/1586624197029-AddObjectStorageUseProxy.ts

@@ -0,0 +1,14 @@
+import {MigrationInterface, QueryRunner} from 'typeorm';
+
+export class AddObjectStorageUseProxy1586624197029 implements MigrationInterface {
+		name = 'AddObjectStorageUseProxy1586624197029'
+
+		public async up(queryRunner: QueryRunner): Promise<void> {
+				await queryRunner.query(`ALTER TABLE "meta" ADD "objectStorageUseProxy" boolean NOT NULL DEFAULT true`, undefined);
+		}
+
+		public async down(queryRunner: QueryRunner): Promise<void> {
+				await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "objectStorageUseProxy"`, undefined);
+		}
+
+}

src/client/pages/instance/settings.vue

@@ -116,6 +116,7 @@
 					<mk-input v-model="objectStorageSecretKey" :disabled="!useObjectStorage"><template #icon><fa :icon="faKey"/></template>Secret key</mk-input>
 				</div>
 				<mk-switch v-model="objectStorageUseSSL" :disabled="!useObjectStorage">{{ $t('objectStorageUseSSL') }}<template #desc>{{ $t('objectStorageUseSSLDesc') }}</template></mk-switch>
+				<mk-switch v-model="objectStorageUseProxy" :disabled="!useObjectStorage">{{ $t('objectStorageUseProxy') }}<template #desc>{{ $t('objectStorageUseProxyDesc') }}</template></mk-switch>
 			</template>
 		</div>
 		<div class="_footer">
@@ -249,6 +250,7 @@ export default Vue.extend({
 			objectStorageAccessKey: null,
 			objectStorageSecretKey: null,
 			objectStorageUseSSL: false,
+			objectStorageUseProxy: false,
 			enableTwitterIntegration: false,
 			twitterConsumerKey: null,
 			twitterConsumerSecret: null,
@@ -303,6 +305,7 @@ export default Vue.extend({
 		this.objectStorageAccessKey = this.meta.objectStorageAccessKey;
 		this.objectStorageSecretKey = this.meta.objectStorageSecretKey;
 		this.objectStorageUseSSL = this.meta.objectStorageUseSSL;
+		this.objectStorageUseProxy = this.meta.objectStorageUseProxy;
 		this.enableTwitterIntegration = this.meta.enableTwitterIntegration;
 		this.twitterConsumerKey = this.meta.twitterConsumerKey;
 		this.twitterConsumerSecret = this.meta.twitterConsumerSecret;
@@ -411,6 +414,7 @@ export default Vue.extend({
 				objectStorageAccessKey: this.objectStorageAccessKey ? this.objectStorageAccessKey : null,
 				objectStorageSecretKey: this.objectStorageSecretKey ? this.objectStorageSecretKey : null,
 				objectStorageUseSSL: this.objectStorageUseSSL,
+				objectStorageUseProxy: this.objectStorageUseProxy,
 				enableTwitterIntegration: this.enableTwitterIntegration,
 				twitterConsumerKey: this.twitterConsumerKey,
 				twitterConsumerSecret: this.twitterConsumerSecret,

src/config/types.ts

@@ -35,6 +35,7 @@ export type Source = {
 
 	proxy?: string;
 	proxySmtp?: string;
+	proxyBypassHosts?: string[];
 
 	accesslog?: string;
 

src/misc/download-url.ts

@@ -2,7 +2,7 @@ import * as fs from 'fs';
 import * as stream from 'stream';
 import * as util from 'util';
 import fetch from 'node-fetch';
-import { httpAgent, httpsAgent } from './fetch';
+import { getAgentByUrl } from './fetch';
 import { AbortController } from 'abort-controller';
 import config from '../config';
 import * as chalk from 'chalk';
@@ -25,7 +25,7 @@ export async function downloadUrl(url: string, path: string) {
 		},
 		timeout: 10 * 1000,
 		signal: controller.signal,
-		agent: u => u.protocol == 'http:' ? httpAgent : httpsAgent,
+		agent: getAgentByUrl,
 	});
 
 	if (!response.ok) {

src/misc/fetch.ts

@@ -13,7 +13,7 @@ export async function getJson(url: string, accept = 'application/json, */*', tim
 			Accept: accept
 		}, headers || {}),
 		timeout,
-		agent: u => u.protocol == 'http:' ? httpAgent : httpsAgent,
+		agent: getAgentByUrl,
 	});
 
 	if (!res.ok) {
@@ -27,17 +27,46 @@ export async function getJson(url: string, accept = 'application/json, */*', tim
 	return await res.json();
 }
 
+/**
+ * Get http non-proxy agent
+ */
+const _http = new http.Agent({
+	keepAlive: true,
+	keepAliveMsecs: 30 * 1000,
+});
+
+/**
+ * Get https non-proxy agent
+ */
+const _https = new https.Agent({
+	keepAlive: true,
+	keepAliveMsecs: 30 * 1000,
+	lookup: cache.lookup,
+});
+
+/**
+ * Get http proxy or non-proxy agent
+ */
 export const httpAgent = config.proxy
 	? new HttpProxyAgent(config.proxy)
-	: new http.Agent({
-		keepAlive: true,
-		keepAliveMsecs: 30 * 1000,
-	});
+	: _http;
 
+/**
+ * Get https proxy or non-proxy agent
+ */
 export const httpsAgent = config.proxy
 	? new HttpsProxyAgent(config.proxy)
-	: new https.Agent({
-		keepAlive: true,
-		keepAliveMsecs: 30 * 1000,
-		lookup: cache.lookup,
-	});
+	: _https;
+
+/**
+ * Get agent by URL
+ * @param url URL
+ * @param bypassProxy Allways bypass proxy
+ */
+export function getAgentByUrl(url: URL, bypassProxy = false) {
+	if (bypassProxy || (config.proxyBypassHosts || []).includes(url.hostname)) {
+		return url.protocol == 'http:' ? _http : _https;
+	} else {
+		return url.protocol == 'http:' ? httpAgent : httpsAgent;
+	}
+}

src/models/entities/meta.ts

@@ -348,4 +348,9 @@ export class Meta {
 		default: true,
 	})
 	public objectStorageUseSSL: boolean;
+
+	@Column('boolean', {
+		default: true,
+	})
+	public objectStorageUseProxy: boolean;
 }

src/remote/activitypub/request.ts

@@ -6,7 +6,7 @@ import config from '../../config';
 import { ILocalUser } from '../../models/entities/user';
 import { UserKeypairs } from '../../models';
 import { ensure } from '../../prelude/ensure';
-import { httpsAgent } from '../../misc/fetch';
+import { getAgentByUrl } from '../../misc/fetch';
 
 export default async (user: ILocalUser, url: string, object: any) => {
 	const timeout = 10 * 1000;
@@ -25,7 +25,7 @@ export default async (user: ILocalUser, url: string, object: any) => {
 
 	await new Promise((resolve, reject) => {
 		const req = https.request({
-			agent: httpsAgent,
+			agent: getAgentByUrl(new URL(`https://example.net`)),
 			protocol,
 			hostname,
 			port,

src/server/api/endpoints/admin/update-meta.ts

@@ -394,6 +394,10 @@ export const meta = {
 		objectStorageUseSSL: {
 			validator: $.optional.bool
 		},
+
+		objectStorageUseProxy: {
+			validator: $.optional.bool
+		}
 	}
 };
 
@@ -632,6 +636,10 @@ export default define(meta, async (ps, me) => {
 		set.objectStorageUseSSL = ps.objectStorageUseSSL;
 	}
 
+	if (ps.objectStorageUseProxy !== undefined) {
+		set.objectStorageUseProxy = ps.objectStorageUseProxy;
+	}
+
 	await getConnection().transaction(async transactionalEntityManager => {
 		const meta = await transactionalEntityManager.findOne(Meta, {
 			order: {

src/server/api/endpoints/meta.ts

@@ -190,6 +190,7 @@ export default define(meta, async (ps, me) => {
 		response.objectStorageAccessKey = instance.objectStorageAccessKey;
 		response.objectStorageSecretKey = instance.objectStorageSecretKey;
 		response.objectStorageUseSSL = instance.objectStorageUseSSL;
+		response.objectStorageUseProxy = instance.objectStorageUseProxy;
 	}
 
 	return response;

src/services/drive/s3.ts

@@ -1,8 +1,12 @@
 import * as S3 from 'aws-sdk/clients/s3';
 import { Meta } from '../../models/entities/meta';
-import { httpsAgent, httpAgent } from '../../misc/fetch';
+import { getAgentByUrl } from '../../misc/fetch';
 
 export function getS3(meta: Meta) {
+	const u = meta.objectStorageEndpoint != null
+		? `${meta.objectStorageUseSSL ? 'https://' : 'http://'}${meta.objectStorageEndpoint}`
+		: `${meta.objectStorageUseSSL ? 'https://' : 'http://'}example.net`;
+
 	return new S3({
 		endpoint: meta.objectStorageEndpoint || undefined,
 		accessKeyId: meta.objectStorageAccessKey!,
@@ -11,7 +15,7 @@ export function getS3(meta: Meta) {
 		sslEnabled: meta.objectStorageUseSSL,
 		s3ForcePathStyle: !!meta.objectStorageEndpoint,
 		httpOptions: {
-			agent: meta.objectStorageUseSSL ? httpsAgent : httpAgent
+			agent: getAgentByUrl(new URL(u), !meta.objectStorageUseProxy)
 		}
 	});
 }