feat(backend): elasticsearchで検索できるように (MisskeyIO#661)

Co-authored-by: 皐月なふ (Nafu Satsuki) <satsuki@nafusoft.dev>

packages/backend/package.json

@@ -72,6 +72,7 @@
 		"@bull-board/fastify": "5.18.1",
 		"@bull-board/ui": "5.18.1",
 		"@discordapp/twemoji": "15.0.3",
+		"@elastic/elasticsearch": "^8.14.0",
 		"@fastify/accepts": "4.3.0",
 		"@fastify/cookie": "9.3.1",
 		"@fastify/cors": "9.0.1",

packages/backend/src/GlobalModule.ts

@@ -8,6 +8,7 @@ import { Global, Inject, Module } from '@nestjs/common';
 import * as Redis from 'ioredis';
 import { DataSource } from 'typeorm';
 import { MeiliSearch } from 'meilisearch';
+import { Client as ElasticSearch } from '@elastic/elasticsearch';
 import { DI } from './di-symbols.js';
 import { Config, loadConfig } from './config.js';
 import { createPostgresDataSource } from './postgres.js';
@@ -44,6 +45,30 @@ const $meilisearch: Provider = {
 	inject: [DI.config],
 };
 
+const $elasticsearch: Provider = {
+	provide: DI.elasticsearch,
+	useFactory: (config: Config) => {
+		if (config.elasticsearch) {
+			return new ElasticSearch({
+				nodes: {
+					url: new URL(`${config.elasticsearch.ssl ? 'https' : 'http'}://${config.elasticsearch.host}:${config.elasticsearch.port}`),
+					ssl: {
+						rejectUnauthorized: config.elasticsearch.rejectUnauthorized,
+					},
+				},
+				auth: (config.elasticsearch.user && config.elasticsearch.pass) ? {
+					username: config.elasticsearch.user,
+					password: config.elasticsearch.pass,
+				} : undefined,
+				pingTimeout: 30000,
+			});
+		} else {
+			return null;
+		}
+	},
+	inject: [DI.config],
+};
+
 const $redis: Provider = {
 	provide: DI.redis,
 	useFactory: (config: Config) => {
@@ -160,8 +185,8 @@ const $redisForTimelines: Provider = {
 @Global()
 @Module({
 	imports: [RepositoryModule],
-	providers: [$config, $db, $meilisearch, $redis, $redisForPub, $redisForSub, $redisForTimelines],
+	providers: [$config, $db, $meilisearch, $elasticsearch, $redis, $redisForPub, $redisForSub, $redisForTimelines],
-	exports: [$config, $db, $meilisearch, $redis, $redisForPub, $redisForSub, $redisForTimelines, RepositoryModule],
+	exports: [$config, $db, $meilisearch, $elasticsearch, $redis, $redisForPub, $redisForSub, $redisForTimelines, RepositoryModule],
 })
 export class GlobalModule implements OnApplicationShutdown {
 	constructor(

packages/backend/src/config.ts

@@ -66,6 +66,16 @@ type Source = {
 		scope?: 'local' | 'global' | string[];
 	};
 
+	elasticsearch?: {
+		host: string;
+		port: string;
+		user: string;
+		pass: string;
+		ssl?: boolean;
+		rejectUnauthorized?: boolean;
+		index: string;
+	};
+
 	skebStatus?: {
 		method: string;
 		endpoint: string;
@@ -149,6 +159,15 @@ export type Config = {
 		index: string;
 		scope?: 'local' | 'global' | string[];
 	} | undefined;
+	elasticsearch: {
+		host: string;
+		port: string;
+		user: string;
+		pass: string;
+		ssl?: boolean;
+		rejectUnauthorized?: boolean;
+		index: string;
+	} | undefined;
 	skebStatus: {
 		method: string;
 		endpoint: string;
@@ -272,6 +291,7 @@ export function loadConfig(): Config {
 		dbReplications: config.dbReplications,
 		dbSlaves: config.dbSlaves,
 		meilisearch: config.meilisearch,
+		elasticsearch: config.elasticsearch,
 		redis,
 		redisForPubsub: config.redisForPubsub ? convertRedisOptions(config.redisForPubsub, host) : redis,
 		redisForSystemQueue: config.redisForSystemQueue ? convertRedisOptions(config.redisForSystemQueue, host) : redisForJobQueue,

packages/backend/src/core/SearchService.ts

@@ -8,6 +8,7 @@ import { In } from 'typeorm';
 import { DI } from '@/di-symbols.js';
 import type { Config } from '@/config.js';
 import { bindThis } from '@/decorators.js';
+import { LoggerService } from '@/core/LoggerService.js';
 import { MiNote } from '@/models/Note.js';
 import { MiUser } from '@/models/_.js';
 import type { NotesRepository } from '@/models/_.js';
@@ -16,7 +17,9 @@ import { isUserRelated } from '@/misc/is-user-related.js';
 import { CacheService } from '@/core/CacheService.js';
 import { QueryService } from '@/core/QueryService.js';
 import { IdService } from '@/core/IdService.js';
+import type Logger from '@/logger.js';
 import type { Index, MeiliSearch } from 'meilisearch';
+import type { Client as ElasticSearch } from '@elastic/elasticsearch';
 
 type K = string;
 type V = string | number | boolean;
@@ -65,6 +68,8 @@ function compileQuery(q: Q): string {
 export class SearchService {
 	private readonly meilisearchIndexScope: 'local' | 'global' | string[] = 'local';
 	private meilisearchNoteIndex: Index | null = null;
+	private elasticsearchNoteIndex: string | null = null;
+	private logger: Logger;
 
 	constructor(
 		@Inject(DI.config)
@@ -73,15 +78,24 @@ export class SearchService {
 		@Inject(DI.meilisearch)
 		private meilisearch: MeiliSearch | null,
 
+		@Inject(DI.elasticsearch)
+		private elasticsearch: ElasticSearch | null,
+
 		@Inject(DI.notesRepository)
 		private notesRepository: NotesRepository,
 
 		private cacheService: CacheService,
 		private queryService: QueryService,
 		private idService: IdService,
+		private loggerService: LoggerService,
 	) {
+		this.logger = this.loggerService.getLogger('note:search');
+
 		if (meilisearch) {
 			this.meilisearchNoteIndex = meilisearch.index(`${config.meilisearch!.index}---notes`);
+			if (config.meilisearch?.scope) {
+				this.meilisearchIndexScope = config.meilisearch.scope;
+			}
 			/*this.meilisearchNoteIndex.updateSettings({
 				searchableAttributes: [
 					'text',
@@ -104,10 +118,52 @@ export class SearchService {
 					maxTotalHits: 10000,
 				},
 			});*/
-		}
+		} else if (this.elasticsearch) {
-
+			this.elasticsearchNoteIndex = `${config.elasticsearch!.index}---notes`;
-		if (config.meilisearch?.scope) {
+			this.elasticsearch.indices.exists({
-			this.meilisearchIndexScope = config.meilisearch.scope;
+				index: this.elasticsearchNoteIndex,
+			}).then((indexExists) => {
+				if (!indexExists) {
+					this.elasticsearch?.indices.create(
+						{
+							index: this.elasticsearchNoteIndex + `-${new Date().toISOString().slice(0, 7).replace(/-/g, '')}`,
+							mappings: {
+								properties: {
+									text: { type: 'text' },
+									cw: { type: 'text' },
+									createdAt: { type: 'long' },
+									userId: { type: 'keyword' },
+									userHost: { type: 'keyword' },
+									channelId: { type: 'keyword' },
+									tags: { type: 'keyword' },
+								},
+							},
+							settings: {
+								index: {
+									analysis: {
+										tokenizer: {
+											kuromoji: {
+												type: 'kuromoji_tokenizer',
+												mode: 'search',
+											},
+										},
+										analyzer: {
+											kuromoji_analyzer: {
+												type: 'custom',
+												tokenizer: 'kuromoji',
+											},
+										},
+									},
+								},
+							},
+						},
+					).catch((error) => {
+						this.logger.error(error);
+					});
+				}
+			}).catch((error) => {
+				this.logger.error('Error while checking if index exists', error);
+			});
 		}
 	}
 
@@ -144,6 +200,23 @@ export class SearchService {
 			}], {
 				primaryKey: 'id',
 			});
+		}	else if (this.elasticsearch) {
+			const body = {
+				createdAt: this.idService.parse(note.id).date.getTime(),
+				userId: note.userId,
+				userHost: note.userHost,
+				channelId: note.channelId,
+				cw: note.cw,
+				text: note.text,
+				tags: note.tags,
+			};
+			await this.elasticsearch.index({
+				index: this.elasticsearchNoteIndex + `-${new Date().toISOString().slice(0, 7).replace(/-/g, '')}` as string,
+				id: note.id,
+				body: body,
+			}).catch((error) => {
+				console.error(error);
+			});
 		}
 	}
 
@@ -204,6 +277,67 @@ export class SearchService {
 				if (me && isUserRelated(note, userIdsWhoMeMuting)) return false;
 				return true;
 			});
+			return notes.sort((a, b) => a.id > b.id ? -1 : 1);
+		} else if (this.elasticsearch) {
+			const esFilter: any = {
+				bool: {
+					must: [],
+				},
+			};
+
+			if (pagination.untilId) esFilter.bool.must.push({ range: { createdAt: { lt: this.idService.parse(pagination.untilId).date.getTime() } } });
+			if (pagination.sinceId) esFilter.bool.must.push({ range: { createdAt: { gt: this.idService.parse(pagination.sinceId).date.getTime() } } });
+			if (opts.userId) esFilter.bool.must.push({ term: { userId: opts.userId } });
+			if (opts.channelId) esFilter.bool.must.push({ term: { channelId: opts.channelId } });
+			if (opts.host) {
+				if (opts.host === '.') {
+					esFilter.bool.must.push({ bool: { must_not: [{ exists: { field: 'userHost' } }] } });
+				} else {
+					esFilter.bool.must.push({ term: { userHost: opts.host } });
+				}
+			}
+
+			if (q !== '') {
+				esFilter.bool.must.push({
+					bool: {
+						should: [
+							{ wildcard: { 'text': { value: q } } },
+							{ simple_query_string: { fields: ['text'], 'query': q, default_operator: 'and' } },
+							{ wildcard: { 'cw': { value: q } } },
+							{ simple_query_string: { fields: ['cw'], 'query': q, default_operator: 'and' } },
+						],
+						minimum_should_match: 1,
+					},
+				});
+			}
+
+			const res = await (this.elasticsearch.search)({
+				index: this.elasticsearchNoteIndex + '*' as string,
+				body: {
+					query: esFilter,
+					sort: [{ createdAt: { order: 'desc' } }],
+				},
+				_source: ['id', 'createdAt'],
+				size: pagination.limit,
+			});
+
+			const noteIds = res.hits.hits.map((hit: any) => hit._id);
+			if (noteIds.length === 0) return [];
+			const [
+				userIdsWhoMeMuting,
+				userIdsWhoBlockingMe,
+			] = me ? await Promise.all([
+				this.cacheService.userMutingsCache.fetch(me.id),
+				this.cacheService.userBlockedCache.fetch(me.id),
+			]) : [new Set<string>(), new Set<string>()];
+			const notes = (await this.notesRepository.findBy({
+				id: In(noteIds),
+			})).filter(note => {
+				if (me && isUserRelated(note, userIdsWhoBlockingMe)) return false;
+				if (me && isUserRelated(note, userIdsWhoMeMuting)) return false;
+				return true;
+			});
+
 			return notes.sort((a, b) => a.id > b.id ? -1 : 1);
 		} else {
 			const query = this.queryService.makePaginationQuery(this.notesRepository.createQueryBuilder('note'), pagination.sinceId, pagination.untilId);

packages/backend/src/di-symbols.ts

@@ -7,6 +7,7 @@ export const DI = {
 	config: Symbol('config'),
 	db: Symbol('db'),
 	meilisearch: Symbol('meilisearch'),
+	elasticsearch: Symbol('elasticsearch'),
 	redis: Symbol('redis'),
 	redisForPub: Symbol('redisForPub'),
 	redisForSub: Symbol('redisForSub'),

pnpm-lock.yaml

@@ -86,6 +86,9 @@ importers:
       '@discordapp/twemoji':
         specifier: 15.0.3
         version: 15.0.3
+      '@elastic/elasticsearch':
+        specifier: ^8.14.0
+        version: 8.14.0
       '@fastify/accepts':
         specifier: 4.3.0
         version: 4.3.0
@@ -2199,6 +2202,14 @@ packages:
     resolution: {integrity: sha512-dBVuXR082gk3jsFp7Rd/JI4kytwGHecnCoTtXFb7DB6CNHp4rg5k1bhg0nWdLGLnOV71lmDzGQaLMy8iPLY0pw==}
     engines: {node: '>=10.0.0'}
 
+  '@elastic/elasticsearch@8.14.0':
+    resolution: {integrity: sha512-MGrgCI4y+Ozssf5Q2IkVJlqt5bUMnKIICG2qxeOfrJNrVugMCBCAQypyesmSSocAtNm8IX3LxfJ3jQlFHmKe2w==}
+    engines: {node: '>=18'}
+
+  '@elastic/transport@8.7.0':
+    resolution: {integrity: sha512-IqXT7a8DZPJtqP2qmX1I2QKmxYyN27kvSW4g6pInESE1SuGwZDp2FxHJ6W2kwmYOJwQdAt+2aWwzXO5jHo9l4A==}
+    engines: {node: '>=18'}
+
   '@emnapi/runtime@1.2.0':
     resolution: {integrity: sha512-bV21/9LQmcQeCPEg3BDFtvwL6cwiTMksYNWQQ4KOxCZikEGalWtenoZ0wCiukJINlGCIi2KXx01g4FoH/LxpzQ==}
 
@@ -2989,6 +3000,10 @@ packages:
   '@open-draft/until@2.1.0':
     resolution: {integrity: sha512-U69T3ItWHvLwGg5eJ0n3I62nWuE6ilHlmz7zM0npLBRvPRd7e6NYmg54vvRtP5mZG7kZqZCFVdsTWo7BPtBujg==}
 
+  '@opentelemetry/api@1.9.0':
+    resolution: {integrity: sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==}
+    engines: {node: '>=8.0.0'}
+
   '@peculiar/asn1-android@2.3.10':
     resolution: {integrity: sha512-z9Rx9cFJv7UUablZISe7uksNbFJCq13hO0yEAOoIpAymALTLlvUOSLnGiQS7okPaM5dP42oTLhezH6XDXRXjGw==}
 
@@ -10410,6 +10425,10 @@ packages:
     resolution: {integrity: sha512-72RFADWFqKmUb2hmmvNODKL3p9hcB6Gt2DOQMis1SEBaV6a4MH8soBvzg+95CYhCKPFedut2JY9bMfrDl9D23g==}
     engines: {node: '>=14.0'}
 
+  undici@6.19.2:
+    resolution: {integrity: sha512-JfjKqIauur3Q6biAtHJ564e3bWa8VvT+7cSiOJHFbX4Erv6CLGDpg8z+Fmg/1OI/47RA+GI2QZaF48SSaLvyBA==}
+    engines: {node: '>=18.17'}
+
   unicode-canonical-property-names-ecmascript@2.0.0:
     resolution: {integrity: sha512-yY5PpDlfVIU5+y/BSCxAJRBIS1Zc2dDG3Ujq+sR0U+JjUevW2JhocOF+soROYDSaAezOzOKuyyixhD6mBknSmQ==}
     engines: {node: '>=4'}
@@ -10689,6 +10708,9 @@ packages:
   vue-component-type-helpers@2.0.19:
     resolution: {integrity: sha512-cN3f1aTxxKo4lzNeQAkVopswuImUrb5Iurll9Gaw5cqpnbTAxtEMM1mgi6ou4X79OCyqYv1U1mzBHJkzmiK82w==}
 
+  vue-component-type-helpers@2.0.26:
+    resolution: {integrity: sha512-sO9qQ8oC520SW6kqlls0iqDak53gsTVSrYylajgjmkt1c0vcgjsGSy1KzlDrbEx8pm02IEYhlUkU5hCYf8rwtg==}
+
   vue-demi@0.14.7:
     resolution: {integrity: sha512-EOG8KXDQNwkJILkx/gPcoL/7vH+hORoBaKgGe+6W7VFMvCYJfmF2dGbvgDroVnI8LU7/kTu8mbjRZGBU1z9NTA==}
     engines: {node: '>=12'}
@@ -12469,6 +12491,25 @@ snapshots:
 
   '@discoveryjs/json-ext@0.5.7': {}
 
+  '@elastic/elasticsearch@8.14.0':
+    dependencies:
+      '@elastic/transport': 8.7.0
+      tslib: 2.6.2
+    transitivePeerDependencies:
+      - supports-color
+
+  '@elastic/transport@8.7.0':
+    dependencies:
+      '@opentelemetry/api': 1.9.0
+      debug: 4.3.4(supports-color@8.1.1)
+      hpagent: 1.2.0
+      ms: 2.1.3
+      secure-json-parse: 2.7.0
+      tslib: 2.6.2
+      undici: 6.19.2
+    transitivePeerDependencies:
+      - supports-color
+
   '@emnapi/runtime@1.2.0':
     dependencies:
       tslib: 2.6.2
@@ -13346,6 +13387,8 @@ snapshots:
 
   '@open-draft/until@2.1.0': {}
 
+  '@opentelemetry/api@1.9.0': {}
+
   '@peculiar/asn1-android@2.3.10':
     dependencies:
       '@peculiar/asn1-schema': 2.3.8
@@ -14738,7 +14781,7 @@ snapshots:
       ts-dedent: 2.2.0
       type-fest: 2.19.0
       vue: 3.4.15(typescript@5.4.5)
-      vue-component-type-helpers: 2.0.19
+      vue-component-type-helpers: 2.0.26
     transitivePeerDependencies:
       - encoding
       - prettier
@@ -22388,6 +22431,8 @@ snapshots:
     dependencies:
       '@fastify/busboy': 2.1.1
 
+  undici@6.19.2: {}
+
   unicode-canonical-property-names-ecmascript@2.0.0: {}
 
   unicode-match-property-ecmascript@2.0.0:
@@ -22667,6 +22712,8 @@ snapshots:
 
   vue-component-type-helpers@2.0.19: {}
 
+  vue-component-type-helpers@2.0.26: {}
+
   vue-demi@0.14.7(vue@3.4.15(typescript@5.4.5)):
     dependencies:
       vue: 3.4.15(typescript@5.4.5)