mirror of
https://github.com/hotomoe/hotomoe
synced 2024-12-03 01:08:15 +09:00
feat(backend): elasticsearchで検索できるように (MisskeyIO#661)
Co-authored-by: 皐月なふ (Nafu Satsuki) <satsuki@nafusoft.dev>
This commit is contained in:
まっちゃてぃー。
GitHub
parent
0375599e50
commit
a77291be57
@ -72,6 +72,7 @@
|
||||
"@bull-board/fastify": "5.18.1",
|
||||
"@bull-board/ui": "5.18.1",
|
||||
"@discordapp/twemoji": "15.0.3",
|
||||
"@elastic/elasticsearch": "^8.14.0",
|
||||
"@fastify/accepts": "4.3.0",
|
||||
"@fastify/cookie": "9.3.1",
|
||||
"@fastify/cors": "9.0.1",
|
||||
|
||||
@ -8,6 +8,7 @@ import { Global, Inject, Module } from '@nestjs/common';
|
||||
import * as Redis from 'ioredis';
|
||||
import { DataSource } from 'typeorm';
|
||||
import { MeiliSearch } from 'meilisearch';
|
||||
import { Client as ElasticSearch } from '@elastic/elasticsearch';
|
||||
import { DI } from './di-symbols.js';
|
||||
import { Config, loadConfig } from './config.js';
|
||||
import { createPostgresDataSource } from './postgres.js';
|
||||
@ -44,6 +45,30 @@ const $meilisearch: Provider = {
|
||||
inject: [DI.config],
|
||||
};
|
||||
|
||||
const $elasticsearch: Provider = {
|
||||
provide: DI.elasticsearch,
|
||||
useFactory: (config: Config) => {
|
||||
if (config.elasticsearch) {
|
||||
return new ElasticSearch({
|
||||
nodes: {
|
||||
url: new URL(`${config.elasticsearch.ssl ? 'https' : 'http'}://${config.elasticsearch.host}:${config.elasticsearch.port}`),
|
||||
ssl: {
|
||||
rejectUnauthorized: config.elasticsearch.rejectUnauthorized,
|
||||
},
|
||||
},
|
||||
auth: (config.elasticsearch.user && config.elasticsearch.pass) ? {
|
||||
username: config.elasticsearch.user,
|
||||
password: config.elasticsearch.pass,
|
||||
} : undefined,
|
||||
pingTimeout: 30000,
|
||||
});
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
},
|
||||
inject: [DI.config],
|
||||
};
|
||||
|
||||
const $redis: Provider = {
|
||||
provide: DI.redis,
|
||||
useFactory: (config: Config) => {
|
||||
@ -160,8 +185,8 @@ const $redisForTimelines: Provider = {
|
||||
@Global()
|
||||
@Module({
|
||||
imports: [RepositoryModule],
|
||||
providers: [$config, $db, $meilisearch, $redis, $redisForPub, $redisForSub, $redisForTimelines],
|
||||
exports: [$config, $db, $meilisearch, $redis, $redisForPub, $redisForSub, $redisForTimelines, RepositoryModule],
|
||||
providers: [$config, $db, $meilisearch, $elasticsearch, $redis, $redisForPub, $redisForSub, $redisForTimelines],
|
||||
exports: [$config, $db, $meilisearch, $elasticsearch, $redis, $redisForPub, $redisForSub, $redisForTimelines, RepositoryModule],
|
||||
})
|
||||
export class GlobalModule implements OnApplicationShutdown {
|
||||
constructor(
|
||||
|
||||
@ -66,6 +66,16 @@ type Source = {
|
||||
scope?: 'local' | 'global' | string[];
|
||||
};
|
||||
|
||||
elasticsearch?: {
|
||||
host: string;
|
||||
port: string;
|
||||
user: string;
|
||||
pass: string;
|
||||
ssl?: boolean;
|
||||
rejectUnauthorized?: boolean;
|
||||
index: string;
|
||||
};
|
||||
|
||||
skebStatus?: {
|
||||
method: string;
|
||||
endpoint: string;
|
||||
@ -149,6 +159,15 @@ export type Config = {
|
||||
index: string;
|
||||
scope?: 'local' | 'global' | string[];
|
||||
} | undefined;
|
||||
elasticsearch: {
|
||||
host: string;
|
||||
port: string;
|
||||
user: string;
|
||||
pass: string;
|
||||
ssl?: boolean;
|
||||
rejectUnauthorized?: boolean;
|
||||
index: string;
|
||||
} | undefined;
|
||||
skebStatus: {
|
||||
method: string;
|
||||
endpoint: string;
|
||||
@ -272,6 +291,7 @@ export function loadConfig(): Config {
|
||||
dbReplications: config.dbReplications,
|
||||
dbSlaves: config.dbSlaves,
|
||||
meilisearch: config.meilisearch,
|
||||
elasticsearch: config.elasticsearch,
|
||||
redis,
|
||||
redisForPubsub: config.redisForPubsub ? convertRedisOptions(config.redisForPubsub, host) : redis,
|
||||
redisForSystemQueue: config.redisForSystemQueue ? convertRedisOptions(config.redisForSystemQueue, host) : redisForJobQueue,
|
||||
|
||||
@ -8,6 +8,7 @@ import { In } from 'typeorm';
|
||||
import { DI } from '@/di-symbols.js';
|
||||
import type { Config } from '@/config.js';
|
||||
import { bindThis } from '@/decorators.js';
|
||||
import { LoggerService } from '@/core/LoggerService.js';
|
||||
import { MiNote } from '@/models/Note.js';
|
||||
import { MiUser } from '@/models/_.js';
|
||||
import type { NotesRepository } from '@/models/_.js';
|
||||
@ -16,7 +17,9 @@ import { isUserRelated } from '@/misc/is-user-related.js';
|
||||
import { CacheService } from '@/core/CacheService.js';
|
||||
import { QueryService } from '@/core/QueryService.js';
|
||||
import { IdService } from '@/core/IdService.js';
|
||||
import type Logger from '@/logger.js';
|
||||
import type { Index, MeiliSearch } from 'meilisearch';
|
||||
import type { Client as ElasticSearch } from '@elastic/elasticsearch';
|
||||
|
||||
type K = string;
|
||||
type V = string | number | boolean;
|
||||
@ -65,6 +68,8 @@ function compileQuery(q: Q): string {
|
||||
export class SearchService {
|
||||
private readonly meilisearchIndexScope: 'local' | 'global' | string[] = 'local';
|
||||
private meilisearchNoteIndex: Index | null = null;
|
||||
private elasticsearchNoteIndex: string | null = null;
|
||||
private logger: Logger;
|
||||
|
||||
constructor(
|
||||
@Inject(DI.config)
|
||||
@ -73,15 +78,24 @@ export class SearchService {
|
||||
@Inject(DI.meilisearch)
|
||||
private meilisearch: MeiliSearch | null,
|
||||
|
||||
@Inject(DI.elasticsearch)
|
||||
private elasticsearch: ElasticSearch | null,
|
||||
|
||||
@Inject(DI.notesRepository)
|
||||
private notesRepository: NotesRepository,
|
||||
|
||||
private cacheService: CacheService,
|
||||
private queryService: QueryService,
|
||||
private idService: IdService,
|
||||
private loggerService: LoggerService,
|
||||
) {
|
||||
this.logger = this.loggerService.getLogger('note:search');
|
||||
|
||||
if (meilisearch) {
|
||||
this.meilisearchNoteIndex = meilisearch.index(`${config.meilisearch!.index}---notes`);
|
||||
if (config.meilisearch?.scope) {
|
||||
this.meilisearchIndexScope = config.meilisearch.scope;
|
||||
}
|
||||
/*this.meilisearchNoteIndex.updateSettings({
|
||||
searchableAttributes: [
|
||||
'text',
|
||||
@ -104,10 +118,52 @@ export class SearchService {
|
||||
maxTotalHits: 10000,
|
||||
},
|
||||
});*/
|
||||
}
|
||||
|
||||
if (config.meilisearch?.scope) {
|
||||
this.meilisearchIndexScope = config.meilisearch.scope;
|
||||
} else if (this.elasticsearch) {
|
||||
this.elasticsearchNoteIndex = `${config.elasticsearch!.index}---notes`;
|
||||
this.elasticsearch.indices.exists({
|
||||
index: this.elasticsearchNoteIndex,
|
||||
}).then((indexExists) => {
|
||||
if (!indexExists) {
|
||||
this.elasticsearch?.indices.create(
|
||||
{
|
||||
index: this.elasticsearchNoteIndex + `-${new Date().toISOString().slice(0, 7).replace(/-/g, '')}`,
|
||||
mappings: {
|
||||
properties: {
|
||||
text: { type: 'text' },
|
||||
cw: { type: 'text' },
|
||||
createdAt: { type: 'long' },
|
||||
userId: { type: 'keyword' },
|
||||
userHost: { type: 'keyword' },
|
||||
channelId: { type: 'keyword' },
|
||||
tags: { type: 'keyword' },
|
||||
},
|
||||
},
|
||||
settings: {
|
||||
index: {
|
||||
analysis: {
|
||||
tokenizer: {
|
||||
kuromoji: {
|
||||
type: 'kuromoji_tokenizer',
|
||||
mode: 'search',
|
||||
},
|
||||
},
|
||||
analyzer: {
|
||||
kuromoji_analyzer: {
|
||||
type: 'custom',
|
||||
tokenizer: 'kuromoji',
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
).catch((error) => {
|
||||
this.logger.error(error);
|
||||
});
|
||||
}
|
||||
}).catch((error) => {
|
||||
this.logger.error('Error while checking if index exists', error);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@ -144,6 +200,23 @@ export class SearchService {
|
||||
}], {
|
||||
primaryKey: 'id',
|
||||
});
|
||||
} else if (this.elasticsearch) {
|
||||
const body = {
|
||||
createdAt: this.idService.parse(note.id).date.getTime(),
|
||||
userId: note.userId,
|
||||
userHost: note.userHost,
|
||||
channelId: note.channelId,
|
||||
cw: note.cw,
|
||||
text: note.text,
|
||||
tags: note.tags,
|
||||
};
|
||||
await this.elasticsearch.index({
|
||||
index: this.elasticsearchNoteIndex + `-${new Date().toISOString().slice(0, 7).replace(/-/g, '')}` as string,
|
||||
id: note.id,
|
||||
body: body,
|
||||
}).catch((error) => {
|
||||
console.error(error);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@ -204,6 +277,67 @@ export class SearchService {
|
||||
if (me && isUserRelated(note, userIdsWhoMeMuting)) return false;
|
||||
return true;
|
||||
});
|
||||
return notes.sort((a, b) => a.id > b.id ? -1 : 1);
|
||||
} else if (this.elasticsearch) {
|
||||
const esFilter: any = {
|
||||
bool: {
|
||||
must: [],
|
||||
},
|
||||
};
|
||||
|
||||
if (pagination.untilId) esFilter.bool.must.push({ range: { createdAt: { lt: this.idService.parse(pagination.untilId).date.getTime() } } });
|
||||
if (pagination.sinceId) esFilter.bool.must.push({ range: { createdAt: { gt: this.idService.parse(pagination.sinceId).date.getTime() } } });
|
||||
if (opts.userId) esFilter.bool.must.push({ term: { userId: opts.userId } });
|
||||
if (opts.channelId) esFilter.bool.must.push({ term: { channelId: opts.channelId } });
|
||||
if (opts.host) {
|
||||
if (opts.host === '.') {
|
||||
esFilter.bool.must.push({ bool: { must_not: [{ exists: { field: 'userHost' } }] } });
|
||||
} else {
|
||||
esFilter.bool.must.push({ term: { userHost: opts.host } });
|
||||
}
|
||||
}
|
||||
|
||||
if (q !== '') {
|
||||
esFilter.bool.must.push({
|
||||
bool: {
|
||||
should: [
|
||||
{ wildcard: { 'text': { value: q } } },
|
||||
{ simple_query_string: { fields: ['text'], 'query': q, default_operator: 'and' } },
|
||||
{ wildcard: { 'cw': { value: q } } },
|
||||
{ simple_query_string: { fields: ['cw'], 'query': q, default_operator: 'and' } },
|
||||
],
|
||||
minimum_should_match: 1,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
const res = await (this.elasticsearch.search)({
|
||||
index: this.elasticsearchNoteIndex + '*' as string,
|
||||
body: {
|
||||
query: esFilter,
|
||||
sort: [{ createdAt: { order: 'desc' } }],
|
||||
},
|
||||
_source: ['id', 'createdAt'],
|
||||
size: pagination.limit,
|
||||
});
|
||||
|
||||
const noteIds = res.hits.hits.map((hit: any) => hit._id);
|
||||
if (noteIds.length === 0) return [];
|
||||
const [
|
||||
userIdsWhoMeMuting,
|
||||
userIdsWhoBlockingMe,
|
||||
] = me ? await Promise.all([
|
||||
this.cacheService.userMutingsCache.fetch(me.id),
|
||||
this.cacheService.userBlockedCache.fetch(me.id),
|
||||
]) : [new Set<string>(), new Set<string>()];
|
||||
const notes = (await this.notesRepository.findBy({
|
||||
id: In(noteIds),
|
||||
})).filter(note => {
|
||||
if (me && isUserRelated(note, userIdsWhoBlockingMe)) return false;
|
||||
if (me && isUserRelated(note, userIdsWhoMeMuting)) return false;
|
||||
return true;
|
||||
});
|
||||
|
||||
return notes.sort((a, b) => a.id > b.id ? -1 : 1);
|
||||
} else {
|
||||
const query = this.queryService.makePaginationQuery(this.notesRepository.createQueryBuilder('note'), pagination.sinceId, pagination.untilId);
|
||||
|
||||
@ -7,6 +7,7 @@ export const DI = {
|
||||
config: Symbol('config'),
|
||||
db: Symbol('db'),
|
||||
meilisearch: Symbol('meilisearch'),
|
||||
elasticsearch: Symbol('elasticsearch'),
|
||||
redis: Symbol('redis'),
|
||||
redisForPub: Symbol('redisForPub'),
|
||||
redisForSub: Symbol('redisForSub'),
|
||||
|
||||
@ -86,6 +86,9 @@ importers:
|
||||
'@discordapp/twemoji':
|
||||
specifier: 15.0.3
|
||||
version: 15.0.3
|
||||
'@elastic/elasticsearch':
|
||||
specifier: ^8.14.0
|
||||
version: 8.14.0
|
||||
'@fastify/accepts':
|
||||
specifier: 4.3.0
|
||||
version: 4.3.0
|
||||
@ -2199,6 +2202,14 @@ packages:
|
||||
resolution: {integrity: sha512-dBVuXR082gk3jsFp7Rd/JI4kytwGHecnCoTtXFb7DB6CNHp4rg5k1bhg0nWdLGLnOV71lmDzGQaLMy8iPLY0pw==}
|
||||
engines: {node: '>=10.0.0'}
|
||||
|
||||
'@elastic/elasticsearch@8.14.0':
|
||||
resolution: {integrity: sha512-MGrgCI4y+Ozssf5Q2IkVJlqt5bUMnKIICG2qxeOfrJNrVugMCBCAQypyesmSSocAtNm8IX3LxfJ3jQlFHmKe2w==}
|
||||
engines: {node: '>=18'}
|
||||
|
||||
'@elastic/transport@8.7.0':
|
||||
resolution: {integrity: sha512-IqXT7a8DZPJtqP2qmX1I2QKmxYyN27kvSW4g6pInESE1SuGwZDp2FxHJ6W2kwmYOJwQdAt+2aWwzXO5jHo9l4A==}
|
||||
engines: {node: '>=18'}
|
||||
|
||||
'@emnapi/runtime@1.2.0':
|
||||
resolution: {integrity: sha512-bV21/9LQmcQeCPEg3BDFtvwL6cwiTMksYNWQQ4KOxCZikEGalWtenoZ0wCiukJINlGCIi2KXx01g4FoH/LxpzQ==}
|
||||
|
||||
@ -2989,6 +3000,10 @@ packages:
|
||||
'@open-draft/until@2.1.0':
|
||||
resolution: {integrity: sha512-U69T3ItWHvLwGg5eJ0n3I62nWuE6ilHlmz7zM0npLBRvPRd7e6NYmg54vvRtP5mZG7kZqZCFVdsTWo7BPtBujg==}
|
||||
|
||||
'@opentelemetry/api@1.9.0':
|
||||
resolution: {integrity: sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==}
|
||||
engines: {node: '>=8.0.0'}
|
||||
|
||||
'@peculiar/asn1-android@2.3.10':
|
||||
resolution: {integrity: sha512-z9Rx9cFJv7UUablZISe7uksNbFJCq13hO0yEAOoIpAymALTLlvUOSLnGiQS7okPaM5dP42oTLhezH6XDXRXjGw==}
|
||||
|
||||
@ -10410,6 +10425,10 @@ packages:
|
||||
resolution: {integrity: sha512-72RFADWFqKmUb2hmmvNODKL3p9hcB6Gt2DOQMis1SEBaV6a4MH8soBvzg+95CYhCKPFedut2JY9bMfrDl9D23g==}
|
||||
engines: {node: '>=14.0'}
|
||||
|
||||
undici@6.19.2:
|
||||
resolution: {integrity: sha512-JfjKqIauur3Q6biAtHJ564e3bWa8VvT+7cSiOJHFbX4Erv6CLGDpg8z+Fmg/1OI/47RA+GI2QZaF48SSaLvyBA==}
|
||||
engines: {node: '>=18.17'}
|
||||
|
||||
unicode-canonical-property-names-ecmascript@2.0.0:
|
||||
resolution: {integrity: sha512-yY5PpDlfVIU5+y/BSCxAJRBIS1Zc2dDG3Ujq+sR0U+JjUevW2JhocOF+soROYDSaAezOzOKuyyixhD6mBknSmQ==}
|
||||
engines: {node: '>=4'}
|
||||
@ -10689,6 +10708,9 @@ packages:
|
||||
vue-component-type-helpers@2.0.19:
|
||||
resolution: {integrity: sha512-cN3f1aTxxKo4lzNeQAkVopswuImUrb5Iurll9Gaw5cqpnbTAxtEMM1mgi6ou4X79OCyqYv1U1mzBHJkzmiK82w==}
|
||||
|
||||
vue-component-type-helpers@2.0.26:
|
||||
resolution: {integrity: sha512-sO9qQ8oC520SW6kqlls0iqDak53gsTVSrYylajgjmkt1c0vcgjsGSy1KzlDrbEx8pm02IEYhlUkU5hCYf8rwtg==}
|
||||
|
||||
vue-demi@0.14.7:
|
||||
resolution: {integrity: sha512-EOG8KXDQNwkJILkx/gPcoL/7vH+hORoBaKgGe+6W7VFMvCYJfmF2dGbvgDroVnI8LU7/kTu8mbjRZGBU1z9NTA==}
|
||||
engines: {node: '>=12'}
|
||||
@ -12469,6 +12491,25 @@ snapshots:
|
||||
|
||||
'@discoveryjs/json-ext@0.5.7': {}
|
||||
|
||||
'@elastic/elasticsearch@8.14.0':
|
||||
dependencies:
|
||||
'@elastic/transport': 8.7.0
|
||||
tslib: 2.6.2
|
||||
transitivePeerDependencies:
|
||||
- supports-color
|
||||
|
||||
'@elastic/transport@8.7.0':
|
||||
dependencies:
|
||||
'@opentelemetry/api': 1.9.0
|
||||
debug: 4.3.4(supports-color@8.1.1)
|
||||
hpagent: 1.2.0
|
||||
ms: 2.1.3
|
||||
secure-json-parse: 2.7.0
|
||||
tslib: 2.6.2
|
||||
undici: 6.19.2
|
||||
transitivePeerDependencies:
|
||||
- supports-color
|
||||
|
||||
'@emnapi/runtime@1.2.0':
|
||||
dependencies:
|
||||
tslib: 2.6.2
|
||||
@ -13346,6 +13387,8 @@ snapshots:
|
||||
|
||||
'@open-draft/until@2.1.0': {}
|
||||
|
||||
'@opentelemetry/api@1.9.0': {}
|
||||
|
||||
'@peculiar/asn1-android@2.3.10':
|
||||
dependencies:
|
||||
'@peculiar/asn1-schema': 2.3.8
|
||||
@ -14738,7 +14781,7 @@ snapshots:
|
||||
ts-dedent: 2.2.0
|
||||
type-fest: 2.19.0
|
||||
vue: 3.4.15(typescript@5.4.5)
|
||||
vue-component-type-helpers: 2.0.19
|
||||
vue-component-type-helpers: 2.0.26
|
||||
transitivePeerDependencies:
|
||||
- encoding
|
||||
- prettier
|
||||
@ -22388,6 +22431,8 @@ snapshots:
|
||||
dependencies:
|
||||
'@fastify/busboy': 2.1.1
|
||||
|
||||
undici@6.19.2: {}
|
||||
|
||||
unicode-canonical-property-names-ecmascript@2.0.0: {}
|
||||
|
||||
unicode-match-property-ecmascript@2.0.0:
|
||||
@ -22667,6 +22712,8 @@ snapshots:
|
||||
|
||||
vue-component-type-helpers@2.0.19: {}
|
||||
|
||||
vue-component-type-helpers@2.0.26: {}
|
||||
|
||||
vue-demi@0.14.7(vue@3.4.15(typescript@5.4.5)):
|
||||
dependencies:
|
||||
vue: 3.4.15(typescript@5.4.5)
|
||||
|
||||
Loading…
Reference in New Issue
Block a user