mirror of
https://iceshrimp.dev/iceshrimp/iceshrimp
synced 2024-11-27 14:28:08 +09:00
[backend] Remove external search backends
This commit is contained in:
Laura Hausmann
parent
cdec8c4efd
commit
9b2e966c19
44
.pnp.cjs
generated
44
.pnp.cjs
generated
@ -1513,19 +1513,6 @@ const RAW_RUNTIME_STATE =
|
||||
"linkType": "HARD"\
|
||||
}]\
|
||||
]],\
|
||||
["@elastic/elasticsearch", [\
|
||||
["npm:7.17.0", {\
|
||||
"packageLocation": "./.yarn/cache/@elastic-elasticsearch-npm-7.17.0-f4178789c0-d54330ce50.zip/node_modules/@elastic/elasticsearch/",\
|
||||
"packageDependencies": [\
|
||||
["@elastic/elasticsearch", "npm:7.17.0"],\
|
||||
["debug", "virtual:ac3d8e680759ce54399273724d44e041d6c9b73454d191d411a8c44bb27e22f02aaf6ed9d3ad0ac1c298eac4833cff369c9c7b84c573016112c4f84be2cd8543#npm:4.3.4"],\
|
||||
["hpagent", "npm:0.1.2"],\
|
||||
["ms", "npm:2.1.3"],\
|
||||
["secure-json-parse", "npm:2.7.0"]\
|
||||
],\
|
||||
"linkType": "HARD"\
|
||||
}]\
|
||||
]],\
|
||||
["@es-joy/jsdoccomment", [\
|
||||
["npm:0.39.4", {\
|
||||
"packageLocation": "./.yarn/cache/@es-joy-jsdoccomment-npm-0.39.4-48cba32ec8-10d18c2de8.zip/node_modules/@es-joy/jsdoccomment/",\
|
||||
@ -6991,7 +6978,6 @@ const RAW_RUNTIME_STATE =
|
||||
["@bull-board/koa", "npm:5.6.0"],\
|
||||
["@bull-board/ui", "npm:5.6.0"],\
|
||||
["@discordapp/twemoji", "npm:14.1.2"],\
|
||||
["@elastic/elasticsearch", "npm:7.17.0"],\
|
||||
["@koa/cors", "npm:3.4.3"],\
|
||||
["@koa/multer", "virtual:aa59773ac87791c4813d53447077fcf8a847d6de5a301d34dc31286584b1dbb26d30d3adb5b4c41c1e8aea04371e926fda05c09c6253647c432e11d872a304ba#npm:3.0.2"],\
|
||||
["@koa/router", "npm:9.0.1"],\
|
||||
@ -7105,7 +7091,6 @@ const RAW_RUNTIME_STATE =
|
||||
["koa-send", "npm:5.0.1"],\
|
||||
["koa-slow", "npm:2.1.0"],\
|
||||
["koa-views", "virtual:aa59773ac87791c4813d53447077fcf8a847d6de5a301d34dc31286584b1dbb26d30d3adb5b4c41c1e8aea04371e926fda05c09c6253647c432e11d872a304ba#npm:7.0.2"],\
|
||||
["meilisearch", "npm:0.33.0"],\
|
||||
["mfm-js", "npm:0.23.3"],\
|
||||
["mime-types", "npm:2.1.35"],\
|
||||
["mocha", "npm:10.2.0"],\
|
||||
@ -7140,7 +7125,6 @@ const RAW_RUNTIME_STATE =
|
||||
["seedrandom", "npm:3.0.5"],\
|
||||
["semver", "npm:7.5.4"],\
|
||||
["sharp", "npm:0.32.1"],\
|
||||
["sonic-channel", "npm:1.3.1"],\
|
||||
["strict-event-emitter-types", "npm:2.0.0"],\
|
||||
["stringz", "npm:2.1.0"],\
|
||||
["summaly", "npm:2.7.0"],\
|
||||
@ -17519,16 +17503,6 @@ const RAW_RUNTIME_STATE =
|
||||
"linkType": "HARD"\
|
||||
}]\
|
||||
]],\
|
||||
["meilisearch", [\
|
||||
["npm:0.33.0", {\
|
||||
"packageLocation": "./.yarn/cache/meilisearch-npm-0.33.0-a8742f194e-d2aff57b3d.zip/node_modules/meilisearch/",\
|
||||
"packageDependencies": [\
|
||||
["meilisearch", "npm:0.33.0"],\
|
||||
["cross-fetch", "npm:3.1.8"]\
|
||||
],\
|
||||
"linkType": "HARD"\
|
||||
}]\
|
||||
]],\
|
||||
["meow", [\
|
||||
["npm:9.0.0", {\
|
||||
"packageLocation": "./.yarn/cache/meow-npm-9.0.0-8b2707248e-3d0f199b9c.zip/node_modules/meow/",\
|
||||
@ -21742,15 +21716,6 @@ const RAW_RUNTIME_STATE =
|
||||
"linkType": "HARD"\
|
||||
}]\
|
||||
]],\
|
||||
["secure-json-parse", [\
|
||||
["npm:2.7.0", {\
|
||||
"packageLocation": "./.yarn/cache/secure-json-parse-npm-2.7.0-d5b89b0a3e-9743865870.zip/node_modules/secure-json-parse/",\
|
||||
"packageDependencies": [\
|
||||
["secure-json-parse", "npm:2.7.0"]\
|
||||
],\
|
||||
"linkType": "HARD"\
|
||||
}]\
|
||||
]],\
|
||||
["seedrandom", [\
|
||||
["npm:2.4.2", {\
|
||||
"packageLocation": "./.yarn/cache/seedrandom-npm-2.4.2-b435b54ae9-a0b6707cb7.zip/node_modules/seedrandom/",\
|
||||
@ -22159,15 +22124,6 @@ const RAW_RUNTIME_STATE =
|
||||
"linkType": "HARD"\
|
||||
}]\
|
||||
]],\
|
||||
["sonic-channel", [\
|
||||
["npm:1.3.1", {\
|
||||
"packageLocation": "./.yarn/cache/sonic-channel-npm-1.3.1-8f21a07e24-ee849863a3.zip/node_modules/sonic-channel/",\
|
||||
"packageDependencies": [\
|
||||
["sonic-channel", "npm:1.3.1"]\
|
||||
],\
|
||||
"linkType": "HARD"\
|
||||
}]\
|
||||
]],\
|
||||
["sort-keys", [\
|
||||
["npm:1.1.2", {\
|
||||
"packageLocation": "./.yarn/cache/sort-keys-npm-1.1.2-2ac0ab2d94-0ac2ea2327.zip/node_modules/sort-keys/",\
|
||||
|
||||
BIN
.yarn/cache/@elastic-elasticsearch-npm-7.17.0-f4178789c0-d54330ce50.zip
(Stored with Git LFS)
vendored
BIN
.yarn/cache/@elastic-elasticsearch-npm-7.17.0-f4178789c0-d54330ce50.zip
(Stored with Git LFS)
vendored
Binary file not shown.
BIN
.yarn/cache/meilisearch-npm-0.33.0-a8742f194e-d2aff57b3d.zip
(Stored with Git LFS)
vendored
BIN
.yarn/cache/meilisearch-npm-0.33.0-a8742f194e-d2aff57b3d.zip
(Stored with Git LFS)
vendored
Binary file not shown.
BIN
.yarn/cache/secure-json-parse-npm-2.7.0-d5b89b0a3e-9743865870.zip
(Stored with Git LFS)
vendored
BIN
.yarn/cache/secure-json-parse-npm-2.7.0-d5b89b0a3e-9743865870.zip
(Stored with Git LFS)
vendored
Binary file not shown.
BIN
.yarn/cache/sonic-channel-npm-1.3.1-8f21a07e24-ee849863a3.zip
(Stored with Git LFS)
vendored
BIN
.yarn/cache/sonic-channel-npm-1.3.1-8f21a07e24-ee849863a3.zip
(Stored with Git LFS)
vendored
Binary file not shown.
@ -27,7 +27,6 @@
|
||||
"@bull-board/koa": "5.6.0",
|
||||
"@bull-board/ui": "5.6.0",
|
||||
"@discordapp/twemoji": "14.1.2",
|
||||
"@elastic/elasticsearch": "7.17.0",
|
||||
"@koa/cors": "3.4.3",
|
||||
"@koa/multer": "3.0.2",
|
||||
"@koa/router": "9.0.1",
|
||||
@ -88,7 +87,6 @@
|
||||
"koa-send": "5.0.1",
|
||||
"koa-slow": "2.1.0",
|
||||
"koa-views": "7.0.2",
|
||||
"meilisearch": "0.33.0",
|
||||
"mfm-js": "0.23.3",
|
||||
"mime-types": "2.1.35",
|
||||
"msgpackr": "1.9.5",
|
||||
@ -121,7 +119,6 @@
|
||||
"seedrandom": "^3.0.5",
|
||||
"semver": "7.5.4",
|
||||
"sharp": "0.32.1",
|
||||
"sonic-channel": "^1.3.1",
|
||||
"stringz": "2.1.0",
|
||||
"summaly": "2.7.0",
|
||||
"syslog-pro": "1.0.0",
|
||||
|
||||
@ -27,27 +27,6 @@ export type Source = {
|
||||
user?: string;
|
||||
tls?: { [y: string]: string };
|
||||
};
|
||||
elasticsearch: {
|
||||
host: string;
|
||||
port: number;
|
||||
ssl?: boolean;
|
||||
user?: string;
|
||||
pass?: string;
|
||||
index?: string;
|
||||
};
|
||||
sonic: {
|
||||
host: string;
|
||||
port: number;
|
||||
auth?: string;
|
||||
collection?: string;
|
||||
bucket?: string;
|
||||
};
|
||||
meilisearch: {
|
||||
host: string;
|
||||
port: number;
|
||||
apiKey?: string;
|
||||
ssl: boolean;
|
||||
};
|
||||
|
||||
mediaCleanup?: {
|
||||
cron?: boolean;
|
||||
|
||||
@ -2,7 +2,6 @@ import si from "systeminformation";
|
||||
import Xev from "xev";
|
||||
import * as osUtils from "os-utils";
|
||||
import { fetchMeta } from "@/misc/fetch-meta.js";
|
||||
import meilisearch from "../db/meilisearch.js";
|
||||
|
||||
const ev = new Xev();
|
||||
|
||||
@ -30,7 +29,6 @@ export default function () {
|
||||
const memStats = await mem();
|
||||
const netStats = await net();
|
||||
const fsStats = await fs();
|
||||
const meilisearchStats = await meilisearchStatus();
|
||||
|
||||
const stats = {
|
||||
cpu: roundCpu(cpu),
|
||||
@ -46,8 +44,7 @@ export default function () {
|
||||
fs: {
|
||||
r: round(Math.max(0, fsStats.rIO_sec ?? 0)),
|
||||
w: round(Math.max(0, fsStats.wIO_sec ?? 0)),
|
||||
},
|
||||
meilisearch: meilisearchStats,
|
||||
}
|
||||
};
|
||||
ev.emit("serverStats", stats);
|
||||
log.unshift(stats);
|
||||
@ -86,16 +83,3 @@ async function fs() {
|
||||
const data = await si.disksIO().catch(() => ({ rIO_sec: 0, wIO_sec: 0 }));
|
||||
return data || { rIO_sec: 0, wIO_sec: 0 };
|
||||
}
|
||||
|
||||
// MEILI STAT
|
||||
async function meilisearchStatus() {
|
||||
if (meilisearch) {
|
||||
return meilisearch.serverStats();
|
||||
} else {
|
||||
return {
|
||||
health: "unconfigured",
|
||||
size: 0,
|
||||
indexed_count: 0,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,65 +0,0 @@
|
||||
import * as elasticsearch from "@elastic/elasticsearch";
|
||||
import config from "@/config/index.js";
|
||||
|
||||
const index = {
|
||||
settings: {
|
||||
analysis: {
|
||||
analyzer: {
|
||||
ngram: {
|
||||
tokenizer: "ngram",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
mappings: {
|
||||
properties: {
|
||||
text: {
|
||||
type: "text",
|
||||
index: true,
|
||||
analyzer: "ngram",
|
||||
},
|
||||
userId: {
|
||||
type: "keyword",
|
||||
index: true,
|
||||
},
|
||||
userHost: {
|
||||
type: "keyword",
|
||||
index: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
// Init ElasticSearch connection
|
||||
const client = config.elasticsearch
|
||||
? new elasticsearch.Client({
|
||||
node: `${config.elasticsearch.ssl ? "https://" : "http://"}${
|
||||
config.elasticsearch.host
|
||||
}:${config.elasticsearch.port}`,
|
||||
auth:
|
||||
config.elasticsearch.user && config.elasticsearch.pass
|
||||
? {
|
||||
username: config.elasticsearch.user,
|
||||
password: config.elasticsearch.pass,
|
||||
}
|
||||
: undefined,
|
||||
pingTimeout: 30000,
|
||||
})
|
||||
: null;
|
||||
|
||||
if (client) {
|
||||
client.indices
|
||||
.exists({
|
||||
index: config.elasticsearch.index || "misskey_note",
|
||||
})
|
||||
.then((exist) => {
|
||||
if (!exist.body) {
|
||||
client.indices.create({
|
||||
index: config.elasticsearch.index || "misskey_note",
|
||||
body: index,
|
||||
});
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
export default client;
|
||||
@ -1,411 +0,0 @@
|
||||
import { Health, Index, MeiliSearch, Stats } from "meilisearch";
|
||||
import { dbLogger } from "./logger.js";
|
||||
|
||||
import config from "@/config/index.js";
|
||||
import { Note } from "@/models/entities/note.js";
|
||||
import * as url from "url";
|
||||
import { ILocalUser } from "@/models/entities/user.js";
|
||||
import { Followings, Users } from "@/models/index.js";
|
||||
|
||||
const logger = dbLogger.createSubLogger("meilisearch", "gray", false);
|
||||
|
||||
let posts: Index;
|
||||
let client: MeiliSearch;
|
||||
|
||||
const hasConfig =
|
||||
config.meilisearch &&
|
||||
(config.meilisearch.host ||
|
||||
config.meilisearch.port ||
|
||||
config.meilisearch.apiKey);
|
||||
|
||||
if (hasConfig) {
|
||||
const host = hasConfig ? config.meilisearch.host ?? "localhost" : "";
|
||||
const port = hasConfig ? config.meilisearch.port ?? 7700 : 0;
|
||||
const auth = hasConfig ? config.meilisearch.apiKey ?? "" : "";
|
||||
const ssl = hasConfig ? config.meilisearch.ssl ?? false : false;
|
||||
|
||||
logger.info("Connecting to MeiliSearch");
|
||||
|
||||
client = new MeiliSearch({
|
||||
host: `${ssl ? "https" : "http"}://${host}:${port}`,
|
||||
apiKey: auth,
|
||||
});
|
||||
|
||||
posts = client.index("posts");
|
||||
|
||||
posts
|
||||
.updateSearchableAttributes(["text"])
|
||||
.catch((e) =>
|
||||
logger.error(`Setting searchable attr failed, searches won't work: ${e}`),
|
||||
);
|
||||
|
||||
posts
|
||||
.updateFilterableAttributes([
|
||||
"userName",
|
||||
"userHost",
|
||||
"mediaAttachment",
|
||||
"createdAt",
|
||||
"userId",
|
||||
])
|
||||
.catch((e) =>
|
||||
logger.error(
|
||||
`Setting filterable attr failed, advanced searches won't work: ${e}`,
|
||||
),
|
||||
);
|
||||
|
||||
posts
|
||||
.updateSortableAttributes(["createdAt"])
|
||||
.catch((e) =>
|
||||
logger.error(
|
||||
`Setting sortable attr failed, placeholder searches won't sort properly: ${e}`,
|
||||
),
|
||||
);
|
||||
|
||||
posts
|
||||
.updateStopWords([
|
||||
"the",
|
||||
"a",
|
||||
"as",
|
||||
"be",
|
||||
"of",
|
||||
"they",
|
||||
"these",
|
||||
"これ",
|
||||
"それ",
|
||||
"あれ",
|
||||
"この",
|
||||
"その",
|
||||
"あの",
|
||||
"ここ",
|
||||
"そこ",
|
||||
"あそこ",
|
||||
"こちら",
|
||||
"どこ",
|
||||
"だれ",
|
||||
"なに",
|
||||
"なん",
|
||||
"何",
|
||||
"私",
|
||||
"貴方",
|
||||
"貴方方",
|
||||
"我々",
|
||||
"私達",
|
||||
"あの人",
|
||||
"あのか",
|
||||
"彼女",
|
||||
"彼",
|
||||
"です",
|
||||
"ありま",
|
||||
"おりま",
|
||||
"います",
|
||||
"は",
|
||||
"が",
|
||||
"の",
|
||||
"に",
|
||||
"を",
|
||||
"で",
|
||||
"え",
|
||||
"から",
|
||||
"まで",
|
||||
"より",
|
||||
"も",
|
||||
"どの",
|
||||
"と",
|
||||
"し",
|
||||
"それで",
|
||||
"しかし",
|
||||
])
|
||||
.catch((e) =>
|
||||
logger.error(
|
||||
`Failed to set Meilisearch stop words, database size will be larger: ${e}`,
|
||||
),
|
||||
);
|
||||
|
||||
logger.info("Connected to MeiliSearch");
|
||||
}
|
||||
|
||||
export type MeilisearchNote = {
|
||||
id: string;
|
||||
text: string;
|
||||
userId: string;
|
||||
userHost: string;
|
||||
userName: string;
|
||||
channelId: string;
|
||||
mediaAttachment: string;
|
||||
createdAt: number;
|
||||
};
|
||||
|
||||
function timestampToUnix(timestamp: string) {
|
||||
let unix = 0;
|
||||
|
||||
// Only contains numbers => UNIX timestamp
|
||||
if (/^\d+$/.test(timestamp)) {
|
||||
unix = Number.parseInt(timestamp);
|
||||
}
|
||||
|
||||
if (unix === 0) {
|
||||
// Try to parse the timestamp as JavaScript Date
|
||||
const date = Date.parse(timestamp);
|
||||
if (isNaN(date)) return 0;
|
||||
unix = date / 1000;
|
||||
}
|
||||
|
||||
return unix;
|
||||
}
|
||||
|
||||
export default hasConfig
|
||||
? {
|
||||
search: async (
|
||||
query: string,
|
||||
limit: number,
|
||||
offset: number,
|
||||
userCtx: ILocalUser | null,
|
||||
) => {
|
||||
/// Advanced search syntax
|
||||
/// from:user => filter by user + optional domain
|
||||
/// has:image/video/audio/text/file => filter by attachment types
|
||||
/// domain:domain.com => filter by domain
|
||||
/// before:Date => show posts made before Date
|
||||
/// after: Date => show posts made after Date
|
||||
/// "text" => get posts with exact text between quotes
|
||||
/// filter:following => show results only from users you follow
|
||||
/// filter:followers => show results only from followers
|
||||
|
||||
const constructedFilters: string[] = [];
|
||||
|
||||
const splitSearch = query.split(" ");
|
||||
|
||||
// Detect search operators and remove them from the actual query
|
||||
const filteredSearchTerms = (
|
||||
await Promise.all(
|
||||
splitSearch.map(async (term) => {
|
||||
if (term.startsWith("has:")) {
|
||||
const fileType = term.slice(4);
|
||||
constructedFilters.push(`mediaAttachment = "${fileType}"`);
|
||||
return null;
|
||||
} else if (term.startsWith("from:")) {
|
||||
let user = term.slice(5);
|
||||
|
||||
if (user.length === 0) return null;
|
||||
|
||||
// Cut off leading @, those aren't saved in the DB
|
||||
if (user.charAt(0) === "@") {
|
||||
user = user.slice(1);
|
||||
}
|
||||
|
||||
// Determine if we got a webfinger address or a single username
|
||||
if (user.split("@").length > 1) {
|
||||
let splitUser = user.split("@");
|
||||
|
||||
let domain = splitUser.pop();
|
||||
user = splitUser.join("@");
|
||||
|
||||
constructedFilters.push(
|
||||
`userName = ${user} AND userHost = ${domain}`,
|
||||
);
|
||||
} else {
|
||||
constructedFilters.push(`userName = ${user}`);
|
||||
}
|
||||
|
||||
return null;
|
||||
} else if (term.startsWith("domain:")) {
|
||||
const domain = term.slice(7);
|
||||
constructedFilters.push(`userHost = ${domain}`);
|
||||
return null;
|
||||
} else if (term.startsWith("after:")) {
|
||||
const timestamp = term.slice(6);
|
||||
|
||||
let unix = timestampToUnix(timestamp);
|
||||
|
||||
if (unix !== 0) constructedFilters.push(`createdAt > ${unix}`);
|
||||
|
||||
return null;
|
||||
} else if (term.startsWith("before:")) {
|
||||
const timestamp = term.slice(7);
|
||||
|
||||
let unix = timestampToUnix(timestamp);
|
||||
if (unix !== 0) constructedFilters.push(`createdAt < ${unix}`);
|
||||
|
||||
return null;
|
||||
} else if (term.startsWith("filter:following")) {
|
||||
// Check if we got a context user
|
||||
if (userCtx) {
|
||||
// Fetch user follows from DB
|
||||
const followedUsers = await Followings.find({
|
||||
where: {
|
||||
followerId: userCtx.id,
|
||||
},
|
||||
select: {
|
||||
followeeId: true,
|
||||
},
|
||||
});
|
||||
const followIDs = followedUsers.map(
|
||||
(user) => user.followeeId,
|
||||
);
|
||||
|
||||
if (followIDs.length === 0) return null;
|
||||
|
||||
constructedFilters.push(`userId IN [${followIDs.join(",")}]`);
|
||||
} else {
|
||||
logger.warn(
|
||||
"search filtered to follows called without user context",
|
||||
);
|
||||
}
|
||||
|
||||
return null;
|
||||
} else if (term.startsWith("filter:followers")) {
|
||||
// Check if we got a context user
|
||||
if (userCtx) {
|
||||
// Fetch users follows from DB
|
||||
const followedUsers = await Followings.find({
|
||||
where: {
|
||||
followeeId: userCtx.id,
|
||||
},
|
||||
select: {
|
||||
followerId: true,
|
||||
},
|
||||
});
|
||||
const followIDs = followedUsers.map(
|
||||
(user) => user.followerId,
|
||||
);
|
||||
|
||||
if (followIDs.length === 0) return null;
|
||||
|
||||
constructedFilters.push(`userId IN [${followIDs.join(",")}]`);
|
||||
} else {
|
||||
logger.warn(
|
||||
"search filtered to followers called without user context",
|
||||
);
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
return term;
|
||||
}),
|
||||
)
|
||||
).filter((term) => term !== null);
|
||||
|
||||
const sortRules = [];
|
||||
|
||||
// An empty search term with defined filters means we have a placeholder search => https://www.meilisearch.com/docs/reference/api/search#placeholder-search
|
||||
// These have to be ordered manually, otherwise the *oldest* posts are returned first, which we don't want
|
||||
if (filteredSearchTerms.length === 0 && constructedFilters.length > 0) {
|
||||
sortRules.push("createdAt:desc");
|
||||
}
|
||||
|
||||
logger.info(`Searching for ${filteredSearchTerms.join(" ")}`);
|
||||
logger.info(`Limit: ${limit}`);
|
||||
logger.info(`Offset: ${offset}`);
|
||||
logger.info(`Filters: ${constructedFilters}`);
|
||||
logger.info(`Ordering: ${sortRules}`);
|
||||
|
||||
return posts.search(filteredSearchTerms.join(" "), {
|
||||
limit: limit,
|
||||
offset: offset,
|
||||
filter: constructedFilters,
|
||||
sort: sortRules,
|
||||
});
|
||||
},
|
||||
ingestNote: async (ingestNotes: Note | Note[]) => {
|
||||
if (ingestNotes instanceof Note) {
|
||||
ingestNotes = [ingestNotes];
|
||||
}
|
||||
|
||||
const indexingBatch: MeilisearchNote[] = [];
|
||||
|
||||
for (const note of ingestNotes) {
|
||||
if (note.user === undefined) {
|
||||
note.user = await Users.findOne({
|
||||
where: {
|
||||
id: note.userId,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
let attachmentType = "";
|
||||
if (note.attachedFileTypes.length > 0) {
|
||||
attachmentType = note.attachedFileTypes[0].split("/")[0];
|
||||
switch (attachmentType) {
|
||||
case "image":
|
||||
case "video":
|
||||
case "audio":
|
||||
case "text":
|
||||
break;
|
||||
default:
|
||||
attachmentType = "file";
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
indexingBatch.push(<MeilisearchNote>{
|
||||
id: note.id.toString(),
|
||||
text: note.text ? note.text : "",
|
||||
userId: note.userId,
|
||||
userHost:
|
||||
note.userHost !== ""
|
||||
? note.userHost
|
||||
: config.domain,
|
||||
channelId: note.channelId ? note.channelId : "",
|
||||
mediaAttachment: attachmentType,
|
||||
userName: note.user?.username ?? "UNKNOWN",
|
||||
createdAt: note.createdAt.getTime() / 1000, // division by 1000 is necessary because Node returns in ms-accuracy
|
||||
});
|
||||
}
|
||||
|
||||
return posts
|
||||
.addDocuments(indexingBatch, {
|
||||
primaryKey: "id",
|
||||
})
|
||||
.then(() =>
|
||||
logger.info(`sent ${indexingBatch.length} posts for indexing`),
|
||||
);
|
||||
},
|
||||
serverStats: async () => {
|
||||
const health: Health = await client.health();
|
||||
const stats: Stats = await client.getStats();
|
||||
|
||||
return {
|
||||
health: health.status,
|
||||
size: stats.databaseSize,
|
||||
indexed_count: stats.indexes["posts"].numberOfDocuments,
|
||||
};
|
||||
},
|
||||
deleteNotes: async (note: Note | Note[] | string | string[]) => {
|
||||
if (note instanceof Note) {
|
||||
note = [note];
|
||||
}
|
||||
if (typeof note === "string") {
|
||||
note = [note];
|
||||
}
|
||||
|
||||
const deletionBatch = note
|
||||
.map((n) => {
|
||||
if (n instanceof Note) {
|
||||
return n.id;
|
||||
}
|
||||
|
||||
if (n.length > 0) return n;
|
||||
|
||||
logger.error(
|
||||
`Failed to delete note from Meilisearch, invalid post ID: ${JSON.stringify(
|
||||
n,
|
||||
)}`,
|
||||
);
|
||||
|
||||
throw new Error(
|
||||
`Invalid note ID passed to meilisearch deleteNote: ${JSON.stringify(
|
||||
n,
|
||||
)}`,
|
||||
);
|
||||
})
|
||||
.filter((el) => el !== null);
|
||||
|
||||
await posts.deleteDocuments(deletionBatch as string[]).then(() => {
|
||||
logger.info(
|
||||
`submitted ${deletionBatch.length} large batch for deletion`,
|
||||
);
|
||||
});
|
||||
},
|
||||
}
|
||||
: null;
|
||||
@ -1,51 +0,0 @@
|
||||
import * as SonicChannel from "sonic-channel";
|
||||
import { dbLogger } from "./logger.js";
|
||||
|
||||
import config from "@/config/index.js";
|
||||
|
||||
const logger = dbLogger.createSubLogger("sonic", "gray", false);
|
||||
|
||||
const handlers = (type: string): SonicChannel.Handlers => ({
|
||||
connected: () => {
|
||||
logger.succ(`Connected to Sonic ${type}`);
|
||||
},
|
||||
disconnected: (error) => {
|
||||
logger.warn(`Disconnected from Sonic ${type}, error: ${error}`);
|
||||
},
|
||||
error: (error) => {
|
||||
logger.warn(`Sonic ${type} error: ${error}`);
|
||||
},
|
||||
retrying: () => {
|
||||
logger.info(`Sonic ${type} retrying`);
|
||||
},
|
||||
timeout: () => {
|
||||
logger.warn(`Sonic ${type} timeout`);
|
||||
},
|
||||
});
|
||||
|
||||
const hasConfig =
|
||||
config.sonic && (config.sonic.host || config.sonic.port || config.sonic.auth);
|
||||
|
||||
if (hasConfig) {
|
||||
logger.info("Connecting to Sonic");
|
||||
}
|
||||
|
||||
const host = hasConfig ? config.sonic.host ?? "localhost" : "";
|
||||
const port = hasConfig ? config.sonic.port ?? 1491 : 0;
|
||||
const auth = hasConfig ? config.sonic.auth ?? "SecretPassword" : "";
|
||||
const collection = hasConfig ? config.sonic.collection ?? "main" : "";
|
||||
const bucket = hasConfig ? config.sonic.bucket ?? "default" : "";
|
||||
|
||||
export default hasConfig
|
||||
? {
|
||||
search: new SonicChannel.Search({ host, port, auth }).connect(
|
||||
handlers("search"),
|
||||
),
|
||||
ingest: new SonicChannel.Ingest({ host, port, auth }).connect(
|
||||
handlers("ingest"),
|
||||
),
|
||||
|
||||
collection,
|
||||
bucket,
|
||||
}
|
||||
: null;
|
||||
@ -1,88 +0,0 @@
|
||||
import type Bull from "bull";
|
||||
import type { DoneCallback } from "bull";
|
||||
|
||||
import { queueLogger } from "../../logger.js";
|
||||
import { Notes } from "@/models/index.js";
|
||||
import { MoreThan } from "typeorm";
|
||||
import { index } from "@/services/note/create.js";
|
||||
import { Note } from "@/models/entities/note.js";
|
||||
import meilisearch from "../../../db/meilisearch.js";
|
||||
|
||||
const logger = queueLogger.createSubLogger("index-all-notes");
|
||||
|
||||
export default async function indexAllNotes(
|
||||
job: Bull.Job<Record<string, unknown>>,
|
||||
done: DoneCallback,
|
||||
): Promise<void> {
|
||||
logger.info("Indexing all notes...");
|
||||
|
||||
let cursor: string | null = (job.data.cursor as string) ?? null;
|
||||
let indexedCount: number = (job.data.indexedCount as number) ?? 0;
|
||||
let total: number = (job.data.total as number) ?? 0;
|
||||
|
||||
let running = true;
|
||||
const take = 10000;
|
||||
const batch = 100;
|
||||
while (running) {
|
||||
logger.info(
|
||||
`Querying for ${take} notes ${indexedCount}/${
|
||||
total ? total : "?"
|
||||
} at ${cursor}`,
|
||||
);
|
||||
|
||||
let notes: Note[] = [];
|
||||
try {
|
||||
notes = await Notes.find({
|
||||
where: {
|
||||
...(cursor ? { id: MoreThan(cursor) } : {}),
|
||||
},
|
||||
take: take,
|
||||
order: {
|
||||
id: 1,
|
||||
},
|
||||
relations: ["user"],
|
||||
});
|
||||
} catch (e: any) {
|
||||
logger.error(`Failed to query notes ${e}`);
|
||||
done(e);
|
||||
break;
|
||||
}
|
||||
|
||||
if (notes.length === 0) {
|
||||
await job.progress(100);
|
||||
running = false;
|
||||
break;
|
||||
}
|
||||
|
||||
try {
|
||||
const count = await Notes.count();
|
||||
total = count;
|
||||
await job.update({ indexedCount, cursor, total });
|
||||
} catch (e) {}
|
||||
|
||||
for (let i = 0; i < notes.length; i += batch) {
|
||||
const chunk = notes.slice(i, i + batch);
|
||||
|
||||
if (meilisearch) {
|
||||
await meilisearch.ingestNote(chunk);
|
||||
}
|
||||
|
||||
await Promise.all(chunk.map((note) => index(note, true)));
|
||||
|
||||
indexedCount += chunk.length;
|
||||
const pct = (indexedCount / total) * 100;
|
||||
await job.update({ indexedCount, cursor, total });
|
||||
await job.progress(+pct.toFixed(1));
|
||||
logger.info(`Indexed notes ${indexedCount}/${total ? total : "?"}`);
|
||||
}
|
||||
cursor = notes[notes.length - 1].id;
|
||||
await job.update({ indexedCount, cursor, total });
|
||||
|
||||
if (notes.length < take) {
|
||||
running = false;
|
||||
}
|
||||
}
|
||||
|
||||
done();
|
||||
logger.info("All notes have been indexed.");
|
||||
}
|
||||
@ -1,9 +1,6 @@
|
||||
import type Bull from "bull";
|
||||
import indexAllNotes from "./index-all-notes.js";
|
||||
|
||||
const jobs = {
|
||||
indexAllNotes,
|
||||
} as Record<string, Bull.ProcessCallbackFunction<Record<string, unknown>>>;
|
||||
const jobs = {} as Record<string, Bull.ProcessCallbackFunction<Record<string, unknown>>>;
|
||||
|
||||
export default function (q: Bull.Queue) {
|
||||
for (const [k, v] of Object.entries(jobs)) {
|
||||
|
||||
@ -7,7 +7,6 @@ import type { DriveFile } from "@/models/entities/drive-file.js";
|
||||
import { MoreThan } from "typeorm";
|
||||
import { deleteFileSync } from "@/services/drive/delete-file.js";
|
||||
import { sendEmail } from "@/services/send-email.js";
|
||||
import meilisearch from "@/db/meilisearch.js";
|
||||
import { publishInternalEvent } from "@/services/stream.js";
|
||||
|
||||
const logger = queueLogger.createSubLogger("delete-account");
|
||||
@ -44,9 +43,6 @@ export async function deleteAccount(
|
||||
cursor = notes[notes.length - 1].id;
|
||||
|
||||
await Notes.delete(notes.map((note) => note.id));
|
||||
if (meilisearch) {
|
||||
await meilisearch.deleteNotes(notes);
|
||||
}
|
||||
}
|
||||
|
||||
logger.succ("All of notes deleted");
|
||||
|
||||
@ -489,7 +489,7 @@ export default define(meta, paramDef, async (ps, me) => {
|
||||
recommendedTimeline: !instance.disableRecommendedTimeline,
|
||||
globalTimeLine: !instance.disableGlobalTimeline,
|
||||
emailRequiredForSignup: instance.emailRequiredForSignup,
|
||||
searchFilters: !!config.meilisearch,
|
||||
searchFilters: true,
|
||||
hcaptcha: instance.enableHcaptcha,
|
||||
recaptcha: instance.enableRecaptcha,
|
||||
objectStorage: instance.useObjectStorage,
|
||||
|
||||
@ -1,10 +1,5 @@
|
||||
import { In } from "typeorm";
|
||||
import { Notes } from "@/models/index.js";
|
||||
import { Note } from "@/models/entities/note.js";
|
||||
import config from "@/config/index.js";
|
||||
import es from "@/db/elasticsearch.js";
|
||||
import sonic from "@/db/sonic.js";
|
||||
import meilisearch, { MeilisearchNote } from "@/db/meilisearch.js";
|
||||
import define from "../../define.js";
|
||||
import { makePaginationQuery } from "../../common/make-pagination-query.js";
|
||||
import { generateVisibilityQuery } from "../../common/generate-visibility-query.js";
|
||||
@ -63,258 +58,39 @@ export const paramDef = {
|
||||
} as const;
|
||||
|
||||
export default define(meta, paramDef, async (ps, me) => {
|
||||
if (es == null && sonic == null && meilisearch == null) {
|
||||
const query = makePaginationQuery(
|
||||
Notes.createQueryBuilder("note"),
|
||||
ps.sinceId,
|
||||
ps.untilId,
|
||||
);
|
||||
const query = makePaginationQuery(
|
||||
Notes.createQueryBuilder("note"),
|
||||
ps.sinceId,
|
||||
ps.untilId,
|
||||
);
|
||||
|
||||
if (ps.userId) {
|
||||
query.andWhere("note.userId = :userId", { userId: ps.userId });
|
||||
} else if (ps.channelId) {
|
||||
query.andWhere("note.channelId = :channelId", {
|
||||
channelId: ps.channelId,
|
||||
});
|
||||
}
|
||||
|
||||
query
|
||||
.andWhere("note.text ILIKE :q", { q: `%${sqlLikeEscape(ps.query)}%` })
|
||||
.innerJoinAndSelect("note.user", "user")
|
||||
.leftJoinAndSelect("user.avatar", "avatar")
|
||||
.leftJoinAndSelect("user.banner", "banner")
|
||||
.leftJoinAndSelect("note.reply", "reply")
|
||||
.leftJoinAndSelect("note.renote", "renote")
|
||||
.leftJoinAndSelect("reply.user", "replyUser")
|
||||
.leftJoinAndSelect("replyUser.avatar", "replyUserAvatar")
|
||||
.leftJoinAndSelect("replyUser.banner", "replyUserBanner")
|
||||
.leftJoinAndSelect("renote.user", "renoteUser")
|
||||
.leftJoinAndSelect("renoteUser.avatar", "renoteUserAvatar")
|
||||
.leftJoinAndSelect("renoteUser.banner", "renoteUserBanner");
|
||||
|
||||
generateVisibilityQuery(query, me);
|
||||
if (me) generateMutedUserQuery(query, me);
|
||||
if (me) generateBlockedUserQuery(query, me);
|
||||
|
||||
const notes: Note[] = await query.take(ps.limit).getMany();
|
||||
|
||||
return await Notes.packMany(notes, me);
|
||||
} else if (sonic) {
|
||||
let start = 0;
|
||||
const chunkSize = 100;
|
||||
|
||||
// Use sonic to fetch and step through all search results that could match the requirements
|
||||
const ids = [];
|
||||
while (true) {
|
||||
const results = await sonic.search.query(
|
||||
sonic.collection,
|
||||
sonic.bucket,
|
||||
ps.query,
|
||||
{
|
||||
limit: chunkSize,
|
||||
offset: start,
|
||||
},
|
||||
);
|
||||
|
||||
start += chunkSize;
|
||||
|
||||
if (results.length === 0) {
|
||||
break;
|
||||
}
|
||||
|
||||
const res = results
|
||||
.map((k) => JSON.parse(k))
|
||||
.filter((key) => {
|
||||
if (ps.userId && key.userId !== ps.userId) {
|
||||
return false;
|
||||
}
|
||||
if (ps.channelId && key.channelId !== ps.channelId) {
|
||||
return false;
|
||||
}
|
||||
if (ps.sinceId && key.id <= ps.sinceId) {
|
||||
return false;
|
||||
}
|
||||
if (ps.untilId && key.id >= ps.untilId) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
})
|
||||
.map((key) => key.id);
|
||||
|
||||
ids.push(...res);
|
||||
}
|
||||
|
||||
// Sort all the results by note id DESC (newest first)
|
||||
ids.sort((a, b) => b - a);
|
||||
|
||||
// Fetch the notes from the database until we have enough to satisfy the limit
|
||||
start = 0;
|
||||
const found = [];
|
||||
while (found.length < ps.limit && start < ids.length) {
|
||||
const chunk = ids.slice(start, start + chunkSize);
|
||||
const notes: Note[] = await Notes.find({
|
||||
where: {
|
||||
id: In(chunk),
|
||||
},
|
||||
order: {
|
||||
id: "DESC",
|
||||
},
|
||||
});
|
||||
|
||||
// The notes are checked for visibility and muted/blocked users when packed
|
||||
found.push(...(await Notes.packMany(notes, me)));
|
||||
start += chunkSize;
|
||||
}
|
||||
|
||||
// If we have more results than the limit, trim them
|
||||
if (found.length > ps.limit) {
|
||||
found.length = ps.limit;
|
||||
}
|
||||
|
||||
return found;
|
||||
} else if (meilisearch) {
|
||||
let start = 0;
|
||||
const chunkSize = 100;
|
||||
|
||||
// Use meilisearch to fetch and step through all search results that could match the requirements
|
||||
const ids = [];
|
||||
while (true) {
|
||||
const results = await meilisearch.search(ps.query, chunkSize, start, me);
|
||||
|
||||
start += chunkSize;
|
||||
|
||||
if (results.hits.length === 0) {
|
||||
break;
|
||||
}
|
||||
|
||||
const res = results.hits
|
||||
.filter((key: MeilisearchNote) => {
|
||||
if (ps.userId && key.userId !== ps.userId) {
|
||||
return false;
|
||||
}
|
||||
if (ps.channelId && key.channelId !== ps.channelId) {
|
||||
return false;
|
||||
}
|
||||
if (ps.sinceId && key.id <= ps.sinceId) {
|
||||
return false;
|
||||
}
|
||||
if (ps.untilId && key.id >= ps.untilId) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
})
|
||||
.map((key) => key.id);
|
||||
|
||||
ids.push(...res);
|
||||
}
|
||||
|
||||
// Sort all the results by note id DESC (newest first)
|
||||
ids.sort((a, b) => b - a);
|
||||
|
||||
// Fetch the notes from the database until we have enough to satisfy the limit
|
||||
start = 0;
|
||||
const found = [];
|
||||
while (found.length < ps.limit && start < ids.length) {
|
||||
const chunk = ids.slice(start, start + chunkSize);
|
||||
const notes: Note[] = await Notes.find({
|
||||
where: {
|
||||
id: In(chunk),
|
||||
},
|
||||
order: {
|
||||
id: "DESC",
|
||||
},
|
||||
});
|
||||
|
||||
// The notes are checked for visibility and muted/blocked users when packed
|
||||
found.push(...(await Notes.packMany(notes, me)));
|
||||
start += chunkSize;
|
||||
}
|
||||
|
||||
// If we have more results than the limit, trim them
|
||||
if (found.length > ps.limit) {
|
||||
found.length = ps.limit;
|
||||
}
|
||||
|
||||
return found;
|
||||
} else {
|
||||
const userQuery =
|
||||
ps.userId != null
|
||||
? [
|
||||
{
|
||||
term: {
|
||||
userId: ps.userId,
|
||||
},
|
||||
},
|
||||
]
|
||||
: [];
|
||||
|
||||
const hostQuery =
|
||||
ps.userId == null
|
||||
? ps.host === null
|
||||
? [
|
||||
{
|
||||
bool: {
|
||||
must_not: {
|
||||
exists: {
|
||||
field: "userHost",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
]
|
||||
: ps.host !== undefined
|
||||
? [
|
||||
{
|
||||
term: {
|
||||
userHost: ps.host,
|
||||
},
|
||||
},
|
||||
]
|
||||
: []
|
||||
: [];
|
||||
|
||||
const result = await es.search({
|
||||
index: config.elasticsearch.index || "misskey_note",
|
||||
body: {
|
||||
size: ps.limit,
|
||||
from: ps.offset,
|
||||
query: {
|
||||
bool: {
|
||||
must: [
|
||||
{
|
||||
simple_query_string: {
|
||||
fields: ["text"],
|
||||
query: ps.query.toLowerCase(),
|
||||
default_operator: "and",
|
||||
},
|
||||
},
|
||||
...hostQuery,
|
||||
...userQuery,
|
||||
],
|
||||
},
|
||||
},
|
||||
sort: [
|
||||
{
|
||||
_doc: "desc",
|
||||
},
|
||||
],
|
||||
},
|
||||
if (ps.userId) {
|
||||
query.andWhere("note.userId = :userId", { userId: ps.userId });
|
||||
} else if (ps.channelId) {
|
||||
query.andWhere("note.channelId = :channelId", {
|
||||
channelId: ps.channelId,
|
||||
});
|
||||
|
||||
const hits = result.body.hits.hits.map((hit: any) => hit._id);
|
||||
|
||||
if (hits.length === 0) return [];
|
||||
|
||||
// Fetch found notes
|
||||
const notes = await Notes.find({
|
||||
where: {
|
||||
id: In(hits),
|
||||
},
|
||||
order: {
|
||||
id: -1,
|
||||
},
|
||||
});
|
||||
|
||||
return await Notes.packMany(notes, me);
|
||||
}
|
||||
|
||||
query
|
||||
.andWhere("note.text ILIKE :q", { q: `%${sqlLikeEscape(ps.query)}%` })
|
||||
.innerJoinAndSelect("note.user", "user")
|
||||
.leftJoinAndSelect("user.avatar", "avatar")
|
||||
.leftJoinAndSelect("user.banner", "banner")
|
||||
.leftJoinAndSelect("note.reply", "reply")
|
||||
.leftJoinAndSelect("note.renote", "renote")
|
||||
.leftJoinAndSelect("reply.user", "replyUser")
|
||||
.leftJoinAndSelect("replyUser.avatar", "replyUserAvatar")
|
||||
.leftJoinAndSelect("replyUser.banner", "replyUserBanner")
|
||||
.leftJoinAndSelect("renote.user", "renoteUser")
|
||||
.leftJoinAndSelect("renoteUser.avatar", "renoteUserAvatar")
|
||||
.leftJoinAndSelect("renoteUser.banner", "renoteUserBanner");
|
||||
|
||||
generateVisibilityQuery(query, me);
|
||||
if (me) generateMutedUserQuery(query, me);
|
||||
if (me) generateBlockedUserQuery(query, me);
|
||||
|
||||
const notes: Note[] = await query.take(ps.limit).getMany();
|
||||
|
||||
return await Notes.packMany(notes, me);
|
||||
});
|
||||
|
||||
@ -1,7 +1,6 @@
|
||||
import * as os from "node:os";
|
||||
import si from "systeminformation";
|
||||
import define from "../define.js";
|
||||
import meilisearch from "@/db/meilisearch.js";
|
||||
import { fetchMeta } from "@/misc/fetch-meta.js";
|
||||
|
||||
export const meta = {
|
||||
@ -63,15 +62,3 @@ export default define(meta, paramDef, async () => {
|
||||
},
|
||||
};
|
||||
});
|
||||
|
||||
async function meilisearchStatus() {
|
||||
if (meilisearch) {
|
||||
return meilisearch.serverStats();
|
||||
} else {
|
||||
return {
|
||||
health: "unconfigured",
|
||||
size: 0,
|
||||
indexed_count: 0,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,6 +1,3 @@
|
||||
import es from "@/db/elasticsearch.js";
|
||||
import sonic from "@/db/sonic.js";
|
||||
import meilisearch, { MeilisearchNote } from "@/db/meilisearch.js";
|
||||
import { Followings, Hashtags, Notes, Users } from "@/models/index.js";
|
||||
import { sqlLikeEscape } from "@/misc/sql-like-escape.js";
|
||||
import { generateVisibilityQuery } from "@/server/api/common/generate-visibility-query.js";
|
||||
@ -9,7 +6,7 @@ import { generateBlockedUserQuery } from "@/server/api/common/generate-block-que
|
||||
import { Note } from "@/models/entities/note.js";
|
||||
import { PaginationHelpers } from "@/server/api/mastodon/helpers/pagination.js";
|
||||
import { ILocalUser, User } from "@/models/entities/user.js";
|
||||
import { Brackets, In, IsNull } from "typeorm";
|
||||
import { Brackets, IsNull } from "typeorm";
|
||||
import { awaitAll } from "@/prelude/await-all.js";
|
||||
import { NoteConverter } from "@/server/api/mastodon/converters/note.js";
|
||||
import Resolver from "@/remote/activitypub/resolver.js";
|
||||
@ -19,7 +16,6 @@ import { createPerson } from "@/remote/activitypub/models/person.js";
|
||||
import { UserConverter } from "@/server/api/mastodon/converters/user.js";
|
||||
import { resolveUser } from "@/remote/resolve-user.js";
|
||||
import { createNote } from "@/remote/activitypub/models/note.js";
|
||||
import { getUser } from "@/server/api/common/getters.js";
|
||||
import config from "@/config/index.js";
|
||||
import { logger, MastoContext } from "@/server/api/mastodon/index.js";
|
||||
|
||||
@ -144,216 +140,6 @@ export class SearchHelpers {
|
||||
}
|
||||
}
|
||||
|
||||
// Try sonic search first, unless we have advanced filters
|
||||
if (sonic && !accountId && !following) {
|
||||
let start = offset ?? 0;
|
||||
const chunkSize = 100;
|
||||
|
||||
// Use sonic to fetch and step through all search results that could match the requirements
|
||||
const ids = [];
|
||||
while (true) {
|
||||
const results = await sonic.search.query(
|
||||
sonic.collection,
|
||||
sonic.bucket,
|
||||
q,
|
||||
{
|
||||
limit: chunkSize,
|
||||
offset: start,
|
||||
},
|
||||
);
|
||||
|
||||
start += chunkSize;
|
||||
|
||||
if (results.length === 0) {
|
||||
break;
|
||||
}
|
||||
|
||||
const res = results
|
||||
.map((k) => JSON.parse(k))
|
||||
.filter((key) => {
|
||||
if (minId && key.id < minId) return false;
|
||||
if (maxId && key.id > maxId) return false;
|
||||
return true;
|
||||
})
|
||||
.map((key) => key.id);
|
||||
|
||||
ids.push(...res);
|
||||
}
|
||||
|
||||
// Sort all the results by note id DESC (newest first)
|
||||
ids.sort((a, b) => b - a);
|
||||
|
||||
// Fetch the notes from the database until we have enough to satisfy the limit
|
||||
start = 0;
|
||||
const found = [];
|
||||
while (found.length < limit && start < ids.length) {
|
||||
const chunk = ids.slice(start, start + chunkSize);
|
||||
|
||||
const query = Notes.createQueryBuilder("note")
|
||||
.where({ id: In(chunk) })
|
||||
.orderBy({ id: "DESC" })
|
||||
|
||||
generateVisibilityQuery(query, user);
|
||||
|
||||
if (!accountId) {
|
||||
generateMutedUserQuery(query, user);
|
||||
generateBlockedUserQuery(query, user);
|
||||
}
|
||||
|
||||
if (following) {
|
||||
const followingQuery = Followings.createQueryBuilder("following")
|
||||
.select("following.followeeId")
|
||||
.where("following.followerId = :followerId", { followerId: user.id });
|
||||
|
||||
query.andWhere(
|
||||
new Brackets((qb) => {
|
||||
qb.where(`note.userId IN (${followingQuery.getQuery()} UNION ALL VALUES (:meId))`, { meId: user.id });
|
||||
}),
|
||||
)
|
||||
}
|
||||
|
||||
const notes: Note[] = await query.getMany();
|
||||
|
||||
found.push(...notes);
|
||||
start += chunkSize;
|
||||
}
|
||||
|
||||
// If we have more results than the limit, trim them
|
||||
if (found.length > limit) {
|
||||
found.length = limit;
|
||||
}
|
||||
|
||||
return found;
|
||||
}
|
||||
// Try meilisearch next
|
||||
else if (meilisearch) {
|
||||
let start = 0;
|
||||
const chunkSize = 100;
|
||||
|
||||
// Use meilisearch to fetch and step through all search results that could match the requirements
|
||||
const ids = [];
|
||||
if (accountId) {
|
||||
const acc = await getUser(accountId);
|
||||
const append = acc.host !== null ? `from:${acc.usernameLower}@${acc.host} ` : `from:${acc.usernameLower}`;
|
||||
q = append + q;
|
||||
}
|
||||
if (following) {
|
||||
q = `filter:following ${q}`;
|
||||
}
|
||||
while (true) {
|
||||
const results = await meilisearch.search(q, chunkSize, start, user);
|
||||
|
||||
start += chunkSize;
|
||||
|
||||
if (results.hits.length === 0) {
|
||||
break;
|
||||
}
|
||||
|
||||
//TODO test this, it's the same logic the mk api uses but it seems, we need to make .hits already be a MeilisearchNote[] instead of forcing type checks to pass
|
||||
const res = (results.hits as MeilisearchNote[])
|
||||
.filter((key: MeilisearchNote) => {
|
||||
if (accountId && key.userId !== accountId) return false;
|
||||
if (minId && key.id < minId) return false;
|
||||
if (maxId && key.id > maxId) return false;
|
||||
return true;
|
||||
})
|
||||
.map((key) => key.id);
|
||||
|
||||
ids.push(...res);
|
||||
}
|
||||
|
||||
// Sort all the results by note id DESC (newest first)
|
||||
//FIXME: fix this sort function (is it even necessary?)
|
||||
//ids.sort((a, b) => b - a);
|
||||
|
||||
// Fetch the notes from the database until we have enough to satisfy the limit
|
||||
start = 0;
|
||||
const found = [];
|
||||
while (found.length < limit && start < ids.length) {
|
||||
const chunk = ids.slice(start, start + chunkSize);
|
||||
|
||||
const query = Notes.createQueryBuilder("note")
|
||||
.where({ id: In(chunk) })
|
||||
.orderBy({ id: "DESC" })
|
||||
|
||||
generateVisibilityQuery(query, user);
|
||||
|
||||
if (!accountId) {
|
||||
generateMutedUserQuery(query, user);
|
||||
generateBlockedUserQuery(query, user);
|
||||
}
|
||||
|
||||
const notes: Note[] = await query.getMany();
|
||||
|
||||
found.push(...notes);
|
||||
start += chunkSize;
|
||||
}
|
||||
|
||||
// If we have more results than the limit, trim them
|
||||
if (found.length > limit) {
|
||||
found.length = limit;
|
||||
}
|
||||
|
||||
return found;
|
||||
} else if (es) {
|
||||
const userQuery =
|
||||
accountId != null
|
||||
? [
|
||||
{
|
||||
term: {
|
||||
userId: accountId,
|
||||
},
|
||||
},
|
||||
]
|
||||
: [];
|
||||
|
||||
const result = await es.search({
|
||||
index: config.elasticsearch.index || "misskey_note",
|
||||
body: {
|
||||
size: limit,
|
||||
from: offset,
|
||||
query: {
|
||||
bool: {
|
||||
must: [
|
||||
{
|
||||
simple_query_string: {
|
||||
fields: ["text"],
|
||||
query: q.toLowerCase(),
|
||||
default_operator: "and",
|
||||
},
|
||||
},
|
||||
...userQuery,
|
||||
],
|
||||
},
|
||||
},
|
||||
sort: [
|
||||
{
|
||||
_doc: "desc",
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
|
||||
const hits = result.body.hits.hits.map((hit: any) => hit._id);
|
||||
|
||||
if (hits.length === 0) return [];
|
||||
|
||||
// Fetch found notes
|
||||
const notes = await Notes.find({
|
||||
where: {
|
||||
id: In(hits),
|
||||
},
|
||||
order: {
|
||||
id: -1,
|
||||
},
|
||||
});
|
||||
|
||||
//TODO: test this
|
||||
//FIXME: implement pagination
|
||||
return notes;
|
||||
}
|
||||
|
||||
// Fallback to database query
|
||||
const query = PaginationHelpers.makePaginationQuery(
|
||||
Notes.createQueryBuilder("note"),
|
||||
undefined,
|
||||
|
||||
@ -78,7 +78,7 @@ const nodeinfo2 = async () => {
|
||||
disableRecommendedTimeline: meta.disableRecommendedTimeline,
|
||||
disableGlobalTimeline: meta.disableGlobalTimeline,
|
||||
emailRequiredForSignup: meta.emailRequiredForSignup,
|
||||
searchFilters: config.meilisearch ? true : false,
|
||||
searchFilters: true,
|
||||
postEditing: true,
|
||||
postImports: meta.experimentalFeatures?.postImports || false,
|
||||
enableHcaptcha: meta.enableHcaptcha,
|
||||
|
||||
@ -1,6 +1,4 @@
|
||||
import * as mfm from "mfm-js";
|
||||
import es from "../../db/elasticsearch.js";
|
||||
import sonic from "../../db/sonic.js";
|
||||
import {
|
||||
publishMainStream,
|
||||
publishNotesStream,
|
||||
@ -64,7 +62,6 @@ import type { UserProfile } from "@/models/entities/user-profile.js";
|
||||
import { db } from "@/db/postgre.js";
|
||||
import { getActiveWebhooks } from "@/misc/webhook-cache.js";
|
||||
import { shouldSilenceInstance } from "@/misc/should-block-instance.js";
|
||||
import meilisearch from "../../db/meilisearch.js";
|
||||
import { redisClient } from "@/db/redis.js";
|
||||
import { Mutex } from "redis-semaphore";
|
||||
import { RecursionLimiter } from "@/models/repositories/user-profile.js";
|
||||
@ -655,9 +652,6 @@ export default async (
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// Register to search database
|
||||
await index(note, false);
|
||||
});
|
||||
|
||||
async function renderNoteOrRenoteActivity(data: Option, note: Note) {
|
||||
@ -803,40 +797,6 @@ async function insertNote(
|
||||
}
|
||||
}
|
||||
|
||||
export async function index(note: Note, reindexing: boolean): Promise<void> {
|
||||
if (!note.text) return;
|
||||
|
||||
if (config.elasticsearch && es) {
|
||||
es.index({
|
||||
index: config.elasticsearch.index || "misskey_note",
|
||||
id: note.id.toString(),
|
||||
body: {
|
||||
text: normalizeForSearch(note.text),
|
||||
userId: note.userId,
|
||||
userHost: note.userHost,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
if (sonic) {
|
||||
await sonic.ingest.push(
|
||||
sonic.collection,
|
||||
sonic.bucket,
|
||||
JSON.stringify({
|
||||
id: note.id,
|
||||
userId: note.userId,
|
||||
userHost: note.userHost,
|
||||
channelId: note.channelId,
|
||||
}),
|
||||
note.text,
|
||||
);
|
||||
}
|
||||
|
||||
if (meilisearch && !reindexing) {
|
||||
await meilisearch.ingestNote(note);
|
||||
}
|
||||
}
|
||||
|
||||
async function notifyToWatchersOfRenotee(
|
||||
renote: Note,
|
||||
user: { id: User["id"] },
|
||||
|
||||
@ -21,7 +21,6 @@ import {
|
||||
import { countSameRenotes } from "@/misc/count-same-renotes.js";
|
||||
import { registerOrFetchInstanceDoc } from "../register-or-fetch-instance-doc.js";
|
||||
import { deliverToRelays } from "../relay.js";
|
||||
import meilisearch from "@/db/meilisearch.js";
|
||||
|
||||
/**
|
||||
* 投稿を削除します。
|
||||
@ -123,10 +122,6 @@ export default async function (
|
||||
id: note.id,
|
||||
userId: user.id,
|
||||
});
|
||||
|
||||
if (meilisearch) {
|
||||
await meilisearch.deleteNotes(note.id);
|
||||
}
|
||||
}
|
||||
|
||||
async function findCascadingNotes(note: Note) {
|
||||
|
||||
@ -24,7 +24,7 @@ import { genId } from "@/misc/gen-id.js";
|
||||
import type { IPoll } from "@/models/entities/poll.js";
|
||||
import { deliverToRelays } from "../relay.js";
|
||||
import renderUpdate from "@/remote/activitypub/renderer/update.js";
|
||||
import { extractMentionedUsers, index } from "@/services/note/create.js";
|
||||
import { extractMentionedUsers } from "@/services/note/create.js";
|
||||
import { normalizeForSearch } from "@/misc/normalize-for-search.js";
|
||||
|
||||
type Option = {
|
||||
@ -182,8 +182,6 @@ export default async function (
|
||||
note = await Notes.findOneByOrFail({ id: note.id });
|
||||
|
||||
if (publishing) {
|
||||
index(note, true);
|
||||
|
||||
// Publish update event for the updated note details
|
||||
publishNoteStream(note.id, "updated", {
|
||||
updatedAt: update.updatedAt,
|
||||
|
||||
@ -29,26 +29,6 @@
|
||||
<p>Used: {{ bytes(diskUsed, 1) }}</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="_panel">
|
||||
<XPie class="pie" :value="meiliProgress" />
|
||||
<div>
|
||||
<p>
|
||||
<i class="ph-file-search ph-bold ph-lg"></i>MeiliSearch
|
||||
</p>
|
||||
<p>
|
||||
{{ i18n.ts._widgets.meiliStatus }}: {{ meiliAvailable }}
|
||||
</p>
|
||||
<p>
|
||||
{{ i18n.ts._widgets.meiliSize }}:
|
||||
{{ bytes(meiliTotalSize, 1) }}
|
||||
</p>
|
||||
<p>
|
||||
{{ i18n.ts._widgets.meiliIndexCount }}:
|
||||
{{ meiliIndexCount }}
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</template>
|
||||
@ -71,11 +51,6 @@ let memTotal: number = $ref(0);
|
||||
let memUsed: number = $ref(0);
|
||||
let memFree: number = $ref(0);
|
||||
|
||||
let meiliProgress: number = $ref(0);
|
||||
let meiliTotalSize: number = $ref(0);
|
||||
let meiliIndexCount: number = $ref(0);
|
||||
let meiliAvailable: string = $ref("unavailable");
|
||||
|
||||
const diskUsage = $computed(() => meta.fs.used / meta.fs.total);
|
||||
const diskTotal = $computed(() => meta.fs.total);
|
||||
const diskUsed = $computed(() => meta.fs.used);
|
||||
@ -88,11 +63,6 @@ function onStats(stats) {
|
||||
memTotal = stats.mem.total;
|
||||
memUsed = stats.mem.active;
|
||||
memFree = memTotal - memUsed;
|
||||
|
||||
meiliTotalSize = stats.meilisearch.size;
|
||||
meiliIndexCount = stats.meilisearch.indexed_count;
|
||||
meiliAvailable = stats.meilisearch.health;
|
||||
meiliProgress = meiliIndexCount / serverStats.notesCount;
|
||||
}
|
||||
|
||||
const connection = stream.useChannel("serverStats");
|
||||
|
||||
@ -38,13 +38,6 @@
|
||||
:connection="connection"
|
||||
:meta="meta"
|
||||
/>
|
||||
<XMeili
|
||||
v-else-if="
|
||||
instance.features.searchFilters && widgetProps.view === 5
|
||||
"
|
||||
:connection="connection"
|
||||
:meta="meta"
|
||||
/>
|
||||
</div>
|
||||
</MkContainer>
|
||||
</template>
|
||||
@ -62,7 +55,6 @@ import XNet from "./net.vue";
|
||||
import XCpu from "./cpu.vue";
|
||||
import XMemory from "./mem.vue";
|
||||
import XDisk from "./disk.vue";
|
||||
import XMeili from "./meilisearch.vue";
|
||||
import MkContainer from "@/components/MkContainer.vue";
|
||||
import type { GetFormResultType } from "@/scripts/form";
|
||||
import * as os from "@/os";
|
||||
|
||||
@ -1,85 +0,0 @@
|
||||
<template>
|
||||
<div class="verusivbr">
|
||||
<XPie
|
||||
v-tooltip="i18n.ts.meiliIndexCount"
|
||||
class="pie"
|
||||
:value="progress"
|
||||
:reverse="true"
|
||||
/>
|
||||
<div>
|
||||
<p><i class="ph-file-search ph-bold ph-lg"></i>MeiliSearch</p>
|
||||
<p>{{ i18n.ts._widgets.meiliStatus }}: {{ available }}</p>
|
||||
<p>{{ i18n.ts._widgets.meiliSize }}: {{ bytes(totalSize, 1) }}</p>
|
||||
<p>{{ i18n.ts._widgets.meiliIndexCount }}: {{ indexCount }}</p>
|
||||
</div>
|
||||
</div>
|
||||
<br />
|
||||
</template>
|
||||
|
||||
<script lang="ts" setup>
|
||||
import { onBeforeUnmount, onMounted } from "vue";
|
||||
import XPie from "./pie.vue";
|
||||
import bytes from "@/filters/bytes";
|
||||
import { i18n } from "@/i18n";
|
||||
import * as os from "@/os";
|
||||
|
||||
const props = defineProps<{
|
||||
connection: any;
|
||||
meta: any;
|
||||
}>();
|
||||
|
||||
let progress: number = $ref(0),
|
||||
serverStats = $ref(null),
|
||||
totalSize: number = $ref(0),
|
||||
indexCount: number = $ref(0),
|
||||
available: string = $ref("unavailable");
|
||||
|
||||
function onStats(stats) {
|
||||
totalSize = stats.meilisearch.size;
|
||||
indexCount = stats.meilisearch.indexed_count;
|
||||
available = stats.meilisearch.health;
|
||||
progress = indexCount / serverStats.notesCount;
|
||||
}
|
||||
|
||||
onMounted(() => {
|
||||
os.api("stats", {}).then((res) => {
|
||||
serverStats = res;
|
||||
});
|
||||
props.connection.on("stats", onStats);
|
||||
});
|
||||
|
||||
onBeforeUnmount(() => {
|
||||
props.connection.off("stats", onStats);
|
||||
});
|
||||
</script>
|
||||
|
||||
<style lang="scss" scoped>
|
||||
.verusivbr {
|
||||
display: flex;
|
||||
padding: 16px;
|
||||
|
||||
> .pie {
|
||||
height: 82px;
|
||||
flex-shrink: 0;
|
||||
margin-right: 16px;
|
||||
}
|
||||
|
||||
> div {
|
||||
flex: 1;
|
||||
|
||||
> p {
|
||||
margin: 0;
|
||||
font-size: 0.8em;
|
||||
|
||||
&:first-child {
|
||||
font-weight: bold;
|
||||
margin-bottom: 4px;
|
||||
|
||||
> i {
|
||||
margin-right: 4px;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
</style>
|
||||
46
yarn.lock
46
yarn.lock
@ -983,18 +983,6 @@ __metadata:
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"@elastic/elasticsearch@npm:7.17.0":
|
||||
version: 7.17.0
|
||||
resolution: "@elastic/elasticsearch@npm:7.17.0"
|
||||
dependencies:
|
||||
debug: "npm:^4.3.1"
|
||||
hpagent: "npm:^0.1.1"
|
||||
ms: "npm:^2.1.3"
|
||||
secure-json-parse: "npm:^2.4.0"
|
||||
checksum: d54330ce50b4951b7b9db15349413b4961040fb0b73a09d3f07cef5cb2873fd22af17307e07b6c8b1b1e0844e76e9aeb78ce1e01d67a940e3190763a875648be
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"@es-joy/jsdoccomment@npm:~0.39.4":
|
||||
version: 0.39.4
|
||||
resolution: "@es-joy/jsdoccomment@npm:0.39.4"
|
||||
@ -5308,7 +5296,6 @@ __metadata:
|
||||
"@bull-board/koa": "npm:5.6.0"
|
||||
"@bull-board/ui": "npm:5.6.0"
|
||||
"@discordapp/twemoji": "npm:14.1.2"
|
||||
"@elastic/elasticsearch": "npm:7.17.0"
|
||||
"@koa/cors": "npm:3.4.3"
|
||||
"@koa/multer": "npm:3.0.2"
|
||||
"@koa/router": "npm:9.0.1"
|
||||
@ -5422,7 +5409,6 @@ __metadata:
|
||||
koa-send: "npm:5.0.1"
|
||||
koa-slow: "npm:2.1.0"
|
||||
koa-views: "npm:7.0.2"
|
||||
meilisearch: "npm:0.33.0"
|
||||
mfm-js: "npm:0.23.3"
|
||||
mime-types: "npm:2.1.35"
|
||||
mocha: "npm:10.2.0"
|
||||
@ -5457,7 +5443,6 @@ __metadata:
|
||||
seedrandom: "npm:^3.0.5"
|
||||
semver: "npm:7.5.4"
|
||||
sharp: "npm:0.32.1"
|
||||
sonic-channel: "npm:^1.3.1"
|
||||
strict-event-emitter-types: "npm:2.0.0"
|
||||
stringz: "npm:2.1.0"
|
||||
summaly: "npm:2.7.0"
|
||||
@ -7227,7 +7212,7 @@ __metadata:
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"cross-fetch@npm:^3.0.4, cross-fetch@npm:^3.1.6":
|
||||
"cross-fetch@npm:^3.0.4":
|
||||
version: 3.1.8
|
||||
resolution: "cross-fetch@npm:3.1.8"
|
||||
dependencies:
|
||||
@ -7539,7 +7524,7 @@ __metadata:
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"debug@npm:4, debug@npm:4.3.4, debug@npm:^4.0.0, debug@npm:^4.0.1, debug@npm:^4.1.0, debug@npm:^4.1.1, debug@npm:^4.3.1, debug@npm:^4.3.2, debug@npm:^4.3.3, debug@npm:^4.3.4":
|
||||
"debug@npm:4, debug@npm:4.3.4, debug@npm:^4.0.0, debug@npm:^4.0.1, debug@npm:^4.1.0, debug@npm:^4.1.1, debug@npm:^4.3.2, debug@npm:^4.3.3, debug@npm:^4.3.4":
|
||||
version: 4.3.4
|
||||
resolution: "debug@npm:4.3.4"
|
||||
dependencies:
|
||||
@ -10989,7 +10974,7 @@ __metadata:
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"hpagent@npm:0.1.2, hpagent@npm:^0.1.1":
|
||||
"hpagent@npm:0.1.2":
|
||||
version: 0.1.2
|
||||
resolution: "hpagent@npm:0.1.2"
|
||||
checksum: bd033b3700bb523edc9a805f8683c71fddd622df901e73842b5e3357136ce062c2ddb2ab5e9f5b3d84e0977bfe439f5cdc51d755a11e99376eb95e4624312f0a
|
||||
@ -14397,15 +14382,6 @@ __metadata:
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"meilisearch@npm:0.33.0":
|
||||
version: 0.33.0
|
||||
resolution: "meilisearch@npm:0.33.0"
|
||||
dependencies:
|
||||
cross-fetch: "npm:^3.1.6"
|
||||
checksum: d2aff57b3d5f7eea8befe1c404b9afc12f72526836e99fca79ab03cf95fb163a01bc5014f610d5b6631c4a6339a4272da41e866208b2af92074b658bee53c645
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"meow@npm:^9.0.0":
|
||||
version: 9.0.0
|
||||
resolution: "meow@npm:9.0.0"
|
||||
@ -14831,7 +14807,7 @@ __metadata:
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"ms@npm:2.1.3, ms@npm:^2.0.0, ms@npm:^2.1.1, ms@npm:^2.1.3":
|
||||
"ms@npm:2.1.3, ms@npm:^2.0.0, ms@npm:^2.1.1":
|
||||
version: 2.1.3
|
||||
resolution: "ms@npm:2.1.3"
|
||||
checksum: aa92de608021b242401676e35cfa5aa42dd70cbdc082b916da7fb925c542173e36bce97ea3e804923fe92c0ad991434e4a38327e15a1b5b5f945d66df615ae6d
|
||||
@ -18253,13 +18229,6 @@ __metadata:
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"secure-json-parse@npm:^2.4.0":
|
||||
version: 2.7.0
|
||||
resolution: "secure-json-parse@npm:2.7.0"
|
||||
checksum: 974386587060b6fc5b1ac06481b2f9dbbb0d63c860cc73dc7533f27835fdb67b0ef08762dbfef25625c15bc0a0c366899e00076cb0d556af06b71e22f1dede4c
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"seedrandom@npm:2.4.2":
|
||||
version: 2.4.2
|
||||
resolution: "seedrandom@npm:2.4.2"
|
||||
@ -18641,13 +18610,6 @@ __metadata:
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"sonic-channel@npm:^1.3.1":
|
||||
version: 1.3.1
|
||||
resolution: "sonic-channel@npm:1.3.1"
|
||||
checksum: ee849863a378d5cc631d87c1d184f697979c766edc30159dc2fe28cc5741dbf8caa587f4baabb5d0bee301829e15c629f36e8d40c75dcb22c705d98ffdc23731
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"sort-keys-length@npm:^1.0.0":
|
||||
version: 1.0.1
|
||||
resolution: "sort-keys-length@npm:1.0.1"
|
||||
|
||||
Loading…
Reference in New Issue
Block a user