From 282fdf347a4a831c8305fc4dce9d2c28d76b55ad Mon Sep 17 00:00:00 2001 From: PrivateGER Date: Sun, 28 May 2023 02:15:13 +0200 Subject: [PATCH] Implement follower and following searches --- packages/backend/src/db/meilisearch.ts | 164 +++++++++++++----- .../src/server/api/endpoints/notes/search.ts | 2 +- 2 files changed, 123 insertions(+), 43 deletions(-) diff --git a/packages/backend/src/db/meilisearch.ts b/packages/backend/src/db/meilisearch.ts index a58425c54..5d294b95d 100644 --- a/packages/backend/src/db/meilisearch.ts +++ b/packages/backend/src/db/meilisearch.ts @@ -4,8 +4,8 @@ import {dbLogger} from "./logger.js"; import config from "@/config/index.js"; import {Note} from "@/models/entities/note.js"; import * as url from "url"; -import {User} from "@/models/entities/user.js"; -import {Users} from "@/models/index.js"; +import {ILocalUser, User} from "@/models/entities/user.js"; +import {Followings, Users} from "@/models/index.js"; const logger = dbLogger.createSubLogger("meilisearch", "gray", false); @@ -41,6 +41,7 @@ posts "userHost", "mediaAttachment", "createdAt", + "userId", ]) .catch((e) => logger.error( @@ -48,6 +49,14 @@ posts ), ); +posts + .updateSortableAttributes(["createdAt"]) + .catch((e) => + logger.error( + `Setting sortable attr failed, placeholder searches won't sort properly: ${e}`, + ), + ); + logger.info("Connected to MeiliSearch"); export type MeilisearchNote = { @@ -63,60 +72,130 @@ export type MeilisearchNote = { export default hasConfig ? { - search: (query: string, limit: number, offset: number) => { + search: async ( + query: string, + limit: number, + offset: number, + userCtx: ILocalUser | null, + ) => { /// Advanced search syntax /// from:user => filter by user + optional domain /// has:image/video/audio/text/file => filter by attachment types /// domain:domain.com => filter by domain /// before:Date => show posts made before Date /// after: Date => show posts made after Date + /// "text" => get posts with exact text between quotes + /// filter:following => show results only from users you follow + /// filter:followers => show results only from followers let constructedFilters: string[] = []; let splitSearch = query.split(" "); // Detect search operators and remove them from the actual query - splitSearch = splitSearch.filter((term) => { - if (term.startsWith("has:")) { - let fileType = term.slice(4); - constructedFilters.push(`mediaAttachment = "${fileType}"`); - return false; - } else if (term.startsWith("from:")) { - let user = term.slice(5); - constructedFilters.push(`userName = ${user}`); - return false; - } else if (term.startsWith("domain:")) { - let domain = term.slice(7); - constructedFilters.push(`userHost = ${domain}`); - return false; - } else if (term.startsWith("after:")) { - let timestamp = term.slice(6); - // Try to parse the timestamp as JavaScript Date - let date = Date.parse(timestamp); - if (isNaN(date)) return false; - constructedFilters.push(`createdAt > ${date}`); - return false; - } else if (term.startsWith("before:")) { - let timestamp = term.slice(7); - // Try to parse the timestamp as JavaScript Date - let date = Date.parse(timestamp); - if (isNaN(date)) return false; - constructedFilters.push(`createdAt < ${date}`); - return false; - } + let filteredSearchTerms = ( + await Promise.all( + splitSearch.map(async (term) => { + if (term.startsWith("has:")) { + let fileType = term.slice(4); + constructedFilters.push(`mediaAttachment = "${fileType}"`); + return null; + } else if (term.startsWith("from:")) { + let user = term.slice(5); + constructedFilters.push(`userName = ${user}`); + return null; + } else if (term.startsWith("domain:")) { + let domain = term.slice(7); + constructedFilters.push(`userHost = ${domain}`); + return null; + } else if (term.startsWith("after:")) { + let timestamp = term.slice(6); + // Try to parse the timestamp as JavaScript Date + let date = Date.parse(timestamp); + if (isNaN(date)) return null; + constructedFilters.push(`createdAt > ${date / 1000}`); + return null; + } else if (term.startsWith("before:")) { + let timestamp = term.slice(7); + // Try to parse the timestamp as JavaScript Date + let date = Date.parse(timestamp); + if (isNaN(date)) return null; + constructedFilters.push(`createdAt < ${date / 1000}`); + return null; + } else if (term.startsWith("filter:following")) { + // Check if we got a context user + if (userCtx) { + // Fetch user follows from DB + let followedUsers = await Followings.find({ + where: { + followerId: userCtx.id, + }, + select: { + followeeId: true, + }, + }); + let followIDs = followedUsers.map((user) => user.followeeId); - return true; - }); + if (followIDs.length === 0) return null; - logger.info(`Searching for ${splitSearch.join(" ")}`); + constructedFilters.push(`userId IN [${followIDs.join(",")}]`); + } else { + logger.warn( + "search filtered to follows called without user context", + ); + } + + return null; + } else if (term.startsWith("filter:followers")) { + // Check if we got a context user + if (userCtx) { + // Fetch users follows from DB + let followedUsers = await Followings.find({ + where: { + followeeId: userCtx.id, + }, + select: { + followerId: true, + }, + }); + let followIDs = followedUsers.map((user) => user.followerId); + + if (followIDs.length === 0) return null; + + constructedFilters.push(`userId IN [${followIDs.join(",")}]`); + } else { + logger.warn( + "search filtered to followers called without user context", + ); + } + + return null; + } + + return term; + }), + ) + ).filter((term) => term !== null); + + let sortRules = []; + + // An empty search term with defined filters means we have a placeholder search => https://www.meilisearch.com/docs/reference/api/search#placeholder-search + // These have to be ordered manually, otherwise the *oldest* posts are returned first, which we don't want + if (filteredSearchTerms.length === 0 && constructedFilters.length > 0) { + sortRules.push("createdAt:desc"); + } + + logger.info(`Searching for ${filteredSearchTerms.join(" ")}`); logger.info(`Limit: ${limit}`); logger.info(`Offset: ${offset}`); logger.info(`Filters: ${constructedFilters}`); + logger.info(`Ordering: ${sortRules}`); - return posts.search(splitSearch.join(" "), { + return posts.search(filteredSearchTerms.join(" "), { limit: limit, offset: offset, filter: constructedFilters, + sort: sortRules, }); }, ingestNote: async (ingestNotes: Note | Note[]) => { @@ -128,12 +207,11 @@ export default hasConfig for (let note of ingestNotes) { if (note.user === undefined) { - let user = await Users.findOne({ + note.user = await Users.findOne({ where: { id: note.userId, }, }); - note.user = user; } let attachmentType = ""; @@ -166,11 +244,13 @@ export default hasConfig }); } - let indexingIDs = indexingBatch.map((note) => note.id); - - return posts.addDocuments(indexingBatch, { - primaryKey: "id", - }); + return posts + .addDocuments(indexingBatch, { + primaryKey: "id", + }) + .then(() => + console.log(`sent ${indexingBatch.length} posts for indexing`), + ); }, serverStats: async () => { let health: Health = await client.health(); diff --git a/packages/backend/src/server/api/endpoints/notes/search.ts b/packages/backend/src/server/api/endpoints/notes/search.ts index 60f264708..346304470 100644 --- a/packages/backend/src/server/api/endpoints/notes/search.ts +++ b/packages/backend/src/server/api/endpoints/notes/search.ts @@ -179,7 +179,7 @@ export default define(meta, paramDef, async (ps, me) => { // Use meilisearch to fetch and step through all search results that could match the requirements const ids = []; while (true) { - const results = await meilisearch.search(ps.query, chunkSize, start); + const results = await meilisearch.search(ps.query, chunkSize, start, me); start += chunkSize;