mirror/iceshrimp
1
0
Fork 0
mirror of https://iceshrimp.dev/iceshrimp/iceshrimp synced 2024-11-27 14:28:08 +09:00
Code Issues Actions Packages Projects Releases Wiki Activity

[backend] Allow case sensitive and word search for postgres FTS

Browse Source
This commit is contained in:
Laura Hausmann 2023-11-18 20:21:45 +01:00
parent 4f3bf91e7f
commit 768165776d
No known key found for this signature in database
GPG Key ID: D044E84C5BE01605
2 changed files with 29 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
packages/backend/src/misc/sql-regex-escape.ts Normal file
View File

@ -0,0 +1,3 @@
export function sqlRegexEscape(s: string) {
return s.replace(/([!$()*+.:<=>?[\\\]^{|}-])/g, "\\$1");
}

34
packages/backend/src/server/api/common/generate-fts-query.ts
View File

@ -1,7 +1,7 @@
import { Brackets, IsNull, Not, SelectQueryBuilder } from "typeorm"; import { Brackets, SelectQueryBuilder, WhereExpressionBuilder } from "typeorm";
import { sqlLikeEscape } from "@/misc/sql-like-escape.js"; import { sqlLikeEscape } from "@/misc/sql-like-escape.js";
import { sqlRegexEscape } from "@/misc/sql-regex-escape.js";
import { Followings, Users } from "@/models/index.js"; import { Followings, Users } from "@/models/index.js";
import { FILE_TYPE_BROWSERSAFE } from "@/const.js";
const filters = { const filters = {
"from": fromFilter, "from": fromFilter,
@ -26,17 +26,24 @@ const filters = {
} as Record<string, (query: SelectQueryBuilder<any>, search: string, id: number) => any> } as Record<string, (query: SelectQueryBuilder<any>, search: string, id: number) => any>
//TODO: editing the query should be possible, clicking search again resets it (it should be a twitter-like top of the page kind of deal) //TODO: editing the query should be possible, clicking search again resets it (it should be a twitter-like top of the page kind of deal)
//TODO: UI and guide are missing for search|match: and case: filters
export function generateFtsQuery(query: SelectQueryBuilder<any>, q: string): void { export function generateFtsQuery(query: SelectQueryBuilder<any>, q: string): void {
const components = q.trim().split(" "); const components = q.trim().split(" ");
const terms: string[] = []; const terms: string[] = [];
let finalTerms: string[] = []; let finalTerms: string[] = [];
let counter = 0; let counter = 0;
let caseSensitive = false;
let matchWords = false;
for (const component of components) { for (const component of components) {
const split = component.split(":"); const split = component.split(":");
if (split.length > 1 && filters[split[0]] !== undefined) if (split.length > 1 && filters[split[0]] !== undefined)
filters[split[0]](query, split.slice(1).join(":"), counter++); filters[split[0]](query, split.slice(1).join(":"), counter++);
else if(split.length > 1 && (split[0] === "search" || split[0] === "match"))
matchWords = split[1] === 'word' || split[1] === 'words';
else if(split.length > 1 && split[0] === "case")
caseSensitive = split[1] === 'sensitive';
else terms.push(component); else terms.push(component);
} }
@ -62,8 +69,7 @@ export function generateFtsQuery(query: SelectQueryBuilder<any>, q: string): voi
query.andWhere(new Brackets(qb => { query.andWhere(new Brackets(qb => {
for (const term of extractToken(terms, idx, i).split(' OR ')) { for (const term of extractToken(terms, idx, i).split(' OR ')) {
const id = counter++; const id = counter++;
qb.orWhere(`note.text ILIKE :q_${id}`); appendSearchQuery(term, 'or', query, qb, id, term.startsWith('-'), matchWords, caseSensitive);
query.setParameter(`q_${id}`, `%${sqlLikeEscape(term)}%`);
} }
})); }));
state = 'idle'; state = 'idle';
@ -76,10 +82,7 @@ export function generateFtsQuery(query: SelectQueryBuilder<any>, q: string): voi
for (const term of finalTerms) { for (const term of finalTerms) {
const id = counter++; const id = counter++;
if (term.startsWith('-')) query.andWhere(`note.text NOT ILIKE :q_${id}`); appendSearchQuery(term, 'and', query, query, id, term.startsWith('-'), matchWords, caseSensitive);
else query.andWhere(`note.text ILIKE :q_${id}`);
query.setParameter(`q_${id}`, `%${sqlLikeEscape(term.substring(term.startsWith('-') ? 1 : 0))}%`);
} }
} }
@ -218,3 +221,18 @@ function extractToken(array: string[], start: number, end: number, trim: boolean
function trimStartAndEnd(str: string) { function trimStartAndEnd(str: string) {
return str.substring(1, str.length - 1); return str.substring(1, str.length - 1);
} }
function appendSearchQuery(term: string, mode: 'and' | 'or', query: SelectQueryBuilder<any>, qb: SelectQueryBuilder<any> | WhereExpressionBuilder, id: number, negate: boolean, matchWords: boolean, caseSensitive: boolean) {
const sql = `note.text ${getSearchMatchOperator(negate, matchWords, caseSensitive)} :q_${id}`;
if (mode === 'and') qb.andWhere(sql);
else if (mode === 'or') qb.orWhere(sql);
query.setParameter(`q_${id}`, escapeSqlSearchParam(term.substring(negate ? 1 : 0), matchWords));
}
function getSearchMatchOperator(negate: boolean, matchWords: boolean, caseSensitive: boolean) {
return `${negate ? 'NOT ' : ''}${matchWords ? caseSensitive ? '~' : '~*' : caseSensitive ? 'LIKE' : 'ILIKE'}`;
}
function escapeSqlSearchParam(param: string, matchWords: boolean) {
return matchWords ? `\\y${sqlRegexEscape(param)}\\y` : `%${sqlLikeEscape(param)}%`;
}