Revamp post filtering system (#18058)
* Add model for custom filter keywords * Use CustomFilterKeyword internally Does not change the API * Fix /filters/edit and /filters/new * Add migration tests * Remove whole_word column from custom_filters (covered by custom_filter_keywords) * Redesign /filters Instead of a list, present a card that displays more information and handles multiple keywords per filter. * Redesign /filters/new and /filters/edit to add and remove keywords This adds a new gem dependency: cocoon, as well as a npm dependency: cocoon-js-vanilla. Those are used to easily populate and remove form fields from the user interface when manipulating multiple keyword filters at once. * Add /api/v2/filters to edit filter with multiple keywords Entities: - `Filter`: `id`, `title`, `filter_action` (either `hide` or `warn`), `context` `keywords` - `FilterKeyword`: `id`, `keyword`, `whole_word` API endpoits: - `GET /api/v2/filters` to list filters (including keywords) - `POST /api/v2/filters` to create a new filter `keywords_attributes` can also be passed to create keywords in one request - `GET /api/v2/filters/:id` to read a particular filter - `PUT /api/v2/filters/:id` to update a new filter `keywords_attributes` can also be passed to edit, delete or add keywords in one request - `DELETE /api/v2/filters/:id` to delete a particular filter - `GET /api/v2/filters/:id/keywords` to list keywords for a filter - `POST /api/v2/filters/:filter_id/keywords/:id` to add a new keyword to a filter - `GET /api/v2/filter_keywords/:id` to read a particular keyword - `PUT /api/v2/filter_keywords/:id` to edit a particular keyword - `DELETE /api/v2/filter_keywords/:id` to delete a particular keyword * Change from `irreversible` boolean to `action` enum * Remove irrelevent `irreversible_must_be_within_context` check * Fix /filters/new and /filters/edit with update for filter_action * Fix Rubocop/Codeclimate complaining about task names * Refactor FeedManager#phrase_filtered? This moves regexp building and filter caching to the `CustomFilter` class. This does not change the functional behavior yet, but this changes how the cache is built, doing per-custom_filter regexps so that filters can be matched independently, while still offering caching. * Perform server-side filtering and output result in REST API * Fix numerous filters_changed events being sent when editing multiple keywords at once * Add some tests * Use the new API in the WebUI - use client-side logic for filters we have fetched rules for. This is so that filter changes can be retroactively applied without reloading the UI. - use server-side logic for filters we haven't fetched rules for yet (e.g. network error, or initial timeline loading) * Minor optimizations and refactoring * Perform server-side filtering on the streaming server * Change the wording of filter action labels * Fix issues pointed out by linter * Change design of “Show anyway” link in accordence to review comments * Drop “irreversible” filtering behavior * Move /api/v2/filter_keywords to /api/v1/filters/keywords * Rename `filter_results` attribute to `filtered` * Rename REST::LegacyFilterSerializer to REST::V1::FilterSerializer * Fix systemChannelId value in streaming server * Simplify code by removing client-side filtering code The simplifcation comes at a cost though: filters aren't retroactively applied anymore.
This commit is contained in:
Claire
GitHub
parent
5823ae70c4
commit
02851848e9
60 changed files with 1292 additions and 250 deletions
|
|
@ -1,26 +0,0 @@
|
|||
import api from '../api';
|
||||
|
||||
export const FILTERS_FETCH_REQUEST = 'FILTERS_FETCH_REQUEST';
|
||||
export const FILTERS_FETCH_SUCCESS = 'FILTERS_FETCH_SUCCESS';
|
||||
export const FILTERS_FETCH_FAIL = 'FILTERS_FETCH_FAIL';
|
||||
|
||||
export const fetchFilters = () => (dispatch, getState) => {
|
||||
dispatch({
|
||||
type: FILTERS_FETCH_REQUEST,
|
||||
skipLoading: true,
|
||||
});
|
||||
|
||||
api(getState)
|
||||
.get('/api/v1/filters')
|
||||
.then(({ data }) => dispatch({
|
||||
type: FILTERS_FETCH_SUCCESS,
|
||||
filters: data,
|
||||
skipLoading: true,
|
||||
}))
|
||||
.catch(err => dispatch({
|
||||
type: FILTERS_FETCH_FAIL,
|
||||
err,
|
||||
skipLoading: true,
|
||||
skipAlert: true,
|
||||
}));
|
||||
};
|
||||
|
|
@ -5,6 +5,7 @@ export const ACCOUNTS_IMPORT = 'ACCOUNTS_IMPORT';
|
|||
export const STATUS_IMPORT = 'STATUS_IMPORT';
|
||||
export const STATUSES_IMPORT = 'STATUSES_IMPORT';
|
||||
export const POLLS_IMPORT = 'POLLS_IMPORT';
|
||||
export const FILTERS_IMPORT = 'FILTERS_IMPORT';
|
||||
|
||||
function pushUnique(array, object) {
|
||||
if (array.every(element => element.id !== object.id)) {
|
||||
|
|
@ -28,6 +29,10 @@ export function importStatuses(statuses) {
|
|||
return { type: STATUSES_IMPORT, statuses };
|
||||
}
|
||||
|
||||
export function importFilters(filters) {
|
||||
return { type: FILTERS_IMPORT, filters };
|
||||
}
|
||||
|
||||
export function importPolls(polls) {
|
||||
return { type: POLLS_IMPORT, polls };
|
||||
}
|
||||
|
|
@ -61,11 +66,16 @@ export function importFetchedStatuses(statuses) {
|
|||
const accounts = [];
|
||||
const normalStatuses = [];
|
||||
const polls = [];
|
||||
const filters = [];
|
||||
|
||||
function processStatus(status) {
|
||||
pushUnique(normalStatuses, normalizeStatus(status, getState().getIn(['statuses', status.id])));
|
||||
pushUnique(accounts, status.account);
|
||||
|
||||
if (status.filtered) {
|
||||
status.filtered.forEach(result => pushUnique(filters, result.filter));
|
||||
}
|
||||
|
||||
if (status.reblog && status.reblog.id) {
|
||||
processStatus(status.reblog);
|
||||
}
|
||||
|
|
@ -80,6 +90,7 @@ export function importFetchedStatuses(statuses) {
|
|||
dispatch(importPolls(polls));
|
||||
dispatch(importFetchedAccounts(accounts));
|
||||
dispatch(importStatuses(normalStatuses));
|
||||
dispatch(importFilters(filters));
|
||||
};
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -42,6 +42,14 @@ export function normalizeAccount(account) {
|
|||
return account;
|
||||
}
|
||||
|
||||
export function normalizeFilterResult(result) {
|
||||
const normalResult = { ...result };
|
||||
|
||||
normalResult.filter = normalResult.filter.id;
|
||||
|
||||
return normalResult;
|
||||
}
|
||||
|
||||
export function normalizeStatus(status, normalOldStatus) {
|
||||
const normalStatus = { ...status };
|
||||
normalStatus.account = status.account.id;
|
||||
|
|
@ -54,6 +62,10 @@ export function normalizeStatus(status, normalOldStatus) {
|
|||
normalStatus.poll = status.poll.id;
|
||||
}
|
||||
|
||||
if (status.filtered) {
|
||||
normalStatus.filtered = status.filtered.map(normalizeFilterResult);
|
||||
}
|
||||
|
||||
// Only calculate these values when status first encountered and
|
||||
// when the underlying values change. Otherwise keep the ones
|
||||
// already in the reducer
|
||||
|
|
|
|||
|
|
@ -12,10 +12,8 @@ import { saveSettings } from './settings';
|
|||
import { defineMessages } from 'react-intl';
|
||||
import { List as ImmutableList } from 'immutable';
|
||||
import { unescapeHTML } from '../utils/html';
|
||||
import { getFiltersRegex } from '../selectors';
|
||||
import { usePendingItems as preferPendingItems } from 'mastodon/initial_state';
|
||||
import compareId from 'mastodon/compare_id';
|
||||
import { searchTextFromRawStatus } from 'mastodon/actions/importer/normalizer';
|
||||
import { requestNotificationPermission } from '../utils/notifications';
|
||||
|
||||
export const NOTIFICATIONS_UPDATE = 'NOTIFICATIONS_UPDATE';
|
||||
|
|
@ -62,20 +60,17 @@ export function updateNotifications(notification, intlMessages, intlLocale) {
|
|||
const showInColumn = activeFilter === 'all' ? getState().getIn(['settings', 'notifications', 'shows', notification.type], true) : activeFilter === notification.type;
|
||||
const showAlert = getState().getIn(['settings', 'notifications', 'alerts', notification.type], true);
|
||||
const playSound = getState().getIn(['settings', 'notifications', 'sounds', notification.type], true);
|
||||
const filters = getFiltersRegex(getState(), { contextType: 'notifications' });
|
||||
|
||||
let filtered = false;
|
||||
|
||||
if (['mention', 'status'].includes(notification.type)) {
|
||||
const dropRegex = filters[0];
|
||||
const regex = filters[1];
|
||||
const searchIndex = searchTextFromRawStatus(notification.status);
|
||||
if (['mention', 'status'].includes(notification.type) && notification.status.filtered) {
|
||||
const filters = notification.status.filtered.filter(result => result.filter.context.includes('notifications'));
|
||||
|
||||
if (dropRegex && dropRegex.test(searchIndex)) {
|
||||
if (filters.some(result => result.filter.filter_action === 'hide')) {
|
||||
return;
|
||||
}
|
||||
|
||||
filtered = regex && regex.test(searchIndex);
|
||||
filtered = filters.length > 0;
|
||||
}
|
||||
|
||||
if (['follow_request'].includes(notification.type)) {
|
||||
|
|
|
|||
|
|
@ -21,7 +21,6 @@ import {
|
|||
updateReaction as updateAnnouncementsReaction,
|
||||
deleteAnnouncement,
|
||||
} from './announcements';
|
||||
import { fetchFilters } from './filters';
|
||||
import { getLocale } from '../locales';
|
||||
|
||||
const { messages } = getLocale();
|
||||
|
|
@ -97,9 +96,6 @@ export const connectTimelineStream = (timelineId, channelName, params = {}, opti
|
|||
case 'conversation':
|
||||
dispatch(updateConversations(JSON.parse(data.payload)));
|
||||
break;
|
||||
case 'filters_changed':
|
||||
dispatch(fetchFilters());
|
||||
break;
|
||||
case 'announcement':
|
||||
dispatch(updateAnnouncements(JSON.parse(data.payload)));
|
||||
break;
|
||||
|
|
|
|||
|
|
@ -116,6 +116,7 @@ class Status extends ImmutablePureComponent {
|
|||
state = {
|
||||
showMedia: defaultMediaVisibility(this.props.status),
|
||||
statusId: undefined,
|
||||
forceFilter: undefined,
|
||||
};
|
||||
|
||||
static getDerivedStateFromProps(nextProps, prevState) {
|
||||
|
|
@ -277,6 +278,15 @@ class Status extends ImmutablePureComponent {
|
|||
this.handleToggleMediaVisibility();
|
||||
}
|
||||
|
||||
handleUnfilterClick = e => {
|
||||
this.setState({ forceFilter: false });
|
||||
e.preventDefault();
|
||||
}
|
||||
|
||||
handleFilterClick = () => {
|
||||
this.setState({ forceFilter: true });
|
||||
}
|
||||
|
||||
_properStatus () {
|
||||
const { status } = this.props;
|
||||
|
||||
|
|
@ -328,7 +338,8 @@ class Status extends ImmutablePureComponent {
|
|||
);
|
||||
}
|
||||
|
||||
if (status.get('filtered') || status.getIn(['reblog', 'filtered'])) {
|
||||
const matchedFilters = status.get('filtered') || status.getIn(['reblog', 'filtered']);
|
||||
if (this.state.forceFilter === undefined ? matchedFilters : this.state.forceFilter) {
|
||||
const minHandlers = this.props.muted ? {} : {
|
||||
moveUp: this.handleHotkeyMoveUp,
|
||||
moveDown: this.handleHotkeyMoveDown,
|
||||
|
|
@ -337,7 +348,11 @@ class Status extends ImmutablePureComponent {
|
|||
return (
|
||||
<HotKeys handlers={minHandlers}>
|
||||
<div className='status__wrapper status__wrapper--filtered focusable' tabIndex='0' ref={this.handleRef}>
|
||||
<FormattedMessage id='status.filtered' defaultMessage='Filtered' />
|
||||
<FormattedMessage id='status.filtered' defaultMessage='Filtered' />: {matchedFilters.join(', ')}.
|
||||
{' '}
|
||||
<button className='status__wrapper--filtered__button' onClick={this.handleUnfilterClick}>
|
||||
<FormattedMessage id='status.show_filter_reason' defaultMessage='Show anyway' />
|
||||
</button>
|
||||
</div>
|
||||
</HotKeys>
|
||||
);
|
||||
|
|
@ -496,7 +511,7 @@ class Status extends ImmutablePureComponent {
|
|||
|
||||
{media}
|
||||
|
||||
<StatusActionBar scrollKey={scrollKey} status={status} account={account} {...other} />
|
||||
<StatusActionBar scrollKey={scrollKey} status={status} account={account} onFilter={matchedFilters && this.handleFilterClick} {...other} />
|
||||
</div>
|
||||
</div>
|
||||
</HotKeys>
|
||||
|
|
|
|||
|
|
@ -38,6 +38,7 @@ const messages = defineMessages({
|
|||
admin_account: { id: 'status.admin_account', defaultMessage: 'Open moderation interface for @{name}' },
|
||||
admin_status: { id: 'status.admin_status', defaultMessage: 'Open this status in the moderation interface' },
|
||||
copy: { id: 'status.copy', defaultMessage: 'Copy link to status' },
|
||||
hide: { id: 'status.hide', defaultMessage: 'Hide toot' },
|
||||
blockDomain: { id: 'account.block_domain', defaultMessage: 'Block domain {domain}' },
|
||||
unblockDomain: { id: 'account.unblock_domain', defaultMessage: 'Unblock domain {domain}' },
|
||||
unmute: { id: 'account.unmute', defaultMessage: 'Unmute @{name}' },
|
||||
|
|
@ -76,6 +77,7 @@ class StatusActionBar extends ImmutablePureComponent {
|
|||
onMuteConversation: PropTypes.func,
|
||||
onPin: PropTypes.func,
|
||||
onBookmark: PropTypes.func,
|
||||
onFilter: PropTypes.func,
|
||||
withDismiss: PropTypes.bool,
|
||||
withCounters: PropTypes.bool,
|
||||
scrollKey: PropTypes.string,
|
||||
|
|
@ -207,6 +209,10 @@ class StatusActionBar extends ImmutablePureComponent {
|
|||
this.props.onMuteConversation(this.props.status);
|
||||
}
|
||||
|
||||
handleFilter = () => {
|
||||
this.props.onFilter();
|
||||
}
|
||||
|
||||
handleCopy = () => {
|
||||
const url = this.props.status.get('url');
|
||||
const textarea = document.createElement('textarea');
|
||||
|
|
@ -226,6 +232,11 @@ class StatusActionBar extends ImmutablePureComponent {
|
|||
}
|
||||
}
|
||||
|
||||
|
||||
handleFilterClick = () => {
|
||||
this.props.onFilter();
|
||||
}
|
||||
|
||||
render () {
|
||||
const { status, relationship, intl, withDismiss, withCounters, scrollKey } = this.props;
|
||||
|
||||
|
|
@ -329,6 +340,10 @@ class StatusActionBar extends ImmutablePureComponent {
|
|||
<IconButton className='status__action-bar-button' title={intl.formatMessage(messages.share)} icon='share-alt' onClick={this.handleShareClick} />
|
||||
);
|
||||
|
||||
const filterButton = this.props.onFilter && (
|
||||
<IconButton className='status__action-bar-button' title={intl.formatMessage(messages.hide)} icon='eye' onClick={this.handleFilterClick} />
|
||||
);
|
||||
|
||||
return (
|
||||
<div className='status__action-bar'>
|
||||
<IconButton className='status__action-bar-button' title={replyTitle} icon={status.get('in_reply_to_account_id') === status.getIn(['account', 'id']) ? 'reply' : replyIcon} onClick={this.handleReplyClick} counter={status.get('replies_count')} obfuscateCount />
|
||||
|
|
@ -337,6 +352,8 @@ class StatusActionBar extends ImmutablePureComponent {
|
|||
|
||||
{shareButton}
|
||||
|
||||
{filterButton}
|
||||
|
||||
<div className='status__action-bar-dropdown'>
|
||||
<DropdownMenuContainer
|
||||
scrollKey={scrollKey}
|
||||
|
|
|
|||
|
|
@ -13,7 +13,6 @@ import { debounce } from 'lodash';
|
|||
import { uploadCompose, resetCompose, changeComposeSpoilerness } from '../../actions/compose';
|
||||
import { expandHomeTimeline } from '../../actions/timelines';
|
||||
import { expandNotifications } from '../../actions/notifications';
|
||||
import { fetchFilters } from '../../actions/filters';
|
||||
import { fetchRules } from '../../actions/rules';
|
||||
import { clearHeight } from '../../actions/height_cache';
|
||||
import { focusApp, unfocusApp, changeLayout } from 'mastodon/actions/app';
|
||||
|
|
@ -368,7 +367,7 @@ class UI extends React.PureComponent {
|
|||
this.props.dispatch(fetchMarkers());
|
||||
this.props.dispatch(expandHomeTimeline());
|
||||
this.props.dispatch(expandNotifications());
|
||||
setTimeout(() => this.props.dispatch(fetchFilters()), 500);
|
||||
|
||||
setTimeout(() => this.props.dispatch(fetchRules()), 3000);
|
||||
|
||||
this.hotkeys.__mousetrap__.stopCallback = (e, element) => {
|
||||
|
|
|
|||
|
|
@ -1,10 +1,34 @@
|
|||
import { FILTERS_FETCH_SUCCESS } from '../actions/filters';
|
||||
import { List as ImmutableList, fromJS } from 'immutable';
|
||||
import { FILTERS_IMPORT } from '../actions/importer';
|
||||
import { Map as ImmutableMap, is, fromJS } from 'immutable';
|
||||
|
||||
export default function filters(state = ImmutableList(), action) {
|
||||
const normalizeFilter = (state, filter) => {
|
||||
const normalizedFilter = fromJS({
|
||||
id: filter.id,
|
||||
title: filter.title,
|
||||
context: filter.context,
|
||||
filter_action: filter.filter_action,
|
||||
expires_at: filter.expires_at ? Date.parse(filter.expires_at) : null,
|
||||
});
|
||||
|
||||
if (is(state.get(filter.id), normalizedFilter)) {
|
||||
return state;
|
||||
} else {
|
||||
return state.set(filter.id, normalizedFilter);
|
||||
}
|
||||
};
|
||||
|
||||
const normalizeFilters = (state, filters) => {
|
||||
filters.forEach(filter => {
|
||||
state = normalizeFilter(state, filter);
|
||||
});
|
||||
|
||||
return state;
|
||||
};
|
||||
|
||||
export default function filters(state = ImmutableMap(), action) {
|
||||
switch(action.type) {
|
||||
case FILTERS_FETCH_SUCCESS:
|
||||
return fromJS(action.filters);
|
||||
case FILTERS_IMPORT:
|
||||
return normalizeFilters(state, action.filters);
|
||||
default:
|
||||
return state;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -40,15 +40,15 @@ const toServerSideType = columnType => {
|
|||
const escapeRegExp = string =>
|
||||
string.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); // $& means the whole matched string
|
||||
|
||||
const regexFromFilters = filters => {
|
||||
if (filters.size === 0) {
|
||||
const regexFromKeywords = keywords => {
|
||||
if (keywords.size === 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return new RegExp(filters.map(filter => {
|
||||
let expr = escapeRegExp(filter.get('phrase'));
|
||||
return new RegExp(keywords.map(keyword_filter => {
|
||||
let expr = escapeRegExp(keyword_filter.get('keyword'));
|
||||
|
||||
if (filter.get('whole_word')) {
|
||||
if (keyword_filter.get('whole_word')) {
|
||||
if (/^[\w]/.test(expr)) {
|
||||
expr = `\\b${expr}`;
|
||||
}
|
||||
|
|
@ -62,27 +62,15 @@ const regexFromFilters = filters => {
|
|||
}).join('|'), 'i');
|
||||
};
|
||||
|
||||
// Memoize the filter regexps for each valid server contextType
|
||||
const makeGetFiltersRegex = () => {
|
||||
let memo = {};
|
||||
const getFilters = (state, { contextType }) => {
|
||||
if (!contextType) return null;
|
||||
|
||||
return (state, { contextType }) => {
|
||||
if (!contextType) return ImmutableList();
|
||||
const serverSideType = toServerSideType(contextType);
|
||||
const now = new Date();
|
||||
|
||||
const serverSideType = toServerSideType(contextType);
|
||||
const filters = state.get('filters', ImmutableList()).filter(filter => filter.get('context').includes(serverSideType) && (filter.get('expires_at') === null || Date.parse(filter.get('expires_at')) > (new Date())));
|
||||
|
||||
if (!memo[serverSideType] || !is(memo[serverSideType].filters, filters)) {
|
||||
const dropRegex = regexFromFilters(filters.filter(filter => filter.get('irreversible')));
|
||||
const regex = regexFromFilters(filters);
|
||||
memo[serverSideType] = { filters: filters, results: [dropRegex, regex] };
|
||||
}
|
||||
return memo[serverSideType].results;
|
||||
};
|
||||
return state.get('filters').filter((filter) => filter.get('context').includes(serverSideType) && (filter.get('expires_at') === null || filter.get('expires_at') > now));
|
||||
};
|
||||
|
||||
export const getFiltersRegex = makeGetFiltersRegex();
|
||||
|
||||
export const makeGetStatus = () => {
|
||||
return createSelector(
|
||||
[
|
||||
|
|
@ -90,10 +78,10 @@ export const makeGetStatus = () => {
|
|||
(state, { id }) => state.getIn(['statuses', state.getIn(['statuses', id, 'reblog'])]),
|
||||
(state, { id }) => state.getIn(['accounts', state.getIn(['statuses', id, 'account'])]),
|
||||
(state, { id }) => state.getIn(['accounts', state.getIn(['statuses', state.getIn(['statuses', id, 'reblog']), 'account'])]),
|
||||
getFiltersRegex,
|
||||
getFilters,
|
||||
],
|
||||
|
||||
(statusBase, statusReblog, accountBase, accountReblog, filtersRegex) => {
|
||||
(statusBase, statusReblog, accountBase, accountReblog, filters) => {
|
||||
if (!statusBase) {
|
||||
return null;
|
||||
}
|
||||
|
|
@ -104,14 +92,17 @@ export const makeGetStatus = () => {
|
|||
statusReblog = null;
|
||||
}
|
||||
|
||||
const dropRegex = (accountReblog || accountBase).get('id') !== me && filtersRegex[0];
|
||||
if (dropRegex && dropRegex.test(statusBase.get('reblog') ? statusReblog.get('search_index') : statusBase.get('search_index'))) {
|
||||
return null;
|
||||
let filtered = false;
|
||||
if ((accountReblog || accountBase).get('id') !== me && filters) {
|
||||
let filterResults = statusReblog?.get('filtered') || statusBase.get('filtered') || ImmutableList();
|
||||
if (filterResults.some((result) => filters.getIn([result.get('filter'), 'filter_action']) === 'hide')) {
|
||||
return null;
|
||||
}
|
||||
if (!filterResults.isEmpty()) {
|
||||
filtered = filterResults.map(result => filters.getIn([result.get('filter'), 'title']));
|
||||
}
|
||||
}
|
||||
|
||||
const regex = (accountReblog || accountBase).get('id') !== me && filtersRegex[1];
|
||||
const filtered = regex && regex.test(statusBase.get('reblog') ? statusReblog.get('search_index') : statusBase.get('search_index'));
|
||||
|
||||
return statusBase.withMutations(map => {
|
||||
map.set('reblog', statusReblog);
|
||||
map.set('account', accountBase);
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue