Full-text search for authorized statuses (#6423)

* Add full-text search for authorized statuses

- Search API will return statuses that match the query
- Only for logged in users
- Only if you are author of the status,
- Or you were mentioned in it
- Or you favourited or reblogged it
- Configuration over `ES_ENABLED`, `ES_HOST`, `ES_PORT`, `ES_PREFIX`
- Run `rails chewy:deploy` to create & populate index

Fix #5880
Fix #4293
Fix #1152

* Add commented out docker-compose configuration for ES container

* Optimize index import, filter search results

* Add basic normalization to the index

* Add better stemming and normalization to the index

* Skip webfinger request if search query includes both @ and a space

* Fix code style

* Visually separate search result sections

* Fix code style issues

app/chewy/statuses_index.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+class StatusesIndex < Chewy::Index
+  settings index: { refresh_interval: '15m' }, analysis: {
+    filter: {
+      english_stop: {
+        type: 'stop',
+        stopwords: '_english_',
+      },
+      english_stemmer: {
+        type: 'stemmer',
+        language: 'english',
+      },
+      english_possessive_stemmer: {
+        type: 'stemmer',
+        language: 'possessive_english',
+      },
+    },
+    analyzer: {
+      content: {
+        tokenizer: 'uax_url_email',
+        filter: %w(
+          english_possessive_stemmer
+          lowercase
+          asciifolding
+          cjk_width
+          english_stop
+          english_stemmer
+        ),
+      },
+    },
+  }
+
+  define_type ::Status.without_reblogs do
+    crutch :mentions do |collection|
+      data = ::Mention.where(status_id: collection.map(&:id)).pluck(:status_id, :account_id)
+      data.each.with_object({}) { |(id, name), result| (result[id] ||= []).push(name) }
+    end
+
+    crutch :favourites do |collection|
+      data = ::Favourite.where(status_id: collection.map(&:id)).pluck(:status_id, :account_id)
+      data.each.with_object({}) { |(id, name), result| (result[id] ||= []).push(name) }
+    end
+
+    crutch :reblogs do |collection|
+      data = ::Status.where(reblog_of_id: collection.map(&:id)).pluck(:reblog_of_id, :account_id)
+      data.each.with_object({}) { |(id, name), result| (result[id] ||= []).push(name) }
+    end
+
+    root date_detection: false do
+      field :account_id, type: 'long'
+
+      field :text, type: 'text', value: ->(status) { [status.spoiler_text, Formatter.instance.plaintext(status)].join("\n\n") } do
+        field :stemmed, type: 'text', analyzer: 'content'
+      end
+
+      field :searchable_by, type: 'long', value: ->(status, crutches) { status.searchable_by(crutches) }
+      field :created_at, type: 'date'
+    end
+  end
+end

app/javascript/mastodon/features/compose/components/search_results.js

@@ -22,6 +22,8 @@ export default class SearchResults extends ImmutablePureComponent {
       count   += results.get('accounts').size;
       accounts = (
         <div className='search-results__section'>
+          <h5><FormattedMessage id='search_results.accounts' defaultMessage='People' /></h5>
+
           {results.get('accounts').map(accountId => <AccountContainer key={accountId} id={accountId} />)}
         </div>
       );
@@ -31,6 +33,8 @@ export default class SearchResults extends ImmutablePureComponent {
       count   += results.get('statuses').size;
       statuses = (
         <div className='search-results__section'>
+          <h5><FormattedMessage id='search_results.statuses' defaultMessage='Toots' /></h5>
+
           {results.get('statuses').map(statusId => <StatusContainer key={statusId} id={statusId} />)}
         </div>
       );
@@ -40,6 +44,8 @@ export default class SearchResults extends ImmutablePureComponent {
       count += results.get('hashtags').size;
       hashtags = (
         <div className='search-results__section'>
+          <h5><FormattedMessage id='search_results.hashtags' defaultMessage='Hashtags' /></h5>
+
           {results.get('hashtags').map(hashtag => (
             <Link key={hashtag} className='search-results__hashtag' to={`/timelines/tag/${hashtag}`}>
               #{hashtag}

app/javascript/styles/mastodon/components.scss

@@ -1786,7 +1786,7 @@
   flex: 1;
   min-height: 47px;
 
-  > img { 
+  > img {
     display: block;
     object-fit: contain;
     object-position: bottom left;
@@ -3229,6 +3229,43 @@
   font-weight: 500;
 }
 
+.search-results__section {
+  margin-bottom: 20px;
+
+  h5 {
+    position: relative;
+
+    &::before {
+      content: "";
+      display: block;
+      position: absolute;
+      left: 0;
+      right: 0;
+      top: 50%;
+      width: 100%;
+      height: 0;
+      border-top: 1px solid lighten($ui-base-color, 8%);
+    }
+
+    span {
+      display: inline-block;
+      background: $ui-base-color;
+      color: $ui-primary-color;
+      font-size: 14px;
+      font-weight: 500;
+      padding: 10px;
+      position: relative;
+      z-index: 1;
+      cursor: default;
+    }
+  }
+
+  .account:last-child,
+  & > div:last-child .status {
+    border-bottom: 0;
+  }
+}
+
 .search-results__hashtag {
   display: block;
   padding: 10px;

app/lib/status_filter.rb

@@ -9,6 +9,7 @@ class StatusFilter
   end
 
   def filtered?
+    return false if !account.nil? && account.id == status.account_id
     blocked_by_policy? || (account_present? && filtered_status?) || silenced_account?
   end
 

app/models/favourite.rb

@@ -13,6 +13,8 @@
 class Favourite < ApplicationRecord
   include Paginable
 
+  update_index('statuses#status', :status) if Chewy.enabled?
+
   belongs_to :account, inverse_of: :favourites
   belongs_to :status,  inverse_of: :favourites, counter_cache: true
 

app/models/status.rb

@@ -31,6 +31,8 @@ class Status < ApplicationRecord
   include Cacheable
   include StatusThreadingConcern
 
+  update_index('statuses#status', :proper) if Chewy.enabled?
+
   enum visibility: [:public, :unlisted, :private, :direct], _suffix: :visibility
 
   belongs_to :application, class_name: 'Doorkeeper::Application', optional: true
@@ -78,6 +80,22 @@ class Status < ApplicationRecord
 
   delegate :domain, to: :account, prefix: true
 
+  def searchable_by(preloaded = nil)
+    ids = [account_id]
+
+    if preloaded.nil?
+      ids += mentions.pluck(:account_id)
+      ids += favourites.pluck(:account_id)
+      ids += reblogs.pluck(:account_id)
+    else
+      ids += preloaded.mentions[id] || []
+      ids += preloaded.favourites[id] || []
+      ids += preloaded.reblogs[id] || []
+    end
+
+    ids.uniq
+  end
+
   def reply?
     !in_reply_to_id.nil? || attributes['reply']
   end

app/services/search_service.rb

@@ -1,21 +1,43 @@
 # frozen_string_literal: true
 
 class SearchService < BaseService
-  attr_accessor :query
+  attr_accessor :query, :account, :limit, :resolve
 
   def call(query, limit, resolve = false, account = nil)
-    @query = query
+    @query   = query
+    @account = account
+    @limit   = limit
+    @resolve = resolve
 
     default_results.tap do |results|
       if url_query?
         results.merge!(url_resource_results) unless url_resource.nil?
       elsif query.present?
-        results[:accounts] = AccountSearchService.new.call(query, limit, account, resolve: resolve)
-        results[:hashtags] = Tag.search_for(query.gsub(/\A#/, ''), limit) unless query.start_with?('@')
+        results[:accounts] = perform_accounts_search! if account_searchable?
+        results[:statuses] = perform_statuses_search! if full_text_searchable?
+        results[:hashtags] = perform_hashtags_search! if hashtag_searchable?
       end
     end
   end
 
+  private
+
+  def perform_accounts_search!
+    AccountSearchService.new.call(query, limit, account, resolve: resolve)
+  end
+
+  def perform_statuses_search!
+    statuses = StatusesIndex.filter(term: { searchable_by: account.id })
+                            .query(multi_match: { type: 'most_fields', query: query, operator: 'and', fields: %w(text text.stemmed) })
+                            .limit(limit).objects
+
+    statuses.reject { |status| StatusFilter.new(status, account).filtered? }
+  end
+
+  def perform_hashtags_search!
+    Tag.search_for(query.gsub(/\A#/, ''), limit)
+  end
+
   def default_results
     { accounts: [], hashtags: [], statuses: [] }
   end
@@ -35,4 +57,17 @@ class SearchService < BaseService
   def url_resource_symbol
     url_resource.class.name.downcase.pluralize.to_sym
   end
+
+  def full_text_searchable?
+    return false unless Chewy.enabled?
+    !account.nil? && !((query.start_with?('#') || query.include?('@')) && !query.include?(' '))
+  end
+
+  def account_searchable?
+    !(query.include?('@') && query.include?(' '))
+  end
+
+  def hashtag_searchable?
+    !query.include?('@')
+  end
 end