Add new public status index (#26344)

Co-authored-by: Eugen Rochko <eugen@zeonfederated.com>
Co-authored-by: Claire <claire.github-309c@sitedethib.com>

app/lib/importer/public_statuses_index_importer.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+class Importer::PublicStatusesIndexImporter < Importer::BaseImporter
+  def import!
+    indexable_statuses_scope.find_in_batches(batch_size: @batch_size) do |batch|
+      in_work_unit(batch.map(&:status_id)) do |status_ids|
+        bulk = ActiveRecord::Base.connection_pool.with_connection do
+          Chewy::Index::Import::BulkBuilder.new(index, to_index: Status.includes(:media_attachments, :preloadable_poll).where(id: status_ids)).bulk_body
+        end
+
+        indexed = 0
+        deleted = 0
+
+        bulk.map! do |entry|
+          if entry[:index]
+            indexed += 1
+          else
+            deleted += 1
+          end
+          entry
+        end
+
+        Chewy::Index::Import::BulkRequest.new(index).perform(bulk)
+
+        [indexed, deleted]
+      end
+    end
+
+    wait!
+  end
+
+  private
+
+  def index
+    PublicStatusesIndex
+  end
+
+  def indexable_statuses_scope
+    Status.indexable.select('"statuses"."id", COALESCE("statuses"."reblog_of_id", "statuses"."id") AS status_id')
+  end
+end

app/lib/search_query_transformer.rb

@@ -36,7 +36,7 @@ class SearchQueryTransformer < Parslet::Transform
     def clause_to_filter(clause)
       case clause
       when PrefixClause
-        { term: { clause.filter => clause.term } }
+        { clause.type => { clause.filter => clause.term } }
       else
         raise "Unexpected clause type: #{clause}"
       end
@@ -47,12 +47,10 @@ class SearchQueryTransformer < Parslet::Transform
     class << self
       def symbol(str)
         case str
-        when '+'
+        when '+', nil
           :must
         when '-'
           :must_not
-        when nil
-          :should
         else
           raise "Unknown operator: #{str}"
         end
@@ -81,23 +79,52 @@ class SearchQueryTransformer < Parslet::Transform
   end
 
   class PrefixClause
-    attr_reader :filter, :operator, :term
+    attr_reader :type, :filter, :operator, :term
 
     def initialize(prefix, term)
       @operator = :filter
+
       case prefix
+      when 'has', 'is'
+        @filter = :properties
+        @type = :term
+        @term = term
+      when 'language'
+        @filter = :language
+        @type = :term
+        @term = term
       when 'from'
         @filter = :account_id
-
-        username, domain = term.gsub(/\A@/, '').split('@')
-        domain           = nil if TagManager.instance.local_domain?(domain)
-        account          = Account.find_remote!(username, domain)
-
-        @term = account.id
+        @type = :term
+        @term = account_id_from_term(term)
+      when 'before'
+        @filter = :created_at
+        @type = :range
+        @term = { lt: term }
+      when 'after'
+        @filter = :created_at
+        @type = :range
+        @term = { gt: term }
+      when 'during'
+        @filter = :created_at
+        @type = :range
+        @term = { gte: term, lte: term }
       else
         raise Mastodon::SyntaxError
       end
     end
+
+    private
+
+    def account_id_from_term(term)
+      username, domain = term.gsub(/\A@/, '').split('@')
+      domain = nil if TagManager.instance.local_domain?(domain)
+      account = Account.find_remote(username, domain)
+
+      # If the account is not found, we want to return empty results, so return
+      # an ID that does not exist
+      account&.id || -1
+    end
   end
 
   rule(clause: subtree(:clause)) do

app/lib/vacuum/statuses_vacuum.rb

@@ -20,7 +20,10 @@ class Vacuum::StatusesVacuum
       statuses.direct_visibility
               .includes(mentions: :account)
               .find_each(&:unlink_from_conversations!)
-      remove_from_search_index(statuses.ids) if Chewy.enabled?
+      if Chewy.enabled?
+        remove_from_index(statuses.ids, 'chewy:queue:StatusesIndex')
+        remove_from_index(statuses.ids, 'chewy:queue:PublicStatusesIndex')
+      end
 
       # Foreign keys take care of most associated records for us.
       # Media attachments will be orphaned.
@@ -38,7 +41,7 @@ class Vacuum::StatusesVacuum
     Mastodon::Snowflake.id_at(@retention_period.ago, with_random: false)
   end
 
-  def remove_from_search_index(status_ids)
-    with_redis { |redis| redis.sadd('chewy:queue:StatusesIndex', status_ids) }
+  def remove_from_index(status_ids, index)
+    with_redis { |redis| redis.sadd(index, status_ids) }
   end
 end