Add Elasticsearch cluster health check and indexes mismatch check to dashboard (#26448)

app/chewy/instances_index.rb

@@ -6,7 +6,7 @@ class InstancesIndex < Chewy::Index
   index_scope ::Instance.searchable
 
   root date_detection: false do
-    field :domain, type: 'text', index_prefixes: { min_chars: 1 }
+    field :domain, type: 'text', index_prefixes: { min_chars: 1, max_chars: 5 }
     field :accounts_count, type: 'long'
   end
 end

app/lib/admin/system_check/elasticsearch_check.rb

@@ -1,6 +1,13 @@
 # frozen_string_literal: true
 
 class Admin::SystemCheck::ElasticsearchCheck < Admin::SystemCheck::BaseCheck
+  INDEXES = [
+    InstancesIndex,
+    AccountsIndex,
+    TagsIndex,
+    StatusesIndex,
+  ].freeze
+
   def skip?
     !current_user.can?(:view_devops)
   end
@@ -8,11 +15,15 @@ class Admin::SystemCheck::ElasticsearchCheck < Admin::SystemCheck::BaseCheck
   def pass?
     return true unless Chewy.enabled?
 
-    running_version.present? && compatible_version?
+    running_version.present? && compatible_version? && cluster_health['status'] == 'green' && indexes_match? && preset_matches?
+  rescue Faraday::ConnectionFailed, Elasticsearch::Transport::Transport::Error
+    false
   end
 
   def message
-    if running_version.present?
+    if running_version.blank?
+      Admin::SystemCheck::Message.new(:elasticsearch_running_check)
+    elsif !compatible_version?
       Admin::SystemCheck::Message.new(
         :elasticsearch_version_check,
         I18n.t(
@@ -21,13 +32,32 @@ class Admin::SystemCheck::ElasticsearchCheck < Admin::SystemCheck::BaseCheck
           required_version: required_version
         )
       )
+    elsif !indexes_match?
+      Admin::SystemCheck::Message.new(
+        :elasticsearch_index_mismatch,
+        mismatched_indexes.join(' ')
+      )
+    elsif cluster_health['status'] == 'red'
+      Admin::SystemCheck::Message.new(:elasticsearch_health_red)
+    elsif cluster_health['number_of_nodes'] < 2 && es_preset != 'single_node_cluster'
+      Admin::SystemCheck::Message.new(:elasticsearch_preset_single_node, nil, 'https://docs.joinmastodon.org/admin/optional/elasticsearch/#scaling')
+    elsif Chewy.client.indices.get_settings['chewy_specifications'].dig('settings', 'index', 'number_of_replicas')&.to_i&.positive? && es_preset == 'single_node_cluster'
+      Admin::SystemCheck::Message.new(:elasticsearch_reset_chewy)
+    elsif cluster_health['status'] == 'yellow'
+      Admin::SystemCheck::Message.new(:elasticsearch_health_yellow)
     else
-      Admin::SystemCheck::Message.new(:elasticsearch_running_check)
+      Admin::SystemCheck::Message.new(:elasticsearch_preset, nil, 'https://docs.joinmastodon.org/admin/optional/elasticsearch/#scaling')
     end
+  rescue Faraday::ConnectionFailed, Elasticsearch::Transport::Transport::Error
+    Admin::SystemCheck::Message.new(:elasticsearch_running_check)
   end
 
   private
 
+  def cluster_health
+    @cluster_health ||= Chewy.client.cluster.health
+  end
+
   def running_version
     @running_version ||= begin
       Chewy.client.info['version']['number']
@@ -49,5 +79,30 @@ class Admin::SystemCheck::ElasticsearchCheck < Admin::SystemCheck::BaseCheck
 
     Gem::Version.new(running_version) >= Gem::Version.new(required_version) ||
       Gem::Version.new(compatible_wire_version) >= Gem::Version.new(required_version)
+  rescue ArgumentError
+    false
+  end
+
+  def mismatched_indexes
+    @mismatched_indexes ||= INDEXES.filter_map do |klass|
+      klass.index_name if Chewy.client.indices.get_mapping[klass.index_name]&.deep_symbolize_keys != klass.mappings_hash
+    end
+  end
+
+  def indexes_match?
+    mismatched_indexes.empty?
+  end
+
+  def es_preset
+    ENV.fetch('ES_PRESET', 'single_node_cluster')
+  end
+
+  def preset_matches?
+    case es_preset
+    when 'single_node_cluster'
+      cluster_health['number_of_nodes'] == 1
+    else
+      cluster_health['number_of_nodes'] > 1
+    end
   end
 end

config/locales/en.yml

@@ -814,6 +814,20 @@ en:
     system_checks:
       database_schema_check:
         message_html: There are pending database migrations. Please run them to ensure the application behaves as expected
+      elasticsearch_health_red:
+        message_html: Elasticsearch cluster is unhealthy (red status), search features are unavailable
+      elasticsearch_health_yellow:
+        message_html: Elasticsearch cluster is unhealthy (yellow status), you may want to investigate the reason
+      elasticsearch_index_mismatch:
+        message_html: Elasticsearch index mappings are outdated. Please run <code>tootctl search deploy --only=%{value}</code>
+      elasticsearch_preset:
+        action: See documentation
+        message_html: Your Elasticsearch cluster has more than one node, but Mastodon is not configured to use them.
+      elasticsearch_preset_single_node:
+        action: See documentation
+        message_html: Your Elasticsearch cluster has only one node, <code>ES_PRESET</code> should be set to <code>single_node_cluster</code>.
+      elasticsearch_reset_chewy:
+        message_html: Your Elasticsearch system index is outdated due to a setting change. Please run <code>tootctl search deploy --reset-chewy</code> to update it.
       elasticsearch_running_check:
         message_html: Could not connect to Elasticsearch. Please check that it is running, or disable full-text search
       elasticsearch_version_check:

spec/lib/admin/system_check/elasticsearch_check_spec.rb

@@ -11,7 +11,25 @@ describe Admin::SystemCheck::ElasticsearchCheck do
 
   describe 'pass?' do
     context 'when chewy is enabled' do
-      before { allow(Chewy).to receive(:enabled?).and_return(true) }
+      before do
+        allow(Chewy).to receive(:enabled?).and_return(true)
+        allow(Chewy.client.cluster).to receive(:health).and_return({ 'status' => 'green', 'number_of_nodes' => 1 })
+        allow(Chewy.client.indices).to receive(:get_mapping).and_return({
+          AccountsIndex.index_name => AccountsIndex.mappings_hash.deep_stringify_keys,
+          StatusesIndex.index_name => StatusesIndex.mappings_hash.deep_stringify_keys,
+          InstancesIndex.index_name => InstancesIndex.mappings_hash.deep_stringify_keys,
+          TagsIndex.index_name => TagsIndex.mappings_hash.deep_stringify_keys,
+        })
+        allow(Chewy.client.indices).to receive(:get_settings).and_return({
+          'chewy_specifications' => {
+            'settings' => {
+              'index' => {
+                'number_of_replicas' => 0,
+              },
+            },
+          },
+        })
+      end
 
       context 'when running version is present and high enough' do
         before do
@@ -67,8 +85,19 @@ describe Admin::SystemCheck::ElasticsearchCheck do
   end
 
   describe 'message' do
+    before do
+      allow(Chewy).to receive(:enabled?).and_return(true)
+      allow(Chewy.client.cluster).to receive(:health).and_return({ 'status' => 'green', 'number_of_nodes' => 1 })
+      allow(Chewy.client.indices).to receive(:get_mapping).and_return({
+        AccountsIndex.index_name => AccountsIndex.mappings_hash.deep_stringify_keys,
+        StatusesIndex.index_name => StatusesIndex.mappings_hash.deep_stringify_keys,
+        InstancesIndex.index_name => InstancesIndex.mappings_hash.deep_stringify_keys,
+        TagsIndex.index_name => TagsIndex.mappings_hash.deep_stringify_keys,
+      })
+    end
+
     context 'when running version is present' do
-      before { allow(Chewy.client).to receive(:info).and_return({ 'version' => { 'number' => '999.99.9' } }) }
+      before { allow(Chewy.client).to receive(:info).and_return({ 'version' => { 'number' => '1.2.3' } }) }
 
       it 'sends class name symbol to message instance' do
         allow(Admin::SystemCheck::Message).to receive(:new)
@@ -77,7 +106,7 @@ describe Admin::SystemCheck::ElasticsearchCheck do
         check.message
 
         expect(Admin::SystemCheck::Message).to have_received(:new)
-          .with(:elasticsearch_version_check, 'Elasticsearch 999.99.9 is running while 7.x is required')
+          .with(:elasticsearch_version_check, 'Elasticsearch 1.2.3 is running while 7.x is required')
       end
     end