Fix null values being included in some indexes (#17711)

* Fix null values being included in some indexes

* Update lib/mastodon/migration_helpers.rb

Co-authored-by: Claire <claire.github-309c@sitedethib.com>

* Add documentation link to corruption error message

Co-authored-by: Claire <claire.github-309c@sitedethib.com>

db/migrate/20200620164023_add_fixed_lowercase_index_to_accounts.rb

@@ -16,7 +16,7 @@ class AddFixedLowercaseIndexToAccounts < ActiveRecord::Migration[5.2]
       add_index :accounts, "lower (username), COALESCE(lower(domain), '')", name: 'index_accounts_on_username_and_domain_lower', unique: true, algorithm: :concurrently
     rescue ActiveRecord::RecordNotUnique
       remove_index :accounts, name: 'index_accounts_on_username_and_domain_lower'
-      raise CorruptionError
+      raise CorruptionError.new('index_accounts_on_username_and_domain_lower')
     end
 
     remove_index :accounts, name: 'old_index_accounts_on_username_and_domain_lower' if index_name_exists?(:accounts, 'old_index_accounts_on_username_and_domain_lower')

db/migrate/20210421121431_add_case_insensitive_btree_index_to_tags.rb

@@ -10,7 +10,7 @@ class AddCaseInsensitiveBtreeIndexToTags < ActiveRecord::Migration[5.2]
       safety_assured { execute 'CREATE UNIQUE INDEX CONCURRENTLY index_tags_on_name_lower_btree ON tags (lower(name) text_pattern_ops)' }
     rescue ActiveRecord::StatementInvalid => e
       remove_index :tags, name: 'index_tags_on_name_lower_btree'
-      raise CorruptionError if e.is_a?(ActiveRecord::RecordNotUnique)
+      raise CorruptionError.new('index_tags_on_name_lower_btree') if e.is_a?(ActiveRecord::RecordNotUnique)
       raise e
     end
 

db/post_migrate/20220307083603_optimize_null_index_conversations_uri.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+require Rails.root.join('lib', 'mastodon', 'migration_helpers')
+
+class OptimizeNullIndexConversationsUri < ActiveRecord::Migration[5.2]
+  include Mastodon::MigrationHelpers
+
+  disable_ddl_transaction!
+
+  def up
+    update_index :conversations, 'index_conversations_on_uri', :uri, unique: true, where: 'uri IS NOT NULL', opclass: :text_pattern_ops
+  end
+
+  def down
+    update_index :conversations, 'index_conversations_on_uri', :uri, unique: true
+  end
+end

db/post_migrate/20220310060545_optimize_null_index_statuses_in_reply_to_account_id.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+require Rails.root.join('lib', 'mastodon', 'migration_helpers')
+
+class OptimizeNullIndexStatusesInReplyToAccountId < ActiveRecord::Migration[5.2]
+  include Mastodon::MigrationHelpers
+
+  disable_ddl_transaction!
+
+  def up
+    update_index :statuses, 'index_statuses_on_in_reply_to_account_id', :in_reply_to_account_id, where: 'in_reply_to_account_id IS NOT NULL'
+  end
+
+  def down
+    update_index :statuses, 'index_statuses_on_in_reply_to_account_id', :in_reply_to_account_id
+  end
+end

db/post_migrate/20220310060556_optimize_null_index_statuses_in_reply_to_id.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+require Rails.root.join('lib', 'mastodon', 'migration_helpers')
+
+class OptimizeNullIndexStatusesInReplyToId < ActiveRecord::Migration[5.2]
+  include Mastodon::MigrationHelpers
+
+  disable_ddl_transaction!
+
+  def up
+    update_index :statuses, 'index_statuses_on_in_reply_to_id', :in_reply_to_id, where: 'in_reply_to_id IS NOT NULL'
+  end
+
+  def down
+    update_index :statuses, 'index_statuses_on_in_reply_to_id', :in_reply_to_id
+  end
+end

db/post_migrate/20220310060614_optimize_null_index_media_attachments_scheduled_status_id.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+require Rails.root.join('lib', 'mastodon', 'migration_helpers')
+
+class OptimizeNullIndexMediaAttachmentsScheduledStatusId < ActiveRecord::Migration[5.2]
+  include Mastodon::MigrationHelpers
+
+  disable_ddl_transaction!
+
+  def up
+    update_index :media_attachments, 'index_media_attachments_on_scheduled_status_id', :scheduled_status_id, where: 'scheduled_status_id IS NOT NULL'
+  end
+
+  def down
+    update_index :media_attachments, 'index_media_attachments_on_scheduled_status_id', :scheduled_status_id
+  end
+end

db/post_migrate/20220310060626_optimize_null_index_media_attachments_shortcode.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+require Rails.root.join('lib', 'mastodon', 'migration_helpers')
+
+class OptimizeNullIndexMediaAttachmentsShortcode < ActiveRecord::Migration[5.2]
+  include Mastodon::MigrationHelpers
+
+  disable_ddl_transaction!
+
+  def up
+    update_index :media_attachments, 'index_media_attachments_on_shortcode', :shortcode, unique: true, where: 'shortcode IS NOT NULL', opclass: :text_pattern_ops
+  end
+
+  def down
+    update_index :media_attachments, 'index_media_attachments_on_shortcode', :shortcode, unique: true
+  end
+end

db/post_migrate/20220310060641_optimize_null_index_users_reset_password_token.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+require Rails.root.join('lib', 'mastodon', 'migration_helpers')
+
+class OptimizeNullIndexUsersResetPasswordToken < ActiveRecord::Migration[5.2]
+  include Mastodon::MigrationHelpers
+
+  disable_ddl_transaction!
+
+  def up
+    update_index :users, 'index_users_on_reset_password_token', :reset_password_token, unique: true, where: 'reset_password_token IS NOT NULL', opclass: :text_pattern_ops
+  end
+
+  def down
+    update_index :users, 'index_users_on_reset_password_token', :reset_password_token, unique: true
+  end
+end

db/post_migrate/20220310060653_optimize_null_index_users_created_by_application_id.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+require Rails.root.join('lib', 'mastodon', 'migration_helpers')
+
+class OptimizeNullIndexUsersCreatedByApplicationId < ActiveRecord::Migration[5.2]
+  include Mastodon::MigrationHelpers
+
+  disable_ddl_transaction!
+
+  def up
+    update_index :users, 'index_users_on_created_by_application_id', :created_by_application_id, where: 'created_by_application_id IS NOT NULL'
+  end
+
+  def down
+    update_index :users, 'index_users_on_created_by_application_id', :created_by_application_id
+  end
+end

db/post_migrate/20220310060706_optimize_null_index_statuses_uri.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+require Rails.root.join('lib', 'mastodon', 'migration_helpers')
+
+class OptimizeNullIndexStatusesUri < ActiveRecord::Migration[5.2]
+  include Mastodon::MigrationHelpers
+
+  disable_ddl_transaction!
+
+  def up
+    update_index :statuses, 'index_statuses_on_uri', :uri, unique: true, where: 'uri IS NOT NULL', opclass: :text_pattern_ops
+  end
+
+  def down
+    update_index :statuses, 'index_statuses_on_uri', :uri, unique: true
+  end
+end

db/post_migrate/20220310060722_optimize_null_index_accounts_moved_to_account_id.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+require Rails.root.join('lib', 'mastodon', 'migration_helpers')
+
+class OptimizeNullIndexAccountsMovedToAccountId < ActiveRecord::Migration[5.2]
+  include Mastodon::MigrationHelpers
+
+  disable_ddl_transaction!
+
+  def up
+    update_index :accounts, 'index_accounts_on_moved_to_account_id', :moved_to_account_id, where: 'moved_to_account_id IS NOT NULL'
+  end
+
+  def down
+    update_index :accounts, 'index_accounts_on_moved_to_account_id', :moved_to_account_id
+  end
+end

db/post_migrate/20220310060740_optimize_null_index_oauth_access_tokens_refresh_token.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+require Rails.root.join('lib', 'mastodon', 'migration_helpers')
+
+class OptimizeNullIndexOauthAccessTokensRefreshToken < ActiveRecord::Migration[5.2]
+  include Mastodon::MigrationHelpers
+
+  disable_ddl_transaction!
+
+  def up
+    update_index :oauth_access_tokens, 'index_oauth_access_tokens_on_refresh_token', :refresh_token, unique: true, where: 'refresh_token IS NOT NULL', opclass: :text_pattern_ops
+  end
+
+  def down
+    update_index :oauth_access_tokens, 'index_oauth_access_tokens_on_refresh_token', :refresh_token, unique: true
+  end
+end

db/post_migrate/20220310060750_optimize_null_index_accounts_url.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+require Rails.root.join('lib', 'mastodon', 'migration_helpers')
+
+class OptimizeNullIndexAccountsURL < ActiveRecord::Migration[5.2]
+  include Mastodon::MigrationHelpers
+
+  disable_ddl_transaction!
+
+  def up
+    update_index :accounts, 'index_accounts_on_url', :url, where: 'url IS NOT NULL', opclass: :text_pattern_ops
+  end
+
+  def down
+    update_index :accounts, 'index_accounts_on_url', :url
+  end
+end

db/post_migrate/20220310060809_optimize_null_index_oauth_access_tokens_resource_owner_id.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+require Rails.root.join('lib', 'mastodon', 'migration_helpers')
+
+class OptimizeNullIndexOauthAccessTokensResourceOwnerId < ActiveRecord::Migration[5.2]
+  include Mastodon::MigrationHelpers
+
+  disable_ddl_transaction!
+
+  def up
+    update_index :oauth_access_tokens, 'index_oauth_access_tokens_on_resource_owner_id', :resource_owner_id, where: 'resource_owner_id IS NOT NULL'
+  end
+
+  def down
+    update_index :oauth_access_tokens, 'index_oauth_access_tokens_on_resource_owner_id', :resource_owner_id
+  end
+end

db/post_migrate/20220310060833_optimize_null_index_announcement_reactions_custom_emoji_id.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+require Rails.root.join('lib', 'mastodon', 'migration_helpers')
+
+class OptimizeNullIndexAnnouncementReactionsCustomEmojiId < ActiveRecord::Migration[5.2]
+  include Mastodon::MigrationHelpers
+
+  disable_ddl_transaction!
+
+  def up
+    update_index :announcement_reactions, 'index_announcement_reactions_on_custom_emoji_id', :custom_emoji_id, where: 'custom_emoji_id IS NOT NULL'
+  end
+
+  def down
+    update_index :announcement_reactions, 'index_announcement_reactions_on_custom_emoji_id', :custom_emoji_id
+  end
+end

db/post_migrate/20220310060854_optimize_null_index_appeals_approved_by_account_id.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+require Rails.root.join('lib', 'mastodon', 'migration_helpers')
+
+class OptimizeNullIndexAppealsApprovedByAccountId < ActiveRecord::Migration[5.2]
+  include Mastodon::MigrationHelpers
+
+  disable_ddl_transaction!
+
+  def up
+    update_index :appeals, 'index_appeals_on_approved_by_account_id', :approved_by_account_id, where: 'approved_by_account_id IS NOT NULL'
+  end
+
+  def down
+    update_index :appeals, 'index_appeals_on_approved_by_account_id', :approved_by_account_id
+  end
+end

db/post_migrate/20220310060913_optimize_null_index_account_migrations_target_account_id.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+require Rails.root.join('lib', 'mastodon', 'migration_helpers')
+
+class OptimizeNullIndexAccountMigrationsTargetAccountId < ActiveRecord::Migration[5.2]
+  include Mastodon::MigrationHelpers
+
+  disable_ddl_transaction!
+
+  def up
+    update_index :account_migrations, 'index_account_migrations_on_target_account_id', :target_account_id, where: 'target_account_id IS NOT NULL'
+  end
+
+  def down
+    update_index :account_migrations, 'index_account_migrations_on_target_account_id', :target_account_id
+  end
+end

db/post_migrate/20220310060926_optimize_null_index_appeals_rejected_by_account_id.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+require Rails.root.join('lib', 'mastodon', 'migration_helpers')
+
+class OptimizeNullIndexAppealsRejectedByAccountId < ActiveRecord::Migration[5.2]
+  include Mastodon::MigrationHelpers
+
+  disable_ddl_transaction!
+
+  def up
+    update_index :appeals, 'index_appeals_on_rejected_by_account_id', :rejected_by_account_id, where: 'rejected_by_account_id IS NOT NULL'
+  end
+
+  def down
+    update_index :appeals, 'index_appeals_on_rejected_by_account_id', :rejected_by_account_id
+  end
+end

db/post_migrate/20220310060939_optimize_null_index_list_accounts_follow_id.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+require Rails.root.join('lib', 'mastodon', 'migration_helpers')
+
+class OptimizeNullIndexListAccountsFollowId < ActiveRecord::Migration[5.2]
+  include Mastodon::MigrationHelpers
+
+  disable_ddl_transaction!
+
+  def up
+    update_index :list_accounts, 'index_list_accounts_on_follow_id', :follow_id, where: 'follow_id IS NOT NULL'
+  end
+
+  def down
+    update_index :list_accounts, 'index_list_accounts_on_follow_id', :follow_id
+  end
+end

db/post_migrate/20220310060959_optimize_null_index_web_push_subscriptions_access_token_id.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+require Rails.root.join('lib', 'mastodon', 'migration_helpers')
+
+class OptimizeNullIndexWebPushSubscriptionsAccessTokenId < ActiveRecord::Migration[5.2]
+  include Mastodon::MigrationHelpers
+
+  disable_ddl_transaction!
+
+  def up
+    update_index :web_push_subscriptions, 'index_web_push_subscriptions_on_access_token_id', :access_token_id, where: 'access_token_id IS NOT NULL'
+  end
+
+  def down
+    update_index :web_push_subscriptions, 'index_web_push_subscriptions_on_access_token_id', :access_token_id
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2022_03_09_213005) do
+ActiveRecord::Schema.define(version: 2022_03_10_060959) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -59,7 +59,7 @@ ActiveRecord::Schema.define(version: 2022_03_09_213005) do
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
     t.index ["account_id"], name: "index_account_migrations_on_account_id"
-    t.index ["target_account_id"], name: "index_account_migrations_on_target_account_id"
+    t.index ["target_account_id"], name: "index_account_migrations_on_target_account_id", where: "(target_account_id IS NOT NULL)"
   end
 
   create_table "account_moderation_notes", force: :cascade do |t|
@@ -188,9 +188,9 @@ ActiveRecord::Schema.define(version: 2022_03_09_213005) do
     t.datetime "requested_review_at"
     t.index "(((setweight(to_tsvector('simple'::regconfig, (display_name)::text), 'A'::\"char\") || setweight(to_tsvector('simple'::regconfig, (username)::text), 'B'::\"char\")) || setweight(to_tsvector('simple'::regconfig, (COALESCE(domain, ''::character varying))::text), 'C'::\"char\")))", name: "search_index", using: :gin
     t.index "lower((username)::text), COALESCE(lower((domain)::text), ''::text)", name: "index_accounts_on_username_and_domain_lower", unique: true
-    t.index ["moved_to_account_id"], name: "index_accounts_on_moved_to_account_id"
+    t.index ["moved_to_account_id"], name: "index_accounts_on_moved_to_account_id", where: "(moved_to_account_id IS NOT NULL)"
     t.index ["uri"], name: "index_accounts_on_uri"
-    t.index ["url"], name: "index_accounts_on_url"
+    t.index ["url"], name: "index_accounts_on_url", opclass: :text_pattern_ops, where: "(url IS NOT NULL)"
   end
 
   create_table "accounts_tags", id: false, force: :cascade do |t|
@@ -230,7 +230,7 @@ ActiveRecord::Schema.define(version: 2022_03_09_213005) do
     t.datetime "updated_at", null: false
     t.index ["account_id", "announcement_id", "name"], name: "index_announcement_reactions_on_account_id_and_announcement_id", unique: true
     t.index ["announcement_id"], name: "index_announcement_reactions_on_announcement_id"
-    t.index ["custom_emoji_id"], name: "index_announcement_reactions_on_custom_emoji_id"
+    t.index ["custom_emoji_id"], name: "index_announcement_reactions_on_custom_emoji_id", where: "(custom_emoji_id IS NOT NULL)"
   end
 
   create_table "announcements", force: :cascade do |t|
@@ -258,8 +258,8 @@ ActiveRecord::Schema.define(version: 2022_03_09_213005) do
     t.datetime "updated_at", precision: 6, null: false
     t.index ["account_id"], name: "index_appeals_on_account_id"
     t.index ["account_warning_id"], name: "index_appeals_on_account_warning_id", unique: true
-    t.index ["approved_by_account_id"], name: "index_appeals_on_approved_by_account_id"
-    t.index ["rejected_by_account_id"], name: "index_appeals_on_rejected_by_account_id"
+    t.index ["approved_by_account_id"], name: "index_appeals_on_approved_by_account_id", where: "(approved_by_account_id IS NOT NULL)"
+    t.index ["rejected_by_account_id"], name: "index_appeals_on_rejected_by_account_id", where: "(rejected_by_account_id IS NOT NULL)"
   end
 
   create_table "backups", force: :cascade do |t|
@@ -311,7 +311,7 @@ ActiveRecord::Schema.define(version: 2022_03_09_213005) do
     t.string "uri"
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
-    t.index ["uri"], name: "index_conversations_on_uri", unique: true
+    t.index ["uri"], name: "index_conversations_on_uri", unique: true, opclass: :text_pattern_ops, where: "(uri IS NOT NULL)"
   end
 
   create_table "custom_emoji_categories", force: :cascade do |t|
@@ -509,7 +509,7 @@ ActiveRecord::Schema.define(version: 2022_03_09_213005) do
     t.bigint "account_id", null: false
     t.bigint "follow_id"
     t.index ["account_id", "list_id"], name: "index_list_accounts_on_account_id_and_list_id", unique: true
-    t.index ["follow_id"], name: "index_list_accounts_on_follow_id"
+    t.index ["follow_id"], name: "index_list_accounts_on_follow_id", where: "(follow_id IS NOT NULL)"
     t.index ["list_id", "account_id"], name: "index_list_accounts_on_list_id_and_account_id"
   end
 
@@ -568,8 +568,8 @@ ActiveRecord::Schema.define(version: 2022_03_09_213005) do
     t.datetime "thumbnail_updated_at"
     t.string "thumbnail_remote_url"
     t.index ["account_id", "status_id"], name: "index_media_attachments_on_account_id_and_status_id", order: { status_id: :desc }
-    t.index ["scheduled_status_id"], name: "index_media_attachments_on_scheduled_status_id"
-    t.index ["shortcode"], name: "index_media_attachments_on_shortcode", unique: true
+    t.index ["scheduled_status_id"], name: "index_media_attachments_on_scheduled_status_id", where: "(scheduled_status_id IS NOT NULL)"
+    t.index ["shortcode"], name: "index_media_attachments_on_shortcode", unique: true, opclass: :text_pattern_ops, where: "(shortcode IS NOT NULL)"
     t.index ["status_id"], name: "index_media_attachments_on_status_id"
   end
 
@@ -631,8 +631,8 @@ ActiveRecord::Schema.define(version: 2022_03_09_213005) do
     t.bigint "resource_owner_id"
     t.datetime "last_used_at"
     t.inet "last_used_ip"
-    t.index ["refresh_token"], name: "index_oauth_access_tokens_on_refresh_token", unique: true
-    t.index ["resource_owner_id"], name: "index_oauth_access_tokens_on_resource_owner_id"
+    t.index ["refresh_token"], name: "index_oauth_access_tokens_on_refresh_token", unique: true, opclass: :text_pattern_ops, where: "(refresh_token IS NOT NULL)"
+    t.index ["resource_owner_id"], name: "index_oauth_access_tokens_on_resource_owner_id", where: "(resource_owner_id IS NOT NULL)"
     t.index ["token"], name: "index_oauth_access_tokens_on_token", unique: true
   end
 
@@ -899,10 +899,10 @@ ActiveRecord::Schema.define(version: 2022_03_09_213005) do
     t.index ["deleted_at"], name: "index_statuses_on_deleted_at", where: "(deleted_at IS NOT NULL)"
     t.index ["id", "account_id"], name: "index_statuses_local_20190824", order: { id: :desc }, where: "((local OR (uri IS NULL)) AND (deleted_at IS NULL) AND (visibility = 0) AND (reblog_of_id IS NULL) AND ((NOT reply) OR (in_reply_to_account_id = account_id)))"
     t.index ["id", "account_id"], name: "index_statuses_public_20200119", order: { id: :desc }, where: "((deleted_at IS NULL) AND (visibility = 0) AND (reblog_of_id IS NULL) AND ((NOT reply) OR (in_reply_to_account_id = account_id)))"
-    t.index ["in_reply_to_account_id"], name: "index_statuses_on_in_reply_to_account_id"
-    t.index ["in_reply_to_id"], name: "index_statuses_on_in_reply_to_id"
+    t.index ["in_reply_to_account_id"], name: "index_statuses_on_in_reply_to_account_id", where: "(in_reply_to_account_id IS NOT NULL)"
+    t.index ["in_reply_to_id"], name: "index_statuses_on_in_reply_to_id", where: "(in_reply_to_id IS NOT NULL)"
     t.index ["reblog_of_id", "account_id"], name: "index_statuses_on_reblog_of_id_and_account_id"
-    t.index ["uri"], name: "index_statuses_on_uri", unique: true
+    t.index ["uri"], name: "index_statuses_on_uri", unique: true, opclass: :text_pattern_ops, where: "(uri IS NOT NULL)"
   end
 
   create_table "statuses_tags", id: false, force: :cascade do |t|
@@ -996,9 +996,9 @@ ActiveRecord::Schema.define(version: 2022_03_09_213005) do
     t.boolean "skip_sign_in_token"
     t.index ["account_id"], name: "index_users_on_account_id"
     t.index ["confirmation_token"], name: "index_users_on_confirmation_token", unique: true
-    t.index ["created_by_application_id"], name: "index_users_on_created_by_application_id"
+    t.index ["created_by_application_id"], name: "index_users_on_created_by_application_id", where: "(created_by_application_id IS NOT NULL)"
     t.index ["email"], name: "index_users_on_email", unique: true
-    t.index ["reset_password_token"], name: "index_users_on_reset_password_token", unique: true
+    t.index ["reset_password_token"], name: "index_users_on_reset_password_token", unique: true, opclass: :text_pattern_ops, where: "(reset_password_token IS NOT NULL)"
   end
 
   create_table "web_push_subscriptions", force: :cascade do |t|
@@ -1010,7 +1010,7 @@ ActiveRecord::Schema.define(version: 2022_03_09_213005) do
     t.datetime "updated_at", null: false
     t.bigint "access_token_id"
     t.bigint "user_id"
-    t.index ["access_token_id"], name: "index_web_push_subscriptions_on_access_token_id"
+    t.index ["access_token_id"], name: "index_web_push_subscriptions_on_access_token_id", where: "(access_token_id IS NOT NULL)"
     t.index ["user_id"], name: "index_web_push_subscriptions_on_user_id"
   end