From 1d588d58f123e97d4a864ee9f595fb8dcdf24a82 Mon Sep 17 00:00:00 2001 From: Claire Date: Thu, 25 May 2023 09:27:16 +0200 Subject: [PATCH] Improve various queries against account domains (#25126) --- app/models/account.rb | 2 +- app/models/instance.rb | 1 + ...524190515_add_index_accounts_on_domain_and_id.rb | 9 +++++++++ .../20230524192812_fix_account_domain_casing.rb | 13 +++++++++++++ ...4194155_add_index_instances_on_reverse_domain.rb | 9 +++++++++ db/schema.rb | 4 +++- lib/tasks/tests.rake | 7 ++++++- 7 files changed, 42 insertions(+), 3 deletions(-) create mode 100644 db/migrate/20230524190515_add_index_accounts_on_domain_and_id.rb create mode 100644 db/migrate/20230524192812_fix_account_domain_casing.rb create mode 100644 db/migrate/20230524194155_add_index_instances_on_reverse_domain.rb diff --git a/app/models/account.rb b/app/models/account.rb index f17d06be5..8530e664c 100644 --- a/app/models/account.rb +++ b/app/models/account.rb @@ -119,7 +119,7 @@ class Account < ApplicationRecord scope :by_recent_status, -> { order(Arel.sql('(case when account_stats.last_status_at is null then 1 else 0 end) asc, account_stats.last_status_at desc, accounts.id desc')) } scope :by_recent_sign_in, -> { order(Arel.sql('(case when users.current_sign_in_at is null then 1 else 0 end) asc, users.current_sign_in_at desc, accounts.id desc')) } scope :popular, -> { order('account_stats.followers_count desc') } - scope :by_domain_and_subdomains, ->(domain) { where(domain: domain).or(where(arel_table[:domain].matches("%.#{domain}"))) } + scope :by_domain_and_subdomains, ->(domain) { where(domain: Instance.by_domain_and_subdomain(domain).select(:domain)) } scope :not_excluded_by_account, ->(account) { where.not(id: account.excluded_from_timeline_account_ids) } scope :not_domain_blocked_by_account, ->(account) { where(arel_table[:domain].eq(nil).or(arel_table[:domain].not_in(account.excluded_from_timeline_domains))) } diff --git a/app/models/instance.rb b/app/models/instance.rb index 95231c52a..a80e91e94 100644 --- a/app/models/instance.rb +++ b/app/models/instance.rb @@ -22,6 +22,7 @@ class Instance < ApplicationRecord end scope :matches_domain, ->(value) { where(arel_table[:domain].matches("%#{value}%")) } + scope :by_domain_and_subdomain, ->(domain) { where("reverse('.' || domain) LIKE reverse(?)", "%.#{domain}") } def self.refresh Scenic.database.refresh_materialized_view(table_name, concurrently: true, cascade: false) diff --git a/db/migrate/20230524190515_add_index_accounts_on_domain_and_id.rb b/db/migrate/20230524190515_add_index_accounts_on_domain_and_id.rb new file mode 100644 index 000000000..4ee52b0ca --- /dev/null +++ b/db/migrate/20230524190515_add_index_accounts_on_domain_and_id.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +class AddIndexAccountsOnDomainAndId < ActiveRecord::Migration[6.1] + disable_ddl_transaction! + + def change + add_index :accounts, [:domain, :id], name: :index_accounts_on_domain_and_id, algorithm: :concurrently + end +end diff --git a/db/migrate/20230524192812_fix_account_domain_casing.rb b/db/migrate/20230524192812_fix_account_domain_casing.rb new file mode 100644 index 000000000..e1edf2374 --- /dev/null +++ b/db/migrate/20230524192812_fix_account_domain_casing.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +class FixAccountDomainCasing < ActiveRecord::Migration[6.1] + disable_ddl_transaction! + + def up + safety_assured do + execute 'UPDATE accounts SET domain = lower(domain) WHERE domain IS NOT NULL AND domain != lower(domain)' + end + end + + def down; end +end diff --git a/db/migrate/20230524194155_add_index_instances_on_reverse_domain.rb b/db/migrate/20230524194155_add_index_instances_on_reverse_domain.rb new file mode 100644 index 000000000..c3e774936 --- /dev/null +++ b/db/migrate/20230524194155_add_index_instances_on_reverse_domain.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +class AddIndexInstancesOnReverseDomain < ActiveRecord::Migration[6.1] + disable_ddl_transaction! + + def change + add_index :instances, "reverse('.' || domain), domain", name: :index_instances_on_reverse_domain, algorithm: :concurrently + end +end diff --git a/db/schema.rb b/db/schema.rb index 8a32e45ce..f452fef7b 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2023_03_30_155710) do +ActiveRecord::Schema.define(version: 2023_05_24_194155) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -188,6 +188,7 @@ ActiveRecord::Schema.define(version: 2023_03_30_155710) do t.datetime "requested_review_at" t.index "(((setweight(to_tsvector('simple'::regconfig, (display_name)::text), 'A'::\"char\") || setweight(to_tsvector('simple'::regconfig, (username)::text), 'B'::\"char\")) || setweight(to_tsvector('simple'::regconfig, (COALESCE(domain, ''::character varying))::text), 'C'::\"char\")))", name: "search_index", using: :gin t.index "lower((username)::text), COALESCE(lower((domain)::text), ''::text)", name: "index_accounts_on_username_and_domain_lower", unique: true + t.index ["domain", "id"], name: "index_accounts_on_domain_and_id" t.index ["moved_to_account_id"], name: "index_accounts_on_moved_to_account_id", where: "(moved_to_account_id IS NOT NULL)" t.index ["uri"], name: "index_accounts_on_uri" t.index ["url"], name: "index_accounts_on_url", opclass: :text_pattern_ops, where: "(url IS NOT NULL)" @@ -1283,6 +1284,7 @@ ActiveRecord::Schema.define(version: 2023_03_30_155710) do FROM (domain_allows LEFT JOIN domain_counts ON (((domain_counts.domain)::text = (domain_allows.domain)::text))); SQL + add_index "instances", "reverse(('.'::text || (domain)::text)), domain", name: "index_instances_on_reverse_domain" add_index "instances", ["domain"], name: "index_instances_on_domain", unique: true create_view "user_ips", sql_definition: <<-SQL diff --git a/lib/tasks/tests.rake b/lib/tasks/tests.rake index 60399c9de..3c88ce450 100644 --- a/lib/tasks/tests.rake +++ b/lib/tasks/tests.rake @@ -58,6 +58,11 @@ namespace :tests do puts 'User settings not kept as expected' exit(1) end + + unless Account.find_remote('bob', 'ActivityPub.com').domain == 'activitypub.com' + puts 'Account domains not properly normalized' + exit(1) + end end desc 'Populate the database with test data for 2.4.3' @@ -160,7 +165,7 @@ namespace :tests do INSERT INTO "accounts" (id, username, domain, private_key, public_key, created_at, updated_at, protocol, inbox_url, outbox_url, followers_url) VALUES - (6, 'bob', 'activitypub.com', NULL, #{remote_public_key_ap}, now(), now(), + (6, 'bob', 'ActivityPub.com', NULL, #{remote_public_key_ap}, now(), now(), 1, 'https://activitypub.com/users/bob/inbox', 'https://activitypub.com/users/bob/outbox', 'https://activitypub.com/users/bob/followers'); INSERT INTO "accounts"