Unfloyded/Legamunt
0
0
Fork 0
Code Activity

Add trending statuses (#17431)

Browse source 
* Add trending statuses

* Fix dangling items with stale scores in localized sets

* Various fixes and improvements

- Change approve_all/reject_all to approve_accounts/reject_accounts
- Change Trends::Query methods to not mutate the original query
- Change Trends::Query#skip to offset
- Change follow recommendations to be refreshed in a transaction

* Add tests for trending statuses filtering behaviour

* Fix not applying filtering scope in controller
This commit is contained in:
Eugen Rochko 2022-02-25 00:34:14 +01:00 • committed by GitHub
parent a29a982eaa
commit 27965ce5ed
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
71 changed files with 1074 additions and 307 deletions
Download patch file Download diff file Expand all files Collapse all files

30
app/models/account.rb
View file

@ -40,13 +40,15 @@
# also_known_as :string is an Array
# silenced_at :datetime
# suspended_at :datetime
# trust_level :integer
# hide_collections :boolean
# avatar_storage_schema_version :integer
# header_storage_schema_version :integer
# devices_url :string
# suspension_origin :integer
# sensitized_at :datetime
# trendable :boolean
# reviewed_at :datetime
# requested_review_at :datetime
#
class Account < ApplicationRecord
@ -56,6 +58,7 @@ class Account < ApplicationRecord
remote_url
salmon_url
hub_url
trust_level
)
USERNAME_RE = /[a-z0-9_]+([a-z0-9_\.-]+[a-z0-9_]+)?/i
@ -74,11 +77,6 @@ class Account < ApplicationRecord
include DomainMaterializable
include AccountMerging
TRUST_LEVELS = {
untrusted: 0,
trusted: 1,
}.freeze
enum protocol: [:ostatus, :activitypub]
enum suspension_origin: [:local, :remote], _prefix: true
@ -202,10 +200,6 @@ class Account < ApplicationRecord
last_webfingered_at.nil? || last_webfingered_at <= 1.day.ago
end
def trust_level
self[:trust_level] || 0
end
def refresh!
ResolveAccountService.new.call(acct) unless local?
end
@ -388,6 +382,22 @@ class Account < ApplicationRecord
@synchronization_uri_prefix ||= "#{uri[URL_PREFIX_RE]}/"
end
def requires_review?
reviewed_at.nil?
end
def reviewed?
reviewed_at.present?
end
def requested_review?
requested_review_at.present?
end
def requires_review_notification?
requires_review? && !requested_review?
end
class Field < ActiveModelSerializers::Model
attributes :name, :value, :verified_at, :account

12
app/models/status.rb
View file

@ -268,6 +268,18 @@ class Status < ApplicationRecord
update_status_stat!(key => [public_send(key) - 1, 0].max)
end
def trendable?
if attributes['trendable'].nil?
account.trendable?
else
attributes['trendable']
end
end
def requires_review_notification?
attributes['trendable'].nil? && account.requires_review_notification?
end
after_create_commit :increment_counter_caches
after_destroy_commit :decrement_counter_caches

26
app/models/trends.rb
View file

@ -13,15 +13,37 @@ module Trends
@tags ||= Trends::Tags.new
end
def self.statuses
@statuses ||= Trends::Statuses.new
end
def self.register!(status)
[links, tags, statuses].each { |trend_type| trend_type.register(status) }
end
def self.refresh!
[links, tags].each(&:refresh)
[links, tags, statuses].each(&:refresh)
end
def self.request_review!
[links, tags].each(&:request_review) if enabled?
return unless enabled?
links_requiring_review = links.request_review
tags_requiring_review = tags.request_review
statuses_requiring_review = statuses.request_review
return if links_requiring_review.empty? && tags_requiring_review.empty? && statuses_requiring_review.empty?
User.staff.includes(:account).find_each do |user|
AdminMailer.new_trends(user.account, links_requiring_review, tags_requiring_review, statuses_requiring_review).deliver_later! if user.allows_trends_review_emails?
end
end
def self.enabled?
Setting.trends
end
def self.available_locales
@available_locales ||= I18n.available_locales.map { |locale| locale.to_s.split(/[_-]/).first }.uniq
end
end

20
app/models/trends/base.rb
View file

@ -2,6 +2,7 @@
class Trends::Base
include Redisable
include LanguagesHelper
class_attribute :default_options
@ -32,8 +33,8 @@ class Trends::Base
raise NotImplementedError
end
def get(*)
raise NotImplementedError
def query
Trends::Query.new(key_prefix, klass)
end
def score(id)
@ -72,6 +73,21 @@ class Trends::Base
redis.zrevrange("#{key_prefix}:allowed", 0, rank, with_scores: true).last&.last || 0
end
# @param [Integer] id
# @param [Float] score
# @param [Hash<String, Boolean>] subsets
def add_to_and_remove_from_subsets(id, score, subsets = {})
subsets.each_key do |subset|
key = [key_prefix, subset].compact.join(':')
if score.positive? && subsets[subset]
redis.zadd(key, score, id)
else
redis.zrem(key, id)
end
end
end
private
def used_key(at_time)

52
app/models/trends/links.rb
View file

@ -4,8 +4,8 @@ class Trends::Links < Trends::Base
PREFIX = 'trending_links'
self.default_options = {
threshold: 15,
review_threshold: 10,
threshold: 5,
review_threshold: 3,
max_score_cooldown: 2.days.freeze,
max_score_halflife: 8.hours.freeze,
}
@ -27,12 +27,6 @@ class Trends::Links < Trends::Base
record_used_id(preview_card.id, at_time)
end
def get(allowed, limit)
preview_card_ids = currently_trending_ids(allowed, limit)
preview_cards = PreviewCard.where(id: preview_card_ids).index_by(&:id)
preview_card_ids.map { |id| preview_cards[id] }.compact
end
def refresh(at_time = Time.now.utc)
preview_cards = PreviewCard.where(id: (recently_used_ids(at_time) + currently_trending_ids(false, -1)).uniq)
calculate_scores(preview_cards, at_time)
@ -42,7 +36,7 @@ class Trends::Links < Trends::Base
def request_review
preview_cards = PreviewCard.where(id: currently_trending_ids(false, -1))
preview_cards_requiring_review = preview_cards.filter_map do |preview_card|
preview_cards.filter_map do |preview_card|
next unless would_be_trending?(preview_card.id) && !preview_card.trendable? && preview_card.requires_review_notification?
if preview_card.provider.nil?
@ -53,12 +47,6 @@ class Trends::Links < Trends::Base
preview_card
end
return if preview_cards_requiring_review.empty?
User.staff.includes(:account).find_each do |user|
AdminMailer.new_trending_links(user.account, preview_cards_requiring_review).deliver_later! if user.allows_trending_tag_emails?
end
end
protected
@ -67,6 +55,10 @@ class Trends::Links < Trends::Base
PREFIX
end
def klass
PreviewCard
end
private
def calculate_scores(preview_cards, at_time)
@ -96,17 +88,27 @@ class Trends::Links < Trends::Base
decaying_score = max_score * (0.5**((at_time.to_f - max_time.to_f) / options[:max_score_halflife].to_f))
if decaying_score.zero?
redis.zrem("#{PREFIX}:all", preview_card.id)
redis.zrem("#{PREFIX}:allowed", preview_card.id)
else
redis.zadd("#{PREFIX}:all", decaying_score, preview_card.id)
add_to_and_remove_from_subsets(preview_card.id, decaying_score, {
all: true,
allowed: preview_card.trendable?,
})
if preview_card.trendable?
redis.zadd("#{PREFIX}:allowed", decaying_score, preview_card.id)
else
redis.zrem("#{PREFIX}:allowed", preview_card.id)
end
next unless valid_locale?(preview_card.language)
add_to_and_remove_from_subsets(preview_card.id, decaying_score, {
"all:#{preview_card.language}" => true,
"allowed:#{preview_card.language}" => preview_card.trendable?,
})
end
# Clean up localized sets by calculating the intersection with the main
# set. We do this instead of just deleting the localized sets to avoid
# having moments where the API returns empty results
redis.pipelined do
Trends.available_locales.each do |locale|
redis.zinterstore("#{key_prefix}:all:#{locale}", ["#{key_prefix}:all:#{locale}", "#{key_prefix}:all"], aggregate: 'max')
redis.zinterstore("#{key_prefix}:allowed:#{locale}", ["#{key_prefix}:allowed:#{locale}", "#{key_prefix}:all"], aggregate: 'max')
end
end
end

22
app/models/form/preview_card_batch.rb → app/models/trends/preview_card_batch.rb
View file

@ -1,6 +1,6 @@
# frozen_string_literal: true
class Form::PreviewCardBatch
class Trends::PreviewCardBatch
include ActiveModel::Model
include Authorization
@ -10,12 +10,12 @@ class Form::PreviewCardBatch
case action
when 'approve'
approve!
when 'approve_all'
approve_all!
when 'approve_providers'
approve_providers!
when 'reject'
reject!
when 'reject_all'
reject_all!
when 'reject_providers'
reject_providers!
end
end
@ -30,13 +30,13 @@ class Form::PreviewCardBatch
end
def approve!
preview_cards.each { |preview_card| authorize(preview_card, :update?) }
preview_cards.each { |preview_card| authorize(preview_card, :review?) }
preview_cards.update_all(trendable: true)
end
def approve_all!
def approve_providers!
preview_card_providers.each do |provider|
authorize(provider, :update?)
authorize(provider, :review?)
provider.update(trendable: true, reviewed_at: action_time)
end
@ -45,13 +45,13 @@ class Form::PreviewCardBatch
end
def reject!
preview_cards.each { |preview_card| authorize(preview_card, :update?) }
preview_cards.each { |preview_card| authorize(preview_card, :review?) }
preview_cards.update_all(trendable: false)
end
def reject_all!
def reject_providers!
preview_card_providers.each do |provider|
authorize(provider, :update?)
authorize(provider, :review?)
provider.update(trendable: false, reviewed_at: action_time)
end

23
app/models/preview_card_filter.rb → app/models/trends/preview_card_filter.rb
View file

@ -1,8 +1,9 @@
# frozen_string_literal: true
class PreviewCardFilter
class Trends::PreviewCardFilter
KEYS = %i(
trending
locale
).freeze
attr_reader :params
@ -15,7 +16,7 @@ class PreviewCardFilter
scope = PreviewCard.unscoped
params.each do |key, value|
next if key.to_s == 'page'
next if %w(page locale).include?(key.to_s)
scope.merge!(scope_for(key, value.to_s.strip)) if value.present?
end
@ -35,19 +36,11 @@ class PreviewCardFilter
end
def trending_scope(value)
ids = begin
case value.to_s
when 'allowed'
Trends.links.currently_trending_ids(true, -1)
else
Trends.links.currently_trending_ids(false, -1)
end
end
scope = Trends.links.query
if ids.empty?
PreviewCard.none
else
PreviewCard.joins("join unnest(array[#{ids.map(&:to_i).join(',')}]::integer[]) with ordinality as x (id, ordering) on preview_cards.id = x.id").order('x.ordering')
end
scope = scope.in_locale(@params[:locale].to_s) if @params[:locale].present?
scope = scope.allowed if value == 'allowed'
scope.to_arel
end
end

6
app/models/form/preview_card_provider_batch.rb → app/models/trends/preview_card_provider_batch.rb
View file

@ -1,6 +1,6 @@
# frozen_string_literal: true
class Form::PreviewCardProviderBatch
class Trends::PreviewCardProviderBatch
include ActiveModel::Model
include Authorization
@ -22,12 +22,12 @@ class Form::PreviewCardProviderBatch
end
def approve!
preview_card_providers.each { |provider| authorize(provider, :update?) }
preview_card_providers.each { |provider| authorize(provider, :review?) }
preview_card_providers.update_all(trendable: true, reviewed_at: Time.now.utc)
end
def reject!
preview_card_providers.each { |provider| authorize(provider, :update?) }
preview_card_providers.each { |provider| authorize(provider, :review?) }
preview_card_providers.update_all(trendable: false, reviewed_at: Time.now.utc)
end
end

2
app/models/preview_card_provider_filter.rb → app/models/trends/preview_card_provider_filter.rb
View file

@ -1,6 +1,6 @@
# frozen_string_literal: true
class PreviewCardProviderFilter
class Trends::PreviewCardProviderFilter
KEYS = %i(
status
).freeze

106
app/models/trends/query.rb Normal file
View file

@ -0,0 +1,106 @@
# frozen_string_literal: true
class Trends::Query
include Redisable
include Enumerable
attr_reader :prefix, :klass, :loaded
alias loaded? loaded
def initialize(prefix, klass)
@prefix = prefix
@klass = klass
@records = []
@loaded = false
@allowed = false
@limit = -1
@offset = 0
end
def allowed!
@allowed = true
self
end
def allowed
clone.allowed!
end
def in_locale!(value)
@locale = value
self
end
def in_locale(value)
clone.in_locale!(value)
end
def offset!(value)
@offset = value
self
end
def offset(value)
clone.offset!(value)
end
def limit!(value)
@limit = value
self
end
def limit(value)
clone.limit!(value)
end
def records
load
@records
end
delegate :each, :empty?, :first, :last, to: :records
def to_ary
records.dup
end
alias to_a to_ary
def to_arel
tmp_ids = ids
if tmp_ids.empty?
klass.none
else
klass.joins("join unnest(array[#{tmp_ids.join(',')}]) with ordinality as x (id, ordering) on #{klass.table_name}.id = x.id").reorder('x.ordering')
end
end
private
def key
[@prefix, @allowed ? 'allowed' : 'all', @locale].compact.join(':')
end
def load
unless loaded?
@records = perform_queries
@loaded = true
end
self
end
def ids
redis.zrevrange(key, @offset, @limit.positive? ? @limit - 1 : @limit).map(&:to_i)
end
def perform_queries
apply_scopes(to_arel).to_a
end
def apply_scopes(scope)
scope
end
end

65
app/models/trends/status_batch.rb Normal file
View file

@ -0,0 +1,65 @@
# frozen_string_literal: true
class Trends::StatusBatch
include ActiveModel::Model
include Authorization
attr_accessor :status_ids, :action, :current_account
def save
case action
when 'approve'
approve!
when 'approve_accounts'
approve_accounts!
when 'reject'
reject!
when 'reject_accounts'
reject_accounts!
end
end
private
def statuses
@statuses ||= Status.where(id: status_ids)
end
def status_accounts
@status_accounts ||= Account.where(id: statuses.map(&:account_id).uniq)
end
def approve!
statuses.each { |status| authorize(status, :review?) }
statuses.update_all(trendable: true)
end
def approve_accounts!
status_accounts.each do |account|
authorize(account, :review?)
account.update(trendable: true, reviewed_at: action_time)
end
# Reset any individual overrides
statuses.update_all(trendable: nil)
end
def reject!
statuses.each { |status| authorize(status, :review?) }
statuses.update_all(trendable: false)
end
def reject_accounts!
status_accounts.each do |account|
authorize(account, :review?)
account.update(trendable: false, reviewed_at: action_time)
end
# Reset any individual overrides
statuses.update_all(trendable: nil)
end
def action_time
@action_time ||= Time.now.utc
end
end

46
app/models/trends/status_filter.rb Normal file
View file

@ -0,0 +1,46 @@
# frozen_string_literal: true
class Trends::StatusFilter
KEYS = %i(
trending
locale
).freeze
attr_reader :params
def initialize(params)
@params = params
end
def results
scope = Status.unscoped.kept
params.each do |key, value|
next if %w(page locale).include?(key.to_s)
scope.merge!(scope_for(key, value.to_s.strip)) if value.present?
end
scope
end
private
def scope_for(key, value)
case key.to_s
when 'trending'
trending_scope(value)
else
raise "Unknown filter: #{key}"
end
end
def trending_scope(value)
scope = Trends.statuses.query
scope = scope.in_locale(@params[:locale].to_s) if @params[:locale].present?
scope = scope.allowed if value == 'allowed'
scope.to_arel
end
end

142
app/models/trends/statuses.rb Normal file
View file

@ -0,0 +1,142 @@
# frozen_string_literal: true
class Trends::Statuses < Trends::Base
PREFIX = 'trending_statuses'
self.default_options = {
threshold: 5,
review_threshold: 3,
score_halflife: 2.hours.freeze,
}
class Query < Trends::Query
def filtered_for!(account)
@account = account
self
end
def filtered_for(account)
clone.filtered_for!(account)
end
private
def apply_scopes(scope)
scope.includes(:account)
end
def perform_queries
return super if @account.nil?
statuses = super
account_ids = statuses.map(&:account_id)
account_domains = statuses.map(&:account_domain)
preloaded_relations = {
blocking: Account.blocking_map(account_ids, @account.id),
blocked_by: Account.blocked_by_map(account_ids, @account.id),
muting: Account.muting_map(account_ids, @account.id),
following: Account.following_map(account_ids, @account.id),
domain_blocking_by_domain: Account.domain_blocking_map_by_domain(account_domains, @account.id),
}
statuses.reject { |status| StatusFilter.new(status, @account, preloaded_relations).filtered? }
end
end
def register(status, at_time = Time.now.utc)
add(status.proper, status.account_id, at_time) if eligible?(status)
end
def add(status, _account_id, at_time = Time.now.utc)
# We rely on the total reblogs and favourites count, so we
# don't record which account did the what and when here
record_used_id(status.id, at_time)
end
def query
Query.new(key_prefix, klass)
end
def refresh(at_time = Time.now.utc)
statuses = Status.where(id: (recently_used_ids(at_time) + currently_trending_ids(false, -1)).uniq).includes(:account, :media_attachments)
calculate_scores(statuses, at_time)
trim_older_items
end
def request_review
statuses = Status.where(id: currently_trending_ids(false, -1)).includes(:account)
statuses.filter_map do |status|
next unless would_be_trending?(status.id) && !status.trendable? && status.requires_review_notification?
status.account.touch(:requested_review_at)
status
end
end
protected
def key_prefix
PREFIX
end
def klass
Status
end
private
def eligible?(status)
original_status = status.proper
original_status.public_visibility? &&
original_status.account.discoverable? && !original_status.account.silenced? &&
original_status.spoiler_text.blank? && !original_status.sensitive? && !original_status.reply?
end
def calculate_scores(statuses, at_time)
redis.pipelined do
statuses.each do |status|
expected = 1.0
observed = (status.reblogs_count + status.favourites_count).to_f
score = begin
if expected > observed || observed < options[:threshold]
0
else
((observed - expected)**2) / expected
end
end
decaying_score = score * (0.5**((at_time.to_f - status.created_at.to_f) / options[:score_halflife].to_f))
add_to_and_remove_from_subsets(status.id, decaying_score, {
all: true,
allowed: status.trendable? && status.account.discoverable?,
})
next unless valid_locale?(status.language)
add_to_and_remove_from_subsets(status.id, decaying_score, {
"all:#{status.language}" => true,
"allowed:#{status.language}" => status.trendable? && status.account.discoverable?,
})
end
# Clean up localized sets by calculating the intersection with the main
# set. We do this instead of just deleting the localized sets to avoid
# having moments where the API returns empty results
Trends.available_locales.each do |locale|
redis.zinterstore("#{key_prefix}:all:#{locale}", ["#{key_prefix}:all:#{locale}", "#{key_prefix}:all"], aggregate: 'max')
redis.zinterstore("#{key_prefix}:allowed:#{locale}", ["#{key_prefix}:allowed:#{locale}", "#{key_prefix}:all"], aggregate: 'max')
end
end
end
def would_be_trending?(id)
score(id) > score_at_rank(options[:review_threshold] - 1)
end
end

6
app/models/form/tag_batch.rb → app/models/trends/tag_batch.rb
View file

@ -1,6 +1,6 @@
# frozen_string_literal: true
class Form::TagBatch
class Trends::TagBatch
include ActiveModel::Model
include Authorization
@ -22,12 +22,12 @@ class Form::TagBatch
end
def approve!
tags.each { |tag| authorize(tag, :update?) }
tags.each { |tag| authorize(tag, :review?) }
tags.update_all(trendable: true, reviewed_at: action_time)
end
def reject!
tags.each { |tag| authorize(tag, :update?) }
tags.each { |tag| authorize(tag, :review?) }
tags.update_all(trendable: false, reviewed_at: action_time)
end

10
app/models/tag_filter.rb → app/models/trends/tag_filter.rb
View file

@ -1,6 +1,6 @@
# frozen_string_literal: true
class TagFilter
class Trends::TagFilter
KEYS = %i(
trending
status
@ -42,13 +42,7 @@ class TagFilter
end
def trending_scope
ids = Trends.tags.currently_trending_ids(false, -1)
if ids.empty?
Tag.none
else
Tag.joins("join unnest(array[#{ids.map(&:to_i).join(',')}]::integer[]) with ordinality as x (id, ordering) on tags.id = x.id").order('x.ordering')
end
Trends.tags.query.to_arel
end
def status_scope(value)

36
app/models/trends/tags.rb
View file

@ -5,7 +5,7 @@ class Trends::Tags < Trends::Base
self.default_options = {
threshold: 5,
review_threshold: 10,
review_threshold: 3,
max_score_cooldown: 2.days.freeze,
max_score_halflife: 4.hours.freeze,
}
@ -29,27 +29,15 @@ class Trends::Tags < Trends::Base
trim_older_items
end
def get(allowed, limit)
tag_ids = currently_trending_ids(allowed, limit)
tags = Tag.where(id: tag_ids).index_by(&:id)
tag_ids.map { |id| tags[id] }.compact
end
def request_review
tags = Tag.where(id: currently_trending_ids(false, -1))
tags_requiring_review = tags.filter_map do |tag|
tags.filter_map do |tag|
next unless would_be_trending?(tag.id) && !tag.trendable? && tag.requires_review_notification?
tag.touch(:requested_review_at)
tag
end
return if tags_requiring_review.empty?
User.staff.includes(:account).find_each do |user|
AdminMailer.new_trending_tags(user.account, tags_requiring_review).deliver_later! if user.allows_trending_tag_emails?
end
end
protected
@ -58,6 +46,10 @@ class Trends::Tags < Trends::Base
PREFIX
end
def klass
Tag
end
private
def calculate_scores(tags, at_time)
@ -87,18 +79,10 @@ class Trends::Tags < Trends::Base
decaying_score = max_score * (0.5**((at_time.to_f - max_time.to_f) / options[:max_score_halflife].to_f))
if decaying_score.zero?
redis.zrem("#{PREFIX}:all", tag.id)
redis.zrem("#{PREFIX}:allowed", tag.id)
else
redis.zadd("#{PREFIX}:all", decaying_score, tag.id)
if tag.trendable?
redis.zadd("#{PREFIX}:allowed", decaying_score, tag.id)
else
redis.zrem("#{PREFIX}:allowed", tag.id)
end
end
add_to_and_remove_from_subsets(tag.id, decaying_score, {
all: true,
allowed: tag.trendable?,
})
end
end

2
app/models/user.rb
View file

@ -269,7 +269,7 @@ class User < ApplicationRecord
settings.notification_emails['appeal']
end
def allows_trending_tag_emails?
def allows_trends_review_emails?
settings.notification_emails['trending_tag']
end