Unfloyded/Legamunt
0
0
Fork 0
Code Activity

Add trending links (#16917)

Browse source 
* Add trending links

* Add overriding specific links trendability

* Add link type to preview cards and only trend articles

Change trends review notifications from being sent every 5 minutes to being sent every 2 hours

Change threshold from 5 unique accounts to 15 unique accounts

* Fix tests
This commit is contained in:
Eugen Rochko 2021-11-25 13:07:38 +01:00 • committed by GitHub
parent 46e62fc4b3
commit 6e50134a42
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
97 changed files with 2071 additions and 722 deletions
Download patch file Download diff file Expand all files Collapse all files

2
app/lib/activitypub/activity.rb
View file

@ -129,8 +129,6 @@ class ActivityPub::Activity
end
def crawl_links(status)
return if status.spoiler_text?
# Spread out crawling randomly to avoid DDoSing the link
LinkCrawlWorker.perform_in(rand(1..59).seconds, status.id)
end

5
app/lib/activitypub/activity/announce.rb
View file

@ -22,9 +22,8 @@ class ActivityPub::Activity::Announce < ActivityPub::Activity
visibility: visibility_from_audience
)
original_status.tags.each do |tag|
tag.use!(@account)
end
Trends.tags.register(@status)
Trends.links.register(@status)
distribute(@status)
end

7
app/lib/activitypub/activity/create.rb
View file

@ -164,9 +164,14 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
def attach_tags(status)
@tags.each do |tag|
status.tags << tag
tag.use!(@account, status: status, at_time: status.created_at) if status.public_visibility?
tag.update(last_status_at: status.created_at) if tag.last_status_at.nil? || (tag.last_status_at < status.created_at && tag.last_status_at < 12.hours.ago)
end
# If we're processing an old status, this may register tags as being used now
# as opposed to when the status was really published, but this is probably
# not a big deal
Trends.tags.register(status)
@mentions.each do |mention|
mention.status = status
mention.save

9
app/lib/admin/metrics/dimension.rb
View file

@ -7,9 +7,14 @@ class Admin::Metrics::Dimension
servers: Admin::Metrics::Dimension::ServersDimension,
space_usage: Admin::Metrics::Dimension::SpaceUsageDimension,
software_versions: Admin::Metrics::Dimension::SoftwareVersionsDimension,
tag_servers: Admin::Metrics::Dimension::TagServersDimension,
tag_languages: Admin::Metrics::Dimension::TagLanguagesDimension,
}.freeze
def self.retrieve(dimension_keys, start_at, end_at, limit)
Array(dimension_keys).map { |key| DIMENSIONS[key.to_sym]&.new(start_at, end_at, limit) }.compact
def self.retrieve(dimension_keys, start_at, end_at, limit, params)
Array(dimension_keys).map do |key|
klass = DIMENSIONS[key.to_sym]
klass&.new(start_at, end_at, limit, klass.with_params? ? params.require(key.to_sym) : nil)
end.compact
end
end

13
app/lib/admin/metrics/dimension/base_dimension.rb
View file

@ -1,10 +1,15 @@
# frozen_string_literal: true
class Admin::Metrics::Dimension::BaseDimension
def initialize(start_at, end_at, limit)
def self.with_params?
false
end
def initialize(start_at, end_at, limit, params)
@start_at = start_at&.to_datetime
@end_at = end_at&.to_datetime
@limit = limit&.to_i
@params = params
end
def key
@ -26,6 +31,10 @@ class Admin::Metrics::Dimension::BaseDimension
protected
def time_period
(@start_at...@end_at)
(@start_at..@end_at)
end
def params
raise NotImplementedError
end
end

4
app/lib/admin/metrics/dimension/languages_dimension.rb
View file

@ -1,6 +1,8 @@
# frozen_string_literal: true
class Admin::Metrics::Dimension::LanguagesDimension < Admin::Metrics::Dimension::BaseDimension
include LanguagesHelper
def key
'languages'
end
@ -18,6 +20,6 @@ class Admin::Metrics::Dimension::LanguagesDimension < Admin::Metrics::Dimension:
rows = ActiveRecord::Base.connection.select_all(sql, nil, [[nil, @start_at], [nil, @end_at], [nil, @limit]])
rows.map { |row| { key: row['locale'], human_key: SettingsHelper::HUMAN_LOCALES[row['locale'].to_sym], value: row['value'].to_s } }
rows.map { |row| { key: row['locale'], human_key: human_locale(row['locale']), value: row['value'].to_s } }
end
end

36
app/lib/admin/metrics/dimension/tag_languages_dimension.rb Normal file
View file

@ -0,0 +1,36 @@
# frozen_string_literal: true
class Admin::Metrics::Dimension::TagLanguagesDimension < Admin::Metrics::Dimension::BaseDimension
include LanguagesHelper
def self.with_params?
true
end
def key
'tag_languages'
end
def data
sql = <<-SQL.squish
SELECT COALESCE(statuses.language, 'und') AS language, count(*) AS value
FROM statuses
INNER JOIN statuses_tags ON statuses_tags.status_id = statuses.id
WHERE statuses_tags.tag_id = $1
AND statuses.id BETWEEN $2 AND $3
GROUP BY COALESCE(statuses.language, 'und')
ORDER BY count(*) DESC
LIMIT $4
SQL
rows = ActiveRecord::Base.connection.select_all(sql, nil, [[nil, params[:id]], [nil, Mastodon::Snowflake.id_at(@start_at, with_random: false)], [nil, Mastodon::Snowflake.id_at(@end_at, with_random: false)], [nil, @limit]])
rows.map { |row| { key: row['language'], human_key: human_locale(row['language']), value: row['value'].to_s } }
end
private
def params
@params.permit(:id)
end
end

35
app/lib/admin/metrics/dimension/tag_servers_dimension.rb Normal file
View file

@ -0,0 +1,35 @@
# frozen_string_literal: true
class Admin::Metrics::Dimension::TagServersDimension < Admin::Metrics::Dimension::BaseDimension
def self.with_params?
true
end
def key
'tag_servers'
end
def data
sql = <<-SQL.squish
SELECT accounts.domain, count(*) AS value
FROM statuses
INNER JOIN accounts ON accounts.id = statuses.account_id
INNER JOIN statuses_tags ON statuses_tags.status_id = statuses.id
WHERE statuses_tags.tag_id = $1
AND statuses.id BETWEEN $2 AND $3
GROUP BY accounts.domain
ORDER BY count(*) DESC
LIMIT $4
SQL
rows = ActiveRecord::Base.connection.select_all(sql, nil, [[nil, params[:id]], [nil, Mastodon::Snowflake.id_at(@start_at, with_random: false)], [nil, Mastodon::Snowflake.id_at(@end_at, with_random: false)], [nil, @limit]])
rows.map { |row| { key: row['domain'] || Rails.configuration.x.local_domain, human_key: row['domain'] || Rails.configuration.x.local_domain, value: row['value'].to_s } }
end
private
def params
@params.permit(:id)
end
end

10
app/lib/admin/metrics/measure.rb
View file

@ -7,9 +7,15 @@ class Admin::Metrics::Measure
interactions: Admin::Metrics::Measure::InteractionsMeasure,
opened_reports: Admin::Metrics::Measure::OpenedReportsMeasure,
resolved_reports: Admin::Metrics::Measure::ResolvedReportsMeasure,
tag_accounts: Admin::Metrics::Measure::TagAccountsMeasure,
tag_uses: Admin::Metrics::Measure::TagUsesMeasure,
tag_servers: Admin::Metrics::Measure::TagServersMeasure,
}.freeze
def self.retrieve(measure_keys, start_at, end_at)
Array(measure_keys).map { |key| MEASURES[key.to_sym]&.new(start_at, end_at) }.compact
def self.retrieve(measure_keys, start_at, end_at, params)
Array(measure_keys).map do |key|
klass = MEASURES[key.to_sym]
klass&.new(start_at, end_at, klass.with_params? ? params.require(key.to_sym) : nil)
end.compact
end
end

4
app/lib/admin/metrics/measure/active_users_measure.rb
View file

@ -24,10 +24,10 @@ class Admin::Metrics::Measure::ActiveUsersMeasure < Admin::Metrics::Measure::Bas
end
def time_period
(@start_at.to_date...@end_at.to_date)
(@start_at.to_date..@end_at.to_date)
end
def previous_time_period
((@start_at.to_date - length_of_period)...(@end_at.to_date - length_of_period))
((@start_at.to_date - length_of_period)..(@end_at.to_date - length_of_period))
end
end

15
app/lib/admin/metrics/measure/base_measure.rb
View file

@ -1,9 +1,14 @@
# frozen_string_literal: true
class Admin::Metrics::Measure::BaseMeasure
def initialize(start_at, end_at)
def self.with_params?
false
end
def initialize(start_at, end_at, params)
@start_at = start_at&.to_datetime
@end_at = end_at&.to_datetime
@params = params
end
def key
@ -33,14 +38,18 @@ class Admin::Metrics::Measure::BaseMeasure
protected
def time_period
(@start_at...@end_at)
(@start_at..@end_at)
end
def previous_time_period
((@start_at - length_of_period)...(@end_at - length_of_period))
((@start_at - length_of_period)..(@end_at - length_of_period))
end
def length_of_period
@length_of_period ||= @end_at - @start_at
end
def params
raise NotImplementedError
end
end

4
app/lib/admin/metrics/measure/interactions_measure.rb
View file

@ -24,10 +24,10 @@ class Admin::Metrics::Measure::InteractionsMeasure < Admin::Metrics::Measure::Ba
end
def time_period
(@start_at.to_date...@end_at.to_date)
(@start_at.to_date..@end_at.to_date)
end
def previous_time_period
((@start_at.to_date - length_of_period)...(@end_at.to_date - length_of_period))
((@start_at.to_date - length_of_period)..(@end_at.to_date - length_of_period))
end
end

41
app/lib/admin/metrics/measure/tag_accounts_measure.rb Normal file
View file

@ -0,0 +1,41 @@
# frozen_string_literal: true
class Admin::Metrics::Measure::TagAccountsMeasure < Admin::Metrics::Measure::BaseMeasure
def self.with_params?
true
end
def key
'tag_accounts'
end
def total
tag.history.aggregate(time_period).accounts
end
def previous_total
tag.history.aggregate(previous_time_period).accounts
end
def data
time_period.map { |date| { date: date.to_time(:utc).iso8601, value: tag.history.get(date).accounts.to_s } }
end
protected
def tag
@tag ||= Tag.find(params[:id])
end
def time_period
(@start_at.to_date..@end_at.to_date)
end
def previous_time_period
((@start_at.to_date - length_of_period)..(@end_at.to_date - length_of_period))
end
def params
@params.permit(:id)
end
end

47
app/lib/admin/metrics/measure/tag_servers_measure.rb Normal file
View file

@ -0,0 +1,47 @@
# frozen_string_literal: true
class Admin::Metrics::Measure::TagServersMeasure < Admin::Metrics::Measure::BaseMeasure
def self.with_params?
true
end
def key
'tag_servers'
end
def total
tag.statuses.where('statuses.id BETWEEN ? AND ?', Mastodon::Snowflake.id_at(@start_at, with_random: false), Mastodon::Snowflake.id_at(@end_at, with_random: false)).joins(:account).count('distinct accounts.domain')
end
def previous_total
tag.statuses.where('statuses.id BETWEEN ? AND ?', Mastodon::Snowflake.id_at(@start_at - length_of_period, with_random: false), Mastodon::Snowflake.id_at(@end_at - length_of_period, with_random: false)).joins(:account).count('distinct accounts.domain')
end
def data
sql = <<-SQL.squish
SELECT axis.*, (
SELECT count(*) AS value
FROM statuses
WHERE statuses.id BETWEEN $1 AND $2
AND date_trunc('day', statuses.created_at)::date = axis.day
)
FROM (
SELECT generate_series(date_trunc('day', $3::timestamp)::date, date_trunc('day', $4::timestamp)::date, ('1 day')::interval) AS day
) as axis
SQL
rows = ActiveRecord::Base.connection.select_all(sql, nil, [[nil, Mastodon::Snowflake.id_at(@start_at, with_random: false)], [nil, Mastodon::Snowflake.id_at(@end_at, with_random: false)], [nil, @start_at], [nil, @end_at]])
rows.map { |row| { date: row['day'], value: row['value'].to_s } }
end
protected
def tag
@tag ||= Tag.find(params[:id])
end
def params
@params.permit(:id)
end
end

41
app/lib/admin/metrics/measure/tag_uses_measure.rb Normal file
View file

@ -0,0 +1,41 @@
# frozen_string_literal: true
class Admin::Metrics::Measure::TagUsesMeasure < Admin::Metrics::Measure::BaseMeasure
def self.with_params?
true
end
def key
'tag_uses'
end
def total
tag.history.aggregate(time_period).uses
end
def previous_total
tag.history.aggregate(previous_time_period).uses
end
def data
time_period.map { |date| { date: date.to_time(:utc).iso8601, value: tag.history.get(date).uses.to_s } }
end
protected
def tag
@tag ||= Tag.find(params[:id])
end
def time_period
(@start_at.to_date..@end_at.to_date)
end
def previous_time_period
((@start_at.to_date - length_of_period)..(@end_at.to_date - length_of_period))
end
def params
@params.permit(:id)
end
end

49
app/lib/link_details_extractor.rb
View file

@ -4,6 +4,11 @@ class LinkDetailsExtractor
include ActionView::Helpers::TagHelper
class StructuredData
SUPPORTED_TYPES = %w(
NewsArticle
WebPage
).freeze
def initialize(data)
@data = data
end
@ -16,6 +21,14 @@ class LinkDetailsExtractor
json['description']
end
def language
json['inLanguage']
end
def type
json['@type']
end
def image
obj = first_of_value(json['image'])
@ -44,6 +57,10 @@ class LinkDetailsExtractor
publisher['name']
end
def publisher_logo
publisher.dig('logo', 'url')
end
private
def author
@ -58,8 +75,12 @@ class LinkDetailsExtractor
arr.is_a?(Array) ? arr.first : arr
end
def root_array(root)
root.is_a?(Array) ? root : [root]
end
def json
@json ||= first_of_value(Oj.load(@data))
@json ||= root_array(Oj.load(@data)).find { |obj| SUPPORTED_TYPES.include?(obj['@type']) } || {}
end
end
@ -75,6 +96,7 @@ class LinkDetailsExtractor
description: description || '',
image_remote_url: image,
type: type,
link_type: link_type,
width: width || 0,
height: height || 0,
html: html || '',
@ -83,6 +105,7 @@ class LinkDetailsExtractor
author_name: author_name || '',
author_url: author_url || '',
embed_url: embed_url || '',
language: language,
}
end
@ -90,6 +113,14 @@ class LinkDetailsExtractor
player_url.present? ? :video : :link
end
def link_type
if structured_data&.type == 'NewsArticle' || opengraph_tag('og:type') == 'article'
:article
else
:unknown
end
end
def html
player_url.present? ? content_tag(:iframe, nil, src: player_url, width: width, height: height, allowtransparency: 'true', scrolling: 'no', frameborder: '0') : nil
end
@ -138,6 +169,14 @@ class LinkDetailsExtractor
valid_url_or_nil(opengraph_tag('twitter:player:stream'))
end
def language
valid_locale_or_nil(structured_data&.language || opengraph_tag('og:locale') || document.xpath('//html').map { |element| element['lang'] }.first)
end
def icon
valid_url_or_nil(structured_data&.publisher_icon || link_tag('apple-touch-icon') || link_tag('shortcut icon'))
end
private
def player_url
@ -162,6 +201,14 @@ class LinkDetailsExtractor
nil
end
def valid_locale_or_nil(str)
return nil if str.blank?
code, = str.split(/_-/) # Strip out the region from e.g. en_US or ja-JA
locale = ISO_639.find(code)
locale&.alpha2
end
def link_tag(name)
document.xpath("//link[@rel=\"#{name}\"]").map { |link| link['href'] }.first
end