Unfloyded/Legamunt
0
0
Fork 0
Code Activity

Merge remote-tracking branch 'upstream/main'

Browse source
This commit is contained in:
ASTRO:? 2025-03-14 20:25:34 +09:00
commit d564483d30
No known key found for this signature in database
GPG key ID: 2938B9B314D8EF8B
1796 changed files with 48111 additions and 29322 deletions
Download patch file Download diff file Expand all files Collapse all files

7
lib/mastodon/cli/accounts.rb
View file

@ -1,6 +1,5 @@
# frozen_string_literal: true
require 'set'
require_relative 'base'
module Mastodon::CLI
@ -165,7 +164,7 @@ module Mastodon::CLI
user.disabled = false if options[:enable]
user.disabled = true if options[:disable]
user.approved = true if options[:approve]
user.otp_required_for_login = false if options[:disable_2fa]
user.disable_two_factor! if options[:disable_2fa]
if user.save
user.confirm if options[:confirm]
@ -322,7 +321,9 @@ module Mastodon::CLI
unless skip_domains.empty?
say('The following domains were not available during the check:', :yellow)
skip_domains.each { |domain| say(" #{domain}") }
shell.indent(2) do
skip_domains.each { |domain| say(domain) }
end
end
end

4
lib/mastodon/cli/cache.rb
View file

@ -52,7 +52,7 @@ module Mastodon::CLI
account.account_stat.tap do |account_stat|
account_stat.following_count = account.active_relationships.count
account_stat.followers_count = account.passive_relationships.count
account_stat.statuses_count = account.statuses.where.not(visibility: :direct).count
account_stat.statuses_count = account.statuses.not_direct_visibility.count
account_stat.save if account_stat.changed?
end
@ -60,7 +60,7 @@ module Mastodon::CLI
def recount_status_stats(status)
status.status_stat.tap do |status_stat|
status_stat.replies_count = status.replies.where.not(visibility: :direct).count
status_stat.replies_count = status.replies.not_direct_visibility.count
status_stat.reblogs_count = status.reblogs.count
status_stat.favourites_count = status.favourites.count

17
lib/mastodon/cli/email_domain_blocks.rb
View file

@ -7,11 +7,13 @@ module Mastodon::CLI
class EmailDomainBlocks < Base
desc 'list', 'List blocked e-mail domains'
def list
EmailDomainBlock.where(parent_id: nil).find_each do |entry|
say(entry.domain.to_s, :white)
EmailDomainBlock.parents.find_each do |parent|
say(parent.domain.to_s, :white)
EmailDomainBlock.where(parent_id: entry.id).find_each do |child|
say(" #{child.domain}", :cyan)
shell.indent do
EmailDomainBlock.where(parent_id: parent.id).find_each do |child|
say(child.domain, :cyan)
end
end
end
end
@ -43,12 +45,7 @@ module Mastodon::CLI
end
other_domains = []
if options[:with_dns_records]
Resolv::DNS.open do |dns|
dns.timeouts = 5
other_domains = dns.getresources(@email_domain_block.domain, Resolv::DNS::Resource::IN::MX).to_a
end
end
other_domains = DomainResource.new(domain).mx if options[:with_dns_records]
email_domain_block = EmailDomainBlock.new(domain: domain, other_domains: other_domains)
email_domain_block.save!

4
lib/mastodon/cli/emoji.rb
View file

@ -62,7 +62,9 @@ module Mastodon::CLI
failed += 1
say('Failure/Error: ', :red)
say(entry.full_name)
say(" #{custom_emoji.errors[:image].join(', ')}", :red)
shell.indent(2) do
say(custom_emoji.errors[:image].join(', '), :red)
end
end
end
end

2
lib/mastodon/cli/federation.rb
View file

@ -76,7 +76,7 @@ module Mastodon::CLI
def self_destruct_value
Rails
.application
.message_verifier('self-destruct')
.message_verifier(SelfDestructHelper::VERIFY_PURPOSE)
.generate(Rails.configuration.x.local_domain)
end
end

41
lib/mastodon/cli/feeds.rb
View file

@ -5,22 +5,27 @@ require_relative 'base'
module Mastodon::CLI
class Feeds < Base
include Redisable
include DatabaseHelper
option :all, type: :boolean, default: false
option :concurrency, type: :numeric, default: 5, aliases: [:c]
option :verbose, type: :boolean, aliases: [:v]
option :dry_run, type: :boolean, default: false
option :skip_filled_timelines
desc 'build [USERNAME]', 'Build home and list feeds for one or all users'
long_desc <<-LONG_DESC
Build home and list feeds that are stored in Redis from the database.
With the --skip-filled-timelines, timelines which contain more than half
the maximum number of posts will be skipped.
With the --all option, all active users will be processed.
Otherwise, a single user specified by USERNAME.
LONG_DESC
def build(username = nil)
if options[:all] || username.nil?
processed, = parallelize_with_progress(active_user_accounts) do |account|
PrecomputeFeedService.new.call(account) unless dry_run?
PrecomputeFeedService.new.call(account, skip_filled_timelines: options[:skip_filled_timelines]) unless dry_run?
end
say("Regenerated feeds for #{processed} accounts #{dry_run_mode_suffix}", :green, true)
@ -29,7 +34,7 @@ module Mastodon::CLI
fail_with_message 'No such account' if account.nil?
PrecomputeFeedService.new.call(account) unless dry_run?
PrecomputeFeedService.new.call(account, skip_filled_timelines: options[:skip_filled_timelines]) unless dry_run?
say("OK #{dry_run_mode_suffix}", :green, true)
else
@ -44,6 +49,38 @@ module Mastodon::CLI
say('OK', :green)
end
desc 'vacuum', 'Remove home feeds of inactive users from Redis'
long_desc <<-LONG_DESC
Running this task should not be needed in most cases, as Mastodon will
automatically clean up feeds from inactive accounts every day.
However, this task is more aggressive in order to clean up feeds that
may have been missed because of bugs or database mishaps.
LONG_DESC
def vacuum
with_read_replica do
say('Deleting orphaned home feeds…')
redis.scan_each(match: 'feed:home:*').each_slice(1000) do |keys|
ids = keys.map { |key| key.split(':')[2] }.compact_blank
known_ids = User.confirmed.signed_in_recently.where(account_id: ids).pluck(:account_id)
keys_to_delete = keys.filter { |key| known_ids.exclude?(key.split(':')[2]&.to_i) }
redis.del(keys_to_delete)
end
say('Deleting orphaned list feeds…')
redis.scan_each(match: 'feed:list:*').each_slice(1000) do |keys|
ids = keys.map { |key| key.split(':')[2] }.compact_blank
known_ids = List.where(account_id: User.confirmed.signed_in_recently.select(:account_id)).where(id: ids).pluck(:id)
keys_to_delete = keys.filter { |key| known_ids.exclude?(key.split(':')[2]&.to_i) }
redis.del(keys_to_delete)
end
end
end
private
def active_user_accounts

4
lib/mastodon/cli/ip_blocks.rb
View file

@ -80,9 +80,9 @@ module Mastodon::CLI
end
ip_blocks = if options[:force]
IpBlock.where('ip >>= ?', address)
IpBlock.containing(address)
else
IpBlock.where('ip <<= ?', address)
IpBlock.contained_by(address)
end
if ip_blocks.empty?

10
lib/mastodon/cli/maintenance.rb
View file

@ -43,6 +43,7 @@ module Mastodon::CLI
class BulkImport < ApplicationRecord; end
class SoftwareUpdate < ApplicationRecord; end
class SeveredRelationship < ApplicationRecord; end
class TagFollow < ApplicationRecord; end
class DomainBlock < ApplicationRecord
enum :severity, { silence: 0, suspend: 1, noop: 2 }
@ -102,6 +103,7 @@ module Mastodon::CLI
owned_classes << AccountIdentityProof if db_table_exists?(:account_identity_proofs)
owned_classes << Appeal if db_table_exists?(:appeals)
owned_classes << BulkImport if db_table_exists?(:bulk_imports)
owned_classes << TagFollow if db_table_exists?(:tag_follows)
owned_classes.each do |klass|
klass.where(account_id: other_account.id).find_each do |record|
@ -190,6 +192,7 @@ module Mastodon::CLI
verify_schema_version!
verify_sidekiq_not_active!
verify_backup_warning!
disable_timeout!
end
def process_deduplications
@ -249,6 +252,13 @@ module Mastodon::CLI
fail_with_message 'Maintenance process stopped.' unless yes?('Continue? (Yes/No)')
end
def disable_timeout!
# Remove server-configured timeout if present
database_connection.execute(<<~SQL.squish)
SET statement_timeout = 0
SQL
end
def deduplicate_accounts!
remove_index_if_exists!(:accounts, 'index_accounts_on_username_and_domain_lower')

77
lib/mastodon/cli/media.rb
View file

@ -6,6 +6,8 @@ module Mastodon::CLI
class Media < Base
include ActionView::Helpers::NumberHelper
class UnrecognizedOrphanType < StandardError; end
VALID_PATH_SEGMENTS_SIZE = [7, 10].freeze
option :days, type: :numeric, default: 7, aliases: [:d]
@ -120,23 +122,10 @@ module Mastodon::CLI
object.acl.put(acl: s3_permissions) if options[:fix_permissions] && !dry_run?
path_segments = object.key.split('/')
path_segments.delete('cache')
unless VALID_PATH_SEGMENTS_SIZE.include?(path_segments.size)
progress.log(pastel.yellow("Unrecognized file found: #{object.key}"))
next
end
model_name = path_segments.first.classify
attachment_name = path_segments[1].singularize
record_id = path_segments[2...-2].join.to_i
file_name = path_segments.last
record = record_map.dig(model_name, record_id)
attachment = record&.public_send(attachment_name)
progress.increment
next unless attachment.blank? || !attachment.variant?(file_name)
next unless orphaned_file?(path_segments, record_map)
begin
object.delete unless dry_run?
@ -148,6 +137,8 @@ module Mastodon::CLI
rescue => e
progress.log(pastel.red("Error processing #{object.key}: #{e}"))
end
rescue UnrecognizedOrphanType
progress.log(pastel.yellow("Unrecognized file found: #{object.key}"))
end
end
when :fog
@ -165,26 +156,10 @@ module Mastodon::CLI
key = path.gsub("#{root_path}#{File::SEPARATOR}", '')
path_segments = key.split(File::SEPARATOR)
path_segments.delete('cache')
unless VALID_PATH_SEGMENTS_SIZE.include?(path_segments.size)
progress.log(pastel.yellow("Unrecognized file found: #{key}"))
next
end
model_name = path_segments.first.classify
record_id = path_segments[2...-2].join.to_i
attachment_name = path_segments[1].singularize
file_name = path_segments.last
next unless PRELOADED_MODELS.include?(model_name)
record = model_name.constantize.find_by(id: record_id)
attachment = record&.public_send(attachment_name)
progress.increment
next unless attachment.blank? || !attachment.variant?(file_name)
next unless orphaned_file?(path_segments)
begin
size = File.size(path)
@ -205,6 +180,8 @@ module Mastodon::CLI
rescue => e
progress.log(pastel.red("Error processing #{key}: #{e}"))
end
rescue UnrecognizedOrphanType
progress.log(pastel.yellow("Unrecognized file found: #{path}"))
end
end
@ -312,6 +289,16 @@ module Mastodon::CLI
fail_with_message 'Invalid URL'
end
PRELOADED_MODELS = %w(
Account
Backup
CustomEmoji
Import
MediaAttachment
PreviewCard
SiteUpload
).freeze
private
def object_storage_summary
@ -333,16 +320,6 @@ module Mastodon::CLI
SQL
end
PRELOADED_MODELS = %w(
Account
Backup
CustomEmoji
Import
MediaAttachment
PreviewCard
SiteUpload
).freeze
def preload_records_from_mixed_objects(objects)
preload_map = Hash.new { |hash, key| hash[key] = [] }
@ -364,5 +341,23 @@ module Mastodon::CLI
model_map[model_name] = model_name.constantize.where(id: record_ids).index_by(&:id)
end
end
def orphaned_file?(path_segments, record_map = nil)
path_segments.delete('cache')
raise UnrecognizedOrphanType unless VALID_PATH_SEGMENTS_SIZE.include?(path_segments.size)
model_name = path_segments.first.classify
record_id = path_segments[2...-2].join.to_i
attachment_name = path_segments[1].singularize
file_name = path_segments.last
raise UnrecognizedOrphanType unless PRELOADED_MODELS.include?(model_name)
record = record_map.present? ? record_map.dig(model_name, record_id) : model_name.constantize.find_by(id: record_id)
attachment = record&.public_send(attachment_name)
attachment.blank? || !attachment.variant?(file_name)
end
end
end

4
lib/mastodon/cli/progress_helper.rb
View file

@ -1,11 +1,11 @@
# frozen_string_literal: true
dev_null = Logger.new('/dev/null')
dev_null = Logger.new(File::NULL)
Rails.logger = dev_null
ActiveRecord::Base.logger = dev_null
ActiveJob::Base.logger = dev_null
HttpLog.configuration.logger = dev_null
HttpLog.configuration.logger = dev_null if defined?(HttpLog)
Paperclip.options[:log] = false
Chewy.logger = dev_null

19
lib/mastodon/cli/statuses.rb
View file

@ -40,17 +40,11 @@ module Mastodon::CLI
def remove_statuses
return if options[:skip_status_remove]
say('Creating temporary database indices...')
ActiveRecord::Base.connection.add_index(:media_attachments, :remote_url, name: :index_media_attachments_remote_url, where: 'remote_url is not null', algorithm: :concurrently, if_not_exists: true)
max_id = Mastodon::Snowflake.id_at(options[:days].days.ago, with_random: false)
start_at = Time.now.to_f
unless options[:continue] && ActiveRecord::Base.connection.table_exists?('statuses_to_be_deleted')
ActiveRecord::Base.connection.add_index(:accounts, :id, name: :index_accounts_local, where: 'domain is null', algorithm: :concurrently, if_not_exists: true)
ActiveRecord::Base.connection.add_index(:status_pins, :status_id, name: :index_status_pins_status_id, algorithm: :concurrently, if_not_exists: true)
max_id = Mastodon::Snowflake.id_at(options[:days].days.ago, with_random: false)
unless options[:continue] && ActiveRecord::Base.connection.table_exists?('statuses_to_be_deleted')
say('Extract the deletion target from statuses... This might take a while...')
ActiveRecord::Base.connection.create_table('statuses_to_be_deleted', force: true)
@ -72,9 +66,6 @@ module Mastodon::CLI
SQL
say('Removing temporary database indices to restore write performance...')
ActiveRecord::Base.connection.remove_index(:accounts, name: :index_accounts_local, if_exists: true)
ActiveRecord::Base.connection.remove_index(:status_pins, name: :index_status_pins_status_id, if_exists: true)
end
say('Beginning statuses removal... This might take a while...')
@ -102,12 +93,6 @@ module Mastodon::CLI
ActiveRecord::Base.connection.drop_table('statuses_to_be_deleted')
say("Done after #{Time.now.to_f - start_at}s, removed #{removed} out of #{processed} statuses.", :green)
ensure
say('Removing temporary database indices to restore write performance...')
ActiveRecord::Base.connection.remove_index(:accounts, name: :index_accounts_local, if_exists: true)
ActiveRecord::Base.connection.remove_index(:status_pins, name: :index_status_pins_status_id, if_exists: true)
ActiveRecord::Base.connection.remove_index(:media_attachments, name: :index_media_attachments_remote_url, if_exists: true)
end
def remove_orphans_media_attachments

46
lib/mastodon/database.rb Normal file
View file

@ -0,0 +1,46 @@
# frozen_string_literal: true
# This file is entirely lifted from GitLab.
# The original file:
# https://gitlab.com/gitlab-org/gitlab/-/blob/69127d59467185cf4ff1109d88ceec1f499f354f/lib/gitlab/database.rb#L244-258
# Copyright (c) 2011-present GitLab B.V.
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
module Mastodon
module Database
def self.add_post_migrate_path_to_rails(force: false)
return if ENV['SKIP_POST_DEPLOYMENT_MIGRATIONS'] && !force
Rails.application.config.paths['db'].each do |db_path|
path = Rails.root.join(db_path, 'post_migrate').to_s
next if Rails.application.config.paths['db/migrate'].include?(path)
Rails.application.config.paths['db/migrate'] << path
# Rails memoizes migrations at certain points where it won't read the above
# path just yet. As such we must also update the following list of paths.
ActiveRecord::Migrator.migrations_paths << path
end
end
end
end

26
lib/mastodon/feature.rb Normal file
View file

@ -0,0 +1,26 @@
# frozen_string_literal: true
module Mastodon::Feature
class << self
def enabled_features
@enabled_features ||=
(Rails.configuration.x.mastodon.experimental_features || '').split(',').map(&:strip)
end
def method_missing(name)
if respond_to_missing?(name)
feature = name.to_s.delete_suffix('_enabled?')
enabled = enabled_features.include?(feature)
define_singleton_method(name) { enabled }
return enabled
end
super
end
def respond_to_missing?(name, include_all = false)
name.to_s.end_with?('_enabled?') || super
end
end
end

52
lib/mastodon/middleware/public_file_server.rb Normal file
View file

@ -0,0 +1,52 @@
# frozen_string_literal: true
require 'action_dispatch/middleware/static'
module Mastodon
module Middleware
class PublicFileServer
SERVICE_WORKER_TTL = 7.days.to_i
CACHE_TTL = 28.days.to_i
def initialize(app)
@app = app
@file_handler = ActionDispatch::FileHandler.new(Rails.application.paths['public'].first)
end
def call(env)
file = @file_handler.attempt(env)
# If the request is not a static file, move on!
return @app.call(env) if file.nil?
status, headers, response = file
# Set cache headers on static files. Some paths require different cache headers
headers['Cache-Control'] = begin
request_path = env['REQUEST_PATH']
if request_path.start_with?('/sw.js')
"public, max-age=#{SERVICE_WORKER_TTL}, must-revalidate"
elsif request_path.start_with?(paperclip_root_url)
"public, max-age=#{CACHE_TTL}, immutable"
else
"public, max-age=#{CACHE_TTL}, must-revalidate"
end
end
# Override the default CSP header set by the CSP middleware
headers['Content-Security-Policy'] = "default-src 'none'; form-action 'none'" if request_path.start_with?(paperclip_root_url)
headers['X-Content-Type-Options'] = 'nosniff'
[status, headers, response]
end
private
def paperclip_root_url
ENV.fetch('PAPERCLIP_ROOT_URL', '/system')
end
end
end
end

34
lib/mastodon/middleware/socket_cleanup.rb Normal file
View file

@ -0,0 +1,34 @@
# frozen_string_literal: true
module Mastodon
module Middleware
class SocketCleanup
def initialize(app)
@app = app
end
def call(env)
@app.call(env)
ensure
clean_up_sockets!
end
private
def clean_up_sockets!
clean_up_redis_socket!
clean_up_statsd_socket!
end
def clean_up_redis_socket!
RedisConnection.pool.checkin if Thread.current[:redis]
Thread.current[:redis] = nil
end
def clean_up_statsd_socket!
Thread.current[:statsd_socket]&.close
Thread.current[:statsd_socket] = nil
end
end
end
end

2
lib/mastodon/migration_warning.rb
View file

@ -4,7 +4,7 @@ module Mastodon
module MigrationWarning
WARNING_SECONDS = 10
DEFAULT_WARNING = <<~WARNING_MESSAGE
DEFAULT_WARNING = <<~WARNING_MESSAGE.freeze
WARNING: This migration may take a *long* time for large instances.
It will *not* lock tables for any significant time, but it may run
for a very long time. We will pause for #{WARNING_SECONDS} seconds to allow you to

30
lib/mastodon/rack_middleware.rb
View file

@ -1,30 +0,0 @@
# frozen_string_literal: true
class Mastodon::RackMiddleware
def initialize(app)
@app = app
end
def call(env)
@app.call(env)
ensure
clean_up_sockets!
end
private
def clean_up_sockets!
clean_up_redis_socket!
clean_up_statsd_socket!
end
def clean_up_redis_socket!
RedisConnection.pool.checkin if Thread.current[:redis]
Thread.current[:redis] = nil
end
def clean_up_statsd_socket!
Thread.current[:statsd_socket]&.close
Thread.current[:statsd_socket] = nil
end
end

16
lib/mastodon/sidekiq_middleware.rb
View file

@ -3,8 +3,10 @@
class Mastodon::SidekiqMiddleware
BACKTRACE_LIMIT = 3
def call(*, &block)
Chewy.strategy(:mastodon, &block)
def call(_worker_class, job, _queue, &block)
setup_query_log_tags(job) do
Chewy.strategy(:mastodon, &block)
end
rescue Mastodon::HostValidationError
# Do not retry
rescue => e
@ -61,4 +63,14 @@ class Mastodon::SidekiqMiddleware
Thread.current[:statsd_socket]&.close
Thread.current[:statsd_socket] = nil
end
def setup_query_log_tags(job, &block)
if Rails.configuration.active_record.query_log_tags_enabled
# If `wrapped` is set, this is an `ActiveJob` which is already in the execution context
sidekiq_job_class = job['wrapped'].present? ? nil : job['class'].to_s
ActiveSupport::ExecutionContext.set(sidekiq_job_class: sidekiq_job_class, &block)
else
yield
end
end
end

30
lib/mastodon/version.rb
View file

@ -17,15 +17,15 @@ module Mastodon
end
def default_prerelease
'alpha.1'
'alpha.4'
end
def prerelease
ENV['MASTODON_VERSION_PRERELEASE'].presence || default_prerelease
version_configuration[:prerelease].presence || default_prerelease
end
def build_metadata
'instrumental'
'legamunt'
end
def to_a
@ -45,21 +45,21 @@ module Mastodon
def api_versions
{
mastodon: 2,
mastodon: 4,
}
end
def repository
'SWREI/high-school.band'
source_configuration[:repository]
end
def source_base_url
ENV.fetch('SOURCE_BASE_URL', "https://git.psec.dev/#{repository}")
source_configuration[:base_url] || "https://github.com/#{repository}"
end
# specify git tag or commit hash here
def source_tag
ENV.fetch('SOURCE_TAG', nil)
source_configuration[:tag]
end
def source_url
@ -70,8 +70,24 @@ module Mastodon
end
end
def source_commit
ENV.fetch('SOURCE_COMMIT', nil)
end
def user_agent
@user_agent ||= "Mastodon/#{Version} (#{HTTP::Request::USER_AGENT}; +http#{Rails.configuration.x.use_https ? 's' : ''}://#{Rails.configuration.x.web_domain}/)"
end
def version_configuration
mastodon_configuration.version
end
def source_configuration
mastodon_configuration.source
end
def mastodon_configuration
Rails.configuration.x.mastodon
end
end
end