Unfloyded/Legamunt
0
0
Fork 0
Code Activity

Slightly reduce RAM usage (#7301)

Browse source 
* No need to re-require sidekiq plugins, they are required via Gemfile

* Add derailed_benchmarks tool, no need to require TTY gems in Gemfile

* Replace ruby-oembed with FetchOEmbedService

Reduce startup by 45382 allocated objects

* Remove preloaded JSON-LD in favour of caching HTTP responses

Reduce boot RAM by about 6 MiB

* Fix tests

* Fix test suite by stubbing out JSON-LD contexts
This commit is contained in:
Eugen Rochko 2018-05-02 18:58:48 +02:00 • committed by GitHub
parent 71a7cea73f
commit cb5b5cb5f7
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
76 changed files with 784 additions and 471 deletions
Download patch file Download diff file Expand all files Collapse all files

11
app/controllers/api/web/embeds_controller.rb
View file

@ -9,9 +9,12 @@ class Api::Web::EmbedsController < Api::Web::BaseController
status = StatusFinder.new(params[:url]).status
render json: status, serializer: OEmbedSerializer, width: 400
rescue ActiveRecord::RecordNotFound
oembed = OEmbed::Providers.get(params[:url])
render json: Oj.dump(oembed.fields)
rescue OEmbed::NotFound
render json: {}, status: :not_found
oembed = FetchOEmbedService.new.call(params[:url])
if oembed
render json: oembed
else
render json: {}, status: :not_found
end
end
end

2
app/controllers/settings/follower_domains_controller.rb
View file

@ -1,7 +1,5 @@
# frozen_string_literal: true
require 'sidekiq-bulk'
class Settings::FollowerDomainsController < ApplicationController
layout 'admin'

17
app/helpers/jsonld_helper.rb
View file

@ -48,7 +48,7 @@ module JsonLdHelper
end
def canonicalize(json)
graph = RDF::Graph.new << JSON::LD::API.toRdf(json)
graph = RDF::Graph.new << JSON::LD::API.toRdf(json, documentLoader: method(:load_jsonld_context))
graph.dump(:normalize)
end
@ -90,4 +90,19 @@ module JsonLdHelper
request.add_headers('Accept' => 'application/activity+json, application/ld+json')
request
end
def load_jsonld_context(url, _options = {}, &_block)
json = Rails.cache.fetch("jsonld:context:#{url}", expires_in: 30.days, raw: true) do
request = Request.new(:get, url)
request.add_headers('Accept' => 'application/ld+json')
request.perform do |res|
raise JSON::LD::JsonLdError::LoadingDocumentFailed unless res.code == 200 && res.mime_type == 'application/ld+json'
res.body_with_limit
end
end
doc = JSON::LD::API::RemoteDocument.new(url, json)
block_given? ? yield(doc) : doc
end
end

47
app/lib/provider_discovery.rb
View file

@ -1,47 +0,0 @@
# frozen_string_literal: true
class ProviderDiscovery < OEmbed::ProviderDiscovery
class << self
def get(url, **options)
provider = discover_provider(url, options)
options.delete(:html)
provider.get(url, options)
end
def discover_provider(url, **options)
format = options[:format]
html = if options[:html]
Nokogiri::HTML(options[:html])
else
Request.new(:get, url).perform do |res|
raise OEmbed::NotFound, url if res.code != 200 || res.mime_type != 'text/html'
Nokogiri::HTML(res.body_with_limit)
end
end
if format.nil? || format == :json
provider_endpoint ||= html.at_xpath('//link[@type="application/json+oembed"]')&.attribute('href')&.value
format ||= :json if provider_endpoint
end
if format.nil? || format == :xml
provider_endpoint ||= html.at_xpath('//link[@type="text/xml+oembed"]')&.attribute('href')&.value
format ||= :xml if provider_endpoint
end
raise OEmbed::NotFound, url if provider_endpoint.nil?
begin
provider_endpoint = Addressable::URI.parse(provider_endpoint)
provider_endpoint.query = nil
provider_endpoint = provider_endpoint.to_s
rescue Addressable::URI::InvalidURIError
raise OEmbed::NotFound, url
end
OEmbed::Provider.new(provider_endpoint, format)
end
end
end

2
app/services/fan_out_on_write_service.rb
View file

@ -1,7 +1,5 @@
# frozen_string_literal: true
require 'sidekiq-bulk'
class FanOutOnWriteService < BaseService
# Push a status into home and mentions feeds
# @param [Status] status

38
app/services/fetch_link_card_service.rb
View file

@ -85,42 +85,40 @@ class FetchLinkCardService < BaseService
end
def attempt_oembed
embed = OEmbed::Providers.get(@url, html: @html)
embed = FetchOEmbedService.new.call(@url, html: @html)
return false unless embed.respond_to?(:type)
return false if embed.nil?
@card.type = embed.type
@card.title = embed.respond_to?(:title) ? embed.title : ''
@card.author_name = embed.respond_to?(:author_name) ? embed.author_name : ''
@card.author_url = embed.respond_to?(:author_url) ? embed.author_url : ''
@card.provider_name = embed.respond_to?(:provider_name) ? embed.provider_name : ''
@card.provider_url = embed.respond_to?(:provider_url) ? embed.provider_url : ''
@card.type = embed[:type]
@card.title = embed[:title] || ''
@card.author_name = embed[:author_name] || ''
@card.author_url = embed[:author_url] || ''
@card.provider_name = embed[:provider_name] || ''
@card.provider_url = embed[:provider_url] || ''
@card.width = 0
@card.height = 0
case @card.type
when 'link'
@card.image_remote_url = embed.thumbnail_url if embed.respond_to?(:thumbnail_url)
@card.image_remote_url = embed[:thumbnail_url] if embed[:thumbnail_url].present?
when 'photo'
return false unless embed.respond_to?(:url)
return false if embed[:url].blank?
@card.embed_url = embed.url
@card.image_remote_url = embed.url
@card.width = embed.width.presence || 0
@card.height = embed.height.presence || 0
@card.embed_url = embed[:url]
@card.image_remote_url = embed[:url]
@card.width = embed[:width].presence || 0
@card.height = embed[:height].presence || 0
when 'video'
@card.width = embed.width.presence || 0
@card.height = embed.height.presence || 0
@card.html = Formatter.instance.sanitize(embed.html, Sanitize::Config::MASTODON_OEMBED)
@card.image_remote_url = embed.thumbnail_url if embed.respond_to?(:thumbnail_url)
@card.width = embed[:width].presence || 0
@card.height = embed[:height].presence || 0
@card.html = Formatter.instance.sanitize(embed[:html], Sanitize::Config::MASTODON_OEMBED)
@card.image_remote_url = embed[:thumbnail_url] if embed[:thumbnail_url].present?
when 'rich'
# Most providers rely on <script> tags, which is a no-no
return false
end
@card.save_with_optional_image!
rescue OEmbed::NotFound
false
end
def attempt_opengraph

71
app/services/fetch_oembed_service.rb Normal file
View file

@ -0,0 +1,71 @@
# frozen_string_literal: true
class FetchOEmbedService
attr_reader :url, :options, :format, :endpoint_url
def call(url, options = {})
@url = url
@options = options
discover_endpoint!
fetch!
end
private
def discover_endpoint!
return if html.nil?
@format = @options[:format]
page = Nokogiri::HTML(html)
if @format.nil? || @format == :json
@endpoint_url ||= page.at_xpath('//link[@type="application/json+oembed"]')&.attribute('href')&.value
@format ||= :json if @endpoint_url
end
if @format.nil? || @format == :xml
@endpoint_url ||= page.at_xpath('//link[@type="text/xml+oembed"]')&.attribute('href')&.value
@format ||= :xml if @endpoint_url
end
return if @endpoint_url.blank?
@endpoint_url = Addressable::URI.parse(@endpoint_url).to_s
rescue Addressable::URI::InvalidURIError
@endpoint_url = nil
end
def fetch!
return if @endpoint_url.blank?
body = Request.new(:get, @endpoint_url).perform do |res|
res.code != 200 ? nil : res.body_with_limit
end
validate(parse_for_format(body)) unless body.nil?
rescue Oj::ParseError, Ox::ParseError
nil
end
def parse_for_format(body)
case @format
when :json
Oj.load(body, mode: :strict)&.with_indifferent_access
when :xml
Ox.load(body, mode: :hash_no_attrs)&.with_indifferent_access&.dig(:oembed)
end
end
def validate(oembed)
oembed if oembed[:version] == '1.0' && oembed[:type].present?
end
def html
return @html if defined?(@html)
@html = @options[:html] || Request.new(:get, @url).perform do |res|
res.code != 200 || res.mime_type != 'text/html' ? nil : res.body_with_limit
end
end
end

1
app/workers/scheduler/backup_cleanup_scheduler.rb
View file

@ -1,5 +1,4 @@
# frozen_string_literal: true
require 'sidekiq-scheduler'
class Scheduler::BackupCleanupScheduler
include Sidekiq::Worker

1
app/workers/scheduler/doorkeeper_cleanup_scheduler.rb
View file

@ -1,5 +1,4 @@
# frozen_string_literal: true
require 'sidekiq-scheduler'
class Scheduler::DoorkeeperCleanupScheduler
include Sidekiq::Worker

1
app/workers/scheduler/email_scheduler.rb
View file

@ -1,5 +1,4 @@
# frozen_string_literal: true
require 'sidekiq-scheduler'
class Scheduler::EmailScheduler
include Sidekiq::Worker

1
app/workers/scheduler/feed_cleanup_scheduler.rb
View file

@ -1,5 +1,4 @@
# frozen_string_literal: true
require 'sidekiq-scheduler'
class Scheduler::FeedCleanupScheduler
include Sidekiq::Worker

1
app/workers/scheduler/ip_cleanup_scheduler.rb
View file

@ -1,5 +1,4 @@
# frozen_string_literal: true
require 'sidekiq-scheduler'
class Scheduler::IpCleanupScheduler
include Sidekiq::Worker

1
app/workers/scheduler/media_cleanup_scheduler.rb
View file

@ -1,5 +1,4 @@
# frozen_string_literal: true
require 'sidekiq-scheduler'
class Scheduler::MediaCleanupScheduler
include Sidekiq::Worker

2
app/workers/scheduler/subscriptions_cleanup_scheduler.rb
View file

@ -1,7 +1,5 @@
# frozen_string_literal: true
require 'sidekiq-scheduler'
class Scheduler::SubscriptionsCleanupScheduler
include Sidekiq::Worker

3
app/workers/scheduler/subscriptions_scheduler.rb
View file

@ -1,8 +1,5 @@
# frozen_string_literal: true
require 'sidekiq-scheduler'
require 'sidekiq-bulk'
class Scheduler::SubscriptionsScheduler
include Sidekiq::Worker

1
app/workers/scheduler/user_cleanup_scheduler.rb
View file

@ -1,5 +1,4 @@
# frozen_string_literal: true
require 'sidekiq-scheduler'
class Scheduler::UserCleanupScheduler
include Sidekiq::Worker

2
app/workers/soft_block_domain_followers_worker.rb
View file

@ -1,7 +1,5 @@
# frozen_string_literal: true
require 'sidekiq-bulk'
class SoftBlockDomainFollowersWorker
include Sidekiq::Worker