Introduce Ostatus name space (#4164)

* Wrap methods of ProcessFeedService::ProcessEntry in classes

This is a change same with 425acecfdb, except
that it has the following changes:

* Revert irrelevant change in find_or_create_conversation
* Fix error handling for RemoteActivity

* Introduce Ostatus name space

app/lib/ostatus/activity/base.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+class Ostatus::Activity::Base
+  def initialize(xml, account = nil)
+    @xml = xml
+    @account = account
+  end
+
+  def status?
+    [:activity, :note, :comment].include?(type)
+  end
+
+  def verb
+    raw = @xml.at_xpath('./activity:verb', activity: TagManager::AS_XMLNS).content
+    TagManager::VERBS.key(raw)
+  rescue
+    :post
+  end
+
+  def type
+    raw = @xml.at_xpath('./activity:object-type', activity: TagManager::AS_XMLNS).content
+    TagManager::TYPES.key(raw)
+  rescue
+    :activity
+  end
+
+  def id
+    @xml.at_xpath('./xmlns:id', xmlns: TagManager::XMLNS).content
+  end
+
+  def url
+    link = @xml.at_xpath('./xmlns:link[@rel="alternate"]', xmlns: TagManager::XMLNS)
+    link.nil? ? nil : link['href']
+  end
+
+  private
+
+  def find_status(uri)
+    if TagManager.instance.local_id?(uri)
+      local_id = TagManager.instance.unique_tag_to_local_id(uri, 'Status')
+      return Status.find_by(id: local_id)
+    end
+
+    Status.find_by(uri: uri)
+  end
+
+  def redis
+    Redis.current
+  end
+end

app/lib/ostatus/activity/creation.rb

@@ -0,0 +1,149 @@
+# frozen_string_literal: true
+
+class Ostatus::Activity::Creation < Ostatus::Activity::Base
+  def perform
+    if redis.exists("delete_upon_arrival:#{@account.id}:#{id}")
+      Rails.logger.debug "Delete for status #{id} was queued, ignoring"
+      return [nil, false]
+    end
+
+    return [nil, false] if @account.suspended?
+
+    Rails.logger.debug "Creating remote status #{id}"
+
+    # Return early if status already exists in db
+    status = find_status(id)
+
+    return [status, false] unless status.nil?
+
+    status = Status.create!(
+      uri: id,
+      url: url,
+      account: @account,
+      reblog: reblog,
+      text: content,
+      spoiler_text: content_warning,
+      created_at: published,
+      reply: thread?,
+      language: content_language,
+      visibility: visibility_scope,
+      conversation: find_or_create_conversation,
+      thread: thread? ? find_status(thread.first) : nil
+    )
+
+    save_mentions(status)
+    save_hashtags(status)
+    save_media(status)
+
+    if thread? && status.thread.nil?
+      Rails.logger.debug "Trying to attach #{status.id} (#{id}) to #{thread.first}"
+      ThreadResolveWorker.perform_async(status.id, thread.second)
+    end
+
+    Rails.logger.debug "Queuing remote status #{status.id} (#{id}) for distribution"
+
+    LinkCrawlWorker.perform_async(status.id) unless status.spoiler_text?
+    DistributionWorker.perform_async(status.id)
+
+    [status, true]
+  end
+
+  def content
+    @xml.at_xpath('./xmlns:content', xmlns: TagManager::XMLNS).content
+  end
+
+  def content_language
+    @xml.at_xpath('./xmlns:content', xmlns: TagManager::XMLNS)['xml:lang']&.presence || 'en'
+  end
+
+  def content_warning
+    @xml.at_xpath('./xmlns:summary', xmlns: TagManager::XMLNS)&.content || ''
+  end
+
+  def visibility_scope
+    @xml.at_xpath('./mastodon:scope', mastodon: TagManager::MTDN_XMLNS)&.content&.to_sym || :public
+  end
+
+  def published
+    @xml.at_xpath('./xmlns:published', xmlns: TagManager::XMLNS).content
+  end
+
+  def thread?
+    !@xml.at_xpath('./thr:in-reply-to', thr: TagManager::THR_XMLNS).nil?
+  end
+
+  def thread
+    thr = @xml.at_xpath('./thr:in-reply-to', thr: TagManager::THR_XMLNS)
+    [thr['ref'], thr['href']]
+  end
+
+  private
+
+  def find_or_create_conversation
+    uri = @xml.at_xpath('./ostatus:conversation', ostatus: TagManager::OS_XMLNS)&.attribute('ref')&.content
+    return if uri.nil?
+
+    if TagManager.instance.local_id?(uri)
+      local_id = TagManager.instance.unique_tag_to_local_id(uri, 'Conversation')
+      return Conversation.find_by(id: local_id)
+    end
+
+    Conversation.find_by(uri: uri) || Conversation.create!(uri: uri)
+  end
+
+  def save_mentions(parent)
+    processed_account_ids = []
+
+    @xml.xpath('./xmlns:link[@rel="mentioned"]', xmlns: TagManager::XMLNS).each do |link|
+      next if [TagManager::TYPES[:group], TagManager::TYPES[:collection]].include? link['ostatus:object-type']
+
+      mentioned_account = account_from_href(link['href'])
+
+      next if mentioned_account.nil? || processed_account_ids.include?(mentioned_account.id)
+
+      mentioned_account.mentions.where(status: parent).first_or_create(status: parent)
+
+      # So we can skip duplicate mentions
+      processed_account_ids << mentioned_account.id
+    end
+  end
+
+  def save_hashtags(parent)
+    tags = @xml.xpath('./xmlns:category', xmlns: TagManager::XMLNS).map { |category| category['term'] }.select(&:present?)
+    ProcessHashtagsService.new.call(parent, tags)
+  end
+
+  def save_media(parent)
+    do_not_download = DomainBlock.find_by(domain: parent.account.domain)&.reject_media?
+
+    @xml.xpath('./xmlns:link[@rel="enclosure"]', xmlns: TagManager::XMLNS).each do |link|
+      next unless link['href']
+
+      media = MediaAttachment.where(status: parent, remote_url: link['href']).first_or_initialize(account: parent.account, status: parent, remote_url: link['href'])
+      parsed_url = Addressable::URI.parse(link['href']).normalize
+
+      next if !%w(http https).include?(parsed_url.scheme) || parsed_url.host.empty?
+
+      media.save
+
+      next if do_not_download
+
+      begin
+        media.file_remote_url = link['href']
+        media.save!
+      rescue ActiveRecord::RecordInvalid
+        next
+      end
+    end
+  end
+
+  def account_from_href(href)
+    url = Addressable::URI.parse(href).normalize
+
+    if TagManager.instance.web_domain?(url.host)
+      Account.find_local(url.path.gsub('/users/', ''))
+    else
+      Account.where(uri: href).or(Account.where(url: href)).first || FetchRemoteAccountService.new.call(href)
+    end
+  end
+end

app/lib/ostatus/activity/deletion.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+class Ostatus::Activity::Deletion < Ostatus::Activity::Base
+  def perform
+    Rails.logger.debug "Deleting remote status #{id}"
+    status = Status.find_by(uri: id, account: @account)
+
+    if status.nil?
+      redis.setex("delete_upon_arrival:#{@account.id}:#{id}", 6 * 3_600, id)
+    else
+      RemoveStatusService.new.call(status)
+    end
+  end
+end

app/lib/ostatus/activity/general.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+class Ostatus::Activity::General < Ostatus::Activity::Base
+  def specialize
+    special_class&.new(@xml, @account)
+  end
+
+  private
+
+  def special_class
+    case verb
+    when :post
+      Ostatus::Activity::Post
+    when :share
+      Ostatus::Activity::Share
+    when :delete
+      Ostatus::Activity::Deletion
+    end
+  end
+end

app/lib/ostatus/activity/post.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+class Ostatus::Activity::Post < Ostatus::Activity::Creation
+  def perform
+    status, just_created = super
+
+    if just_created
+      status.mentions.includes(:account).each do |mention|
+        mentioned_account = mention.account
+        next unless mentioned_account.local?
+        NotifyService.new.call(mentioned_account, mention)
+      end
+    end
+
+    status
+  end
+
+  private
+
+  def reblog
+    nil
+  end
+end

app/lib/ostatus/activity/remote.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+class Ostatus::Activity::Remote < Ostatus::Activity::Base
+  def perform
+    find_status(id) || FetchRemoteStatusService.new.call(url)
+  end
+end

app/lib/ostatus/activity/share.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+class Ostatus::Activity::Share < Ostatus::Activity::Creation
+  def perform
+    return if reblog.nil?
+
+    status, just_created = super
+    NotifyService.new.call(reblog.account, status) if reblog.account.local? && just_created
+    status
+  end
+
+  def object
+    @xml.at_xpath('.//activity:object', activity: TagManager::AS_XMLNS)
+  end
+
+  private
+
+  def reblog
+    return @reblog if defined? @reblog
+
+    original_status = Ostatus::Activity::Remote.new(object).perform
+    return if original_status.nil?
+
+    @reblog = original_status.reblog? ? original_status.reblog : original_status
+  end
+end

app/lib/ostatus/atom_serializer.rb

@@ -0,0 +1,367 @@
+# frozen_string_literal: true
+
+class Ostatus::AtomSerializer
+  include RoutingHelper
+  include ActionView::Helpers::SanitizeHelper
+
+  class << self
+    def render(element)
+      document = Ox::Document.new(version: '1.0')
+      document << element
+      ('<?xml version="1.0"?>' + Ox.dump(element, effort: :tolerant)).force_encoding('UTF-8')
+    end
+  end
+
+  def author(account)
+    author = Ox::Element.new('author')
+
+    uri = TagManager.instance.uri_for(account)
+
+    append_element(author, 'id', uri)
+    append_element(author, 'activity:object-type', TagManager::TYPES[:person])
+    append_element(author, 'uri', uri)
+    append_element(author, 'name', account.username)
+    append_element(author, 'email', account.local? ? account.local_username_and_domain : account.acct)
+    append_element(author, 'summary', Formatter.instance.simplified_format(account).to_str, type: :html) if account.note?
+    append_element(author, 'link', nil, rel: :alternate, type: 'text/html', href: TagManager.instance.url_for(account))
+    append_element(author, 'link', nil, rel: :avatar, type: account.avatar_content_type, 'media:width': 120, 'media:height': 120, href: full_asset_url(account.avatar.url(:original))) if account.avatar?
+    append_element(author, 'link', nil, rel: :header, type: account.header_content_type, 'media:width': 700, 'media:height': 335, href: full_asset_url(account.header.url(:original))) if account.header?
+    append_element(author, 'poco:preferredUsername', account.username)
+    append_element(author, 'poco:displayName', account.display_name) if account.display_name?
+    append_element(author, 'poco:note', account.local? ? account.note : strip_tags(account.note)) if account.note?
+    append_element(author, 'mastodon:scope', account.locked? ? :private : :public)
+
+    author
+  end
+
+  def feed(account, stream_entries)
+    feed = Ox::Element.new('feed')
+
+    add_namespaces(feed)
+
+    append_element(feed, 'id', account_url(account, format: 'atom'))
+    append_element(feed, 'title', account.display_name.presence || account.username)
+    append_element(feed, 'subtitle', account.note)
+    append_element(feed, 'updated', account.updated_at.iso8601)
+    append_element(feed, 'logo', full_asset_url(account.avatar.url(:original)))
+
+    feed << author(account)
+
+    append_element(feed, 'link', nil, rel: :alternate, type: 'text/html', href: TagManager.instance.url_for(account))
+    append_element(feed, 'link', nil, rel: :self, type: 'application/atom+xml', href: account_url(account, format: 'atom'))
+    append_element(feed, 'link', nil, rel: :next, type: 'application/atom+xml', href: account_url(account, format: 'atom', max_id: stream_entries.last.id)) if stream_entries.size == 20
+    append_element(feed, 'link', nil, rel: :hub, href: api_push_url)
+    append_element(feed, 'link', nil, rel: :salmon, href: api_salmon_url(account.id))
+
+    stream_entries.each do |stream_entry|
+      feed << entry(stream_entry)
+    end
+
+    feed
+  end
+
+  def entry(stream_entry, root = false)
+    entry = Ox::Element.new('entry')
+
+    add_namespaces(entry) if root
+
+    append_element(entry, 'id', TagManager.instance.unique_tag(stream_entry.created_at, stream_entry.activity_id, stream_entry.activity_type))
+    append_element(entry, 'published', stream_entry.created_at.iso8601)
+    append_element(entry, 'updated', stream_entry.updated_at.iso8601)
+    append_element(entry, 'title', stream_entry&.status&.title || "#{stream_entry.account.acct} deleted status")
+
+    entry << author(stream_entry.account) if root
+
+    append_element(entry, 'activity:object-type', TagManager::TYPES[stream_entry.object_type])
+    append_element(entry, 'activity:verb', TagManager::VERBS[stream_entry.verb])
+
+    entry << object(stream_entry.target) if stream_entry.targeted?
+
+    if stream_entry.status.nil?
+      append_element(entry, 'content', 'Deleted status')
+    else
+      serialize_status_attributes(entry, stream_entry.status)
+    end
+
+    append_element(entry, 'link', nil, rel: :alternate, type: 'text/html', href: account_stream_entry_url(stream_entry.account, stream_entry))
+    append_element(entry, 'link', nil, rel: :self, type: 'application/atom+xml', href: account_stream_entry_url(stream_entry.account, stream_entry, format: 'atom'))
+    append_element(entry, 'thr:in-reply-to', nil, ref: TagManager.instance.uri_for(stream_entry.thread), href: TagManager.instance.url_for(stream_entry.thread)) if stream_entry.threaded?
+    append_element(entry, 'ostatus:conversation', nil, ref: conversation_uri(stream_entry.status.conversation)) unless stream_entry&.status&.conversation_id.nil?
+
+    entry
+  end
+
+  def object(status)
+    object = Ox::Element.new('activity:object')
+
+    append_element(object, 'id', TagManager.instance.uri_for(status))
+    append_element(object, 'published', status.created_at.iso8601)
+    append_element(object, 'updated', status.updated_at.iso8601)
+    append_element(object, 'title', status.title)
+
+    object << author(status.account)
+
+    append_element(object, 'activity:object-type', TagManager::TYPES[status.object_type])
+    append_element(object, 'activity:verb', TagManager::VERBS[status.verb])
+
+    serialize_status_attributes(object, status)
+
+    append_element(object, 'link', nil, rel: :alternate, type: 'text/html', href: TagManager.instance.url_for(status))
+    append_element(object, 'thr:in-reply-to', nil, ref: TagManager.instance.uri_for(status.thread), href: TagManager.instance.url_for(status.thread)) unless status.thread.nil?
+    append_element(object, 'ostatus:conversation', nil, ref: conversation_uri(status.conversation)) unless status.conversation_id.nil?
+
+    object
+  end
+
+  def follow_salmon(follow)
+    entry = Ox::Element.new('entry')
+    add_namespaces(entry)
+
+    description = "#{follow.account.acct} started following #{follow.target_account.acct}"
+
+    append_element(entry, 'id', TagManager.instance.unique_tag(follow.created_at, follow.id, 'Follow'))
+    append_element(entry, 'title', description)
+    append_element(entry, 'content', description, type: :html)
+
+    entry << author(follow.account)
+
+    append_element(entry, 'activity:object-type', TagManager::TYPES[:activity])
+    append_element(entry, 'activity:verb', TagManager::VERBS[:follow])
+
+    object = author(follow.target_account)
+    object.value = 'activity:object'
+
+    entry << object
+    entry
+  end
+
+  def follow_request_salmon(follow_request)
+    entry = Ox::Element.new('entry')
+    add_namespaces(entry)
+
+    append_element(entry, 'id', TagManager.instance.unique_tag(follow_request.created_at, follow_request.id, 'FollowRequest'))
+    append_element(entry, 'title', "#{follow_request.account.acct} requested to follow #{follow_request.target_account.acct}")
+
+    entry << author(follow_request.account)
+
+    append_element(entry, 'activity:object-type', TagManager::TYPES[:activity])
+    append_element(entry, 'activity:verb', TagManager::VERBS[:request_friend])
+
+    object = author(follow_request.target_account)
+    object.value = 'activity:object'
+
+    entry << object
+    entry
+  end
+
+  def authorize_follow_request_salmon(follow_request)
+    entry = Ox::Element.new('entry')
+    add_namespaces(entry)
+
+    append_element(entry, 'id', TagManager.instance.unique_tag(Time.now.utc, follow_request.id, 'FollowRequest'))
+    append_element(entry, 'title', "#{follow_request.target_account.acct} authorizes follow request by #{follow_request.account.acct}")
+
+    entry << author(follow_request.target_account)
+
+    append_element(entry, 'activity:object-type', TagManager::TYPES[:activity])
+    append_element(entry, 'activity:verb', TagManager::VERBS[:authorize])
+
+    object = Ox::Element.new('activity:object')
+    object << author(follow_request.account)
+
+    append_element(object, 'activity:object-type', TagManager::TYPES[:activity])
+    append_element(object, 'activity:verb', TagManager::VERBS[:request_friend])
+
+    inner_object = author(follow_request.target_account)
+    inner_object.value = 'activity:object'
+
+    object << inner_object
+    entry  << object
+    entry
+  end
+
+  def reject_follow_request_salmon(follow_request)
+    entry = Ox::Element.new('entry')
+    add_namespaces(entry)
+
+    append_element(entry, 'id', TagManager.instance.unique_tag(Time.now.utc, follow_request.id, 'FollowRequest'))
+    append_element(entry, 'title', "#{follow_request.target_account.acct} rejects follow request by #{follow_request.account.acct}")
+
+    entry << author(follow_request.target_account)
+
+    append_element(entry, 'activity:object-type', TagManager::TYPES[:activity])
+    append_element(entry, 'activity:verb', TagManager::VERBS[:reject])
+
+    object = Ox::Element.new('activity:object')
+    object << author(follow_request.account)
+
+    append_element(object, 'activity:object-type', TagManager::TYPES[:activity])
+    append_element(object, 'activity:verb', TagManager::VERBS[:request_friend])
+
+    inner_object = author(follow_request.target_account)
+    inner_object.value = 'activity:object'
+
+    object << inner_object
+    entry  << object
+    entry
+  end
+
+  def unfollow_salmon(follow)
+    entry = Ox::Element.new('entry')
+    add_namespaces(entry)
+
+    description = "#{follow.account.acct} is no longer following #{follow.target_account.acct}"
+
+    append_element(entry, 'id', TagManager.instance.unique_tag(Time.now.utc, follow.id, 'Follow'))
+    append_element(entry, 'title', description)
+    append_element(entry, 'content', description, type: :html)
+
+    entry << author(follow.account)
+
+    append_element(entry, 'activity:object-type', TagManager::TYPES[:activity])
+    append_element(entry, 'activity:verb', TagManager::VERBS[:unfollow])
+
+    object = author(follow.target_account)
+    object.value = 'activity:object'
+
+    entry << object
+    entry
+  end
+
+  def block_salmon(block)
+    entry = Ox::Element.new('entry')
+    add_namespaces(entry)
+
+    description = "#{block.account.acct} no longer wishes to interact with #{block.target_account.acct}"
+
+    append_element(entry, 'id', TagManager.instance.unique_tag(Time.now.utc, block.id, 'Block'))
+    append_element(entry, 'title', description)
+
+    entry << author(block.account)
+
+    append_element(entry, 'activity:object-type', TagManager::TYPES[:activity])
+    append_element(entry, 'activity:verb', TagManager::VERBS[:block])
+
+    object = author(block.target_account)
+    object.value = 'activity:object'
+
+    entry << object
+    entry
+  end
+
+  def unblock_salmon(block)
+    entry = Ox::Element.new('entry')
+    add_namespaces(entry)
+
+    description = "#{block.account.acct} no longer blocks #{block.target_account.acct}"
+
+    append_element(entry, 'id', TagManager.instance.unique_tag(Time.now.utc, block.id, 'Block'))
+    append_element(entry, 'title', description)
+
+    entry << author(block.account)
+
+    append_element(entry, 'activity:object-type', TagManager::TYPES[:activity])
+    append_element(entry, 'activity:verb', TagManager::VERBS[:unblock])
+
+    object = author(block.target_account)
+    object.value = 'activity:object'
+
+    entry << object
+    entry
+  end
+
+  def favourite_salmon(favourite)
+    entry = Ox::Element.new('entry')
+    add_namespaces(entry)
+
+    description = "#{favourite.account.acct} favourited a status by #{favourite.status.account.acct}"
+
+    append_element(entry, 'id', TagManager.instance.unique_tag(favourite.created_at, favourite.id, 'Favourite'))
+    append_element(entry, 'title', description)
+    append_element(entry, 'content', description, type: :html)
+
+    entry << author(favourite.account)
+
+    append_element(entry, 'activity:object-type', TagManager::TYPES[:activity])
+    append_element(entry, 'activity:verb', TagManager::VERBS[:favorite])
+
+    entry << object(favourite.status)
+
+    append_element(entry, 'thr:in-reply-to', nil, ref: TagManager.instance.uri_for(favourite.status), href: TagManager.instance.url_for(favourite.status))
+
+    entry
+  end
+
+  def unfavourite_salmon(favourite)
+    entry = Ox::Element.new('entry')
+    add_namespaces(entry)
+
+    description = "#{favourite.account.acct} no longer favourites a status by #{favourite.status.account.acct}"
+
+    append_element(entry, 'id', TagManager.instance.unique_tag(Time.now.utc, favourite.id, 'Favourite'))
+    append_element(entry, 'title', description)
+    append_element(entry, 'content', description, type: :html)
+
+    entry << author(favourite.account)
+
+    append_element(entry, 'activity:object-type', TagManager::TYPES[:activity])
+    append_element(entry, 'activity:verb', TagManager::VERBS[:unfavorite])
+
+    entry << object(favourite.status)
+
+    append_element(entry, 'thr:in-reply-to', nil, ref: TagManager.instance.uri_for(favourite.status), href: TagManager.instance.url_for(favourite.status))
+
+    entry
+  end
+
+  private
+
+  def append_element(parent, name, content = nil, attributes = {})
+    element = Ox::Element.new(name)
+    attributes.each { |k, v| element[k] = sanitize_str(v) }
+    element << sanitize_str(content) unless content.nil?
+    parent  << element
+  end
+
+  def sanitize_str(raw_str)
+    raw_str.to_s
+  end
+
+  def conversation_uri(conversation)
+    return conversation.uri if conversation.uri?
+    TagManager.instance.unique_tag(conversation.created_at, conversation.id, 'Conversation')
+  end
+
+  def add_namespaces(parent)
+    parent['xmlns']          = TagManager::XMLNS
+    parent['xmlns:thr']      = TagManager::THR_XMLNS
+    parent['xmlns:activity'] = TagManager::AS_XMLNS
+    parent['xmlns:poco']     = TagManager::POCO_XMLNS
+    parent['xmlns:media']    = TagManager::MEDIA_XMLNS
+    parent['xmlns:ostatus']  = TagManager::OS_XMLNS
+    parent['xmlns:mastodon'] = TagManager::MTDN_XMLNS
+  end
+
+  def serialize_status_attributes(entry, status)
+    append_element(entry, 'summary', status.spoiler_text, 'xml:lang': status.language) if status.spoiler_text?
+    append_element(entry, 'content', Formatter.instance.format(status).to_str, type: 'html', 'xml:lang': status.language)
+
+    status.mentions.each do |mentioned|
+      append_element(entry, 'link', nil, rel: :mentioned, 'ostatus:object-type': TagManager::TYPES[:person], href: TagManager.instance.uri_for(mentioned.account))
+    end
+
+    append_element(entry, 'link', nil, rel: :mentioned, 'ostatus:object-type': TagManager::TYPES[:collection], href: TagManager::COLLECTIONS[:public]) if status.public_visibility?
+
+    status.tags.each do |tag|
+      append_element(entry, 'category', nil, term: tag.name)
+    end
+
+    append_element(entry, 'category', nil, term: 'nsfw') if status.sensitive?
+
+    status.media_attachments.each do |media|
+      append_element(entry, 'link', nil, rel: :enclosure, type: media.file_content_type, length: media.file_file_size, href: full_asset_url(media.file.url(:original, false)))
+    end
+
+    append_element(entry, 'mastodon:scope', status.visibility)
+  end
+end