Custom emoji (#4988)

* Custom emoji

- In OStatus: `<link rel="emoji" name="coolcat" href="http://..." />`
- In ActivityPub: `{ type: "Emoji", name: ":coolcat:", href: "http://..." }`
- In REST API: Status object includes `emojis` array (`shortcode`, `url`)
- Domain blocks with reject media stop emojis
- Emoji file up to 50KB
- Web UI handles custom emojis
- Static pages render custom emojis as `<img />` tags

Side effects:

- Undo #4500 optimization, as I needed to modify it to restore
  shortcode handling in emojify()
- Formatter#plaintext should now make sure stripped out line-breaks
  and paragraphs are replaced with newlines

* Fix emoji at the start not being converted

app/lib/activitypub/activity/create.rb

@@ -61,6 +61,8 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
         process_hashtag tag, status
       when 'Mention'
         process_mention tag, status
+      when 'Emoji'
+        process_emoji tag, status
       end
     end
   end
@@ -79,6 +81,17 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
     account.mentions.create(status: status)
   end
 
+  def process_emoji(tag, _status)
+    shortcode = tag['name'].delete(':')
+    emoji     = CustomEmoji.find_by(shortcode: shortcode, domain: @account.domain)
+
+    return if !emoji.nil? || skip_download?
+
+    emoji = CustomEmoji.new(domain: @account.domain, shortcode: shortcode)
+    emoji.image_remote_url = tag['href']
+    emoji.save
+  end
+
   def process_attachments(status)
     return unless @object['attachment'].is_a?(Array)
 

app/lib/formatter.rb

@@ -9,7 +9,7 @@ class Formatter
 
   include ActionView::Helpers::TextHelper
 
-  def format(status)
+  def format(status, options = {})
     if status.reblog?
       prepend_reblog = status.reblog.account.acct
       status         = status.proper
@@ -19,7 +19,11 @@ class Formatter
 
     raw_content = status.text
 
-    return reformat(raw_content) unless status.local?
+    unless status.local?
+      html = reformat(raw_content)
+      html = encode_custom_emojis(html, status.emojis) if options[:custom_emojify]
+      return html
+    end
 
     linkable_accounts = status.mentions.map(&:account)
     linkable_accounts << status.account
@@ -27,6 +31,7 @@ class Formatter
     html = raw_content
     html = "RT @#{prepend_reblog} #{html}" if prepend_reblog
     html = encode_and_link_urls(html, linkable_accounts)
+    html = encode_custom_emojis(html, status.emojis) if options[:custom_emojify]
     html = simple_format(html, {}, sanitize: false)
     html = html.delete("\n")
 
@@ -39,7 +44,9 @@ class Formatter
 
   def plaintext(status)
     return status.text if status.local?
-    strip_tags(status.text)
+
+    text = status.text.gsub(/(<br \/>|<br>|<\/p>)+/) { |match| "#{match}\n" }
+    strip_tags(text)
   end
 
   def simplified_format(account)
@@ -76,6 +83,47 @@ class Formatter
     end
   end
 
+  def encode_custom_emojis(html, emojis)
+    return html if emojis.empty?
+
+    emoji_map = emojis.map { |e| [e.shortcode, full_asset_url(e.image.url)] }.to_h
+
+    i                     = -1
+    inside_tag            = false
+    inside_shortname      = false
+    shortname_start_index = -1
+
+    while i + 1 < html.size
+      i += 1
+
+      if inside_shortname && html[i] == ':'
+        shortcode = html[shortname_start_index + 1..i - 1]
+        emoji     = emoji_map[shortcode]
+
+        if emoji
+          replacement = "<img draggable=\"false\" class=\"emojione\" alt=\":#{shortcode}:\" title=\":#{shortcode}:\" src=\"#{emoji}\" />"
+          before_html = shortname_start_index.positive? ? html[0..shortname_start_index - 1] : ''
+          html        = before_html + replacement + html[i + 1..-1]
+          i          += replacement.size - (shortcode.size + 2) - 1
+        else
+          i -= 1
+        end
+
+        inside_shortname = false
+      elsif inside_tag && html[i] == '>'
+        inside_tag = false
+      elsif html[i] == '<'
+        inside_tag       = true
+        inside_shortname = false
+      elsif !inside_tag && html[i] == ':'
+        inside_shortname      = true
+        shortname_start_index = i
+      end
+    end
+
+    html
+  end
+
   def rewrite(text, entities)
     chars = text.to_s.to_char_a
 

app/lib/ostatus/activity/creation.rb

@@ -42,6 +42,7 @@ class OStatus::Activity::Creation < OStatus::Activity::Base
       save_mentions(status)
       save_hashtags(status)
       save_media(status)
+      save_emojis(status)
     end
 
     if thread? && status.thread.nil?
@@ -150,6 +151,25 @@ class OStatus::Activity::Creation < OStatus::Activity::Base
     end
   end
 
+  def save_emojis(parent)
+    do_not_download = DomainBlock.find_by(domain: parent.account.domain)&.reject_media?
+
+    return if do_not_download
+
+    @xml.xpath('./xmlns:link[@rel="emoji"]', xmlns: TagManager::XMLNS).each do |link|
+      next unless link['href'] && link['name']
+
+      shortcode = link['name'].delete(':')
+      emoji     = CustomEmoji.find_by(shortcode: shortcode, domain: parent.account.domain)
+
+      next unless emoji.nil?
+
+      emoji = CustomEmoji.new(shortcode: shortcode, domain: parent.account.domain)
+      emoji.image_remote_url = link['href']
+      emoji.save
+    end
+  end
+
   def account_from_href(href)
     url = Addressable::URI.parse(href).normalize
 

app/lib/ostatus/atom_serializer.rb

@@ -368,5 +368,9 @@ class OStatus::AtomSerializer
     end
 
     append_element(entry, 'mastodon:scope', status.visibility)
+
+    status.emojis.each do |emoji|
+      append_element(entry, 'link', nil, rel: :emoji, href: full_asset_url(emoji.image.url), name: emoji.shortcode)
+    end
   end
 end