Add customizable thumbnails for audio and video attachments (#14145)

- Change audio files to not be stripped of metadata
- Automatically extract cover art from audio if it exists
- Add `thumbnail` parameter to `POST /api/v1/media`, `POST /api/v2/media` and `PUT /api/v1/media/:id`
- Add `icon` to represent it in attachments in ActivityPub
- Fix `preview_url` containing URL of missing missing image when there is no thumbnail instead of null
- Fix duration of audio not being displayed on public pages until the file is loaded

lib/mastodon/media_cli.rb

@@ -31,10 +31,11 @@ module Mastodon
       processed, aggregate = parallelize_with_progress(MediaAttachment.cached.where.not(remote_url: '').where('created_at < ?', time_ago)) do |media_attachment|
         next if media_attachment.file.blank?
 
-        size = media_attachment.file_file_size
+        size = media_attachment.file_file_size + (media_attachment.thumbnail_file_size || 0)
 
         unless options[:dry_run]
           media_attachment.file.destroy
+          media_attachment.thumbnail.destroy
           media_attachment.save
         end
 
@@ -227,11 +228,12 @@ module Mastodon
         next if media_attachment.remote_url.blank? || (!options[:force] && media_attachment.file_file_name.present?)
 
         unless options[:dry_run]
-          media_attachment.file_remote_url = media_attachment.remote_url
+          media_attachment.reset_file!
+          media_attachment.reset_thumbnail!
           media_attachment.save
         end
 
-        media_attachment.file_file_size
+        media_attachment.file_file_size + (media_attachment.thumbnail_file_size || 0)
       end
 
       say("Downloaded #{processed} media attachments (approx. #{number_to_human_size(aggregate)})#{dry_run}", :green, true)
@@ -239,7 +241,7 @@ module Mastodon
 
     desc 'usage', 'Calculate disk space consumed by Mastodon'
     def usage
-      say("Attachments:\t#{number_to_human_size(MediaAttachment.sum(:file_file_size))} (#{number_to_human_size(MediaAttachment.where(account: Account.local).sum(:file_file_size))} local)")
+      say("Attachments:\t#{number_to_human_size(MediaAttachment.sum(Arel.sql('COALESCE(file_file_size, 0) + COALESCE(thumbnail_file_size, 0)')))} (#{number_to_human_size(MediaAttachment.where(account: Account.local).sum(Arel.sql('COALESCE(file_file_size, 0) + COALESCE(thumbnail_file_size, 0)')))} local)")
       say("Custom emoji:\t#{number_to_human_size(CustomEmoji.sum(:image_file_size))} (#{number_to_human_size(CustomEmoji.local.sum(:image_file_size))} local)")
       say("Preview cards:\t#{number_to_human_size(PreviewCard.sum(:image_file_size))}")
       say("Avatars:\t#{number_to_human_size(Account.sum(:avatar_file_size))} (#{number_to_human_size(Account.local.sum(:avatar_file_size))} local)")

lib/paperclip/attachment_extensions.rb

@@ -7,7 +7,7 @@ module Paperclip
     # usage, and we still want to generate thumbnails straight
     # away, it's the only style we need to exclude
     def process_style?(style_name, style_args)
-      if style_name == :original && instance.respond_to?(:delay_processing?) && instance.delay_processing?
+      if style_name == :original && instance.respond_to?(:delay_processing_for_attachment?) && instance.delay_processing_for_attachment?(name)
         false
       else
         style_args.empty? || style_args.include?(style_name)

lib/paperclip/image_extractor.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+require 'mime/types/columnar'
+
+module Paperclip
+  class ImageExtractor < Paperclip::Processor
+    IMAGE_EXTRACTION_OPTIONS = {
+      convert_options: {
+        output: {
+          'loglevel' => 'fatal',
+          vf: 'scale=\'min(400\, iw):min(400\, ih)\':force_original_aspect_ratio=decrease',
+        }.freeze,
+      }.freeze,
+      format: 'png',
+      time: -1,
+      file_geometry_parser: FastGeometryParser,
+    }.freeze
+
+    def make
+      return @file unless options[:style] == :original
+
+      image = begin
+        begin
+          Paperclip::Transcoder.make(file, IMAGE_EXTRACTION_OPTIONS.dup, attachment)
+        rescue Paperclip::Error, ::Av::CommandError
+          nil
+        end
+      end
+
+      unless image.nil?
+        begin
+          attachment.instance.thumbnail = image if image.size.positive?
+        ensure
+          # Paperclip does not automatically delete the source file of
+          # a new attachment while working on copies of it, so we need
+          # to make sure it's cleaned up
+
+          begin
+            FileUtils.rm(image)
+          rescue Errno::ENOENT
+            nil
+          end
+        end
+      end
+
+      @file
+    end
+  end
+end

lib/paperclip/type_corrector.rb

@@ -5,13 +5,15 @@ require 'mime/types/columnar'
 module Paperclip
   class TypeCorrector < Paperclip::Processor
     def make
-      target_extension = options[:format]
-      extension        = File.extname(attachment.instance.file_file_name)
+      return @file unless options[:format]
+
+      target_extension = '.' + options[:format]
+      extension        = File.extname(attachment.instance_read(:file_name))
 
       return @file unless options[:style] == :original && target_extension && extension != target_extension
 
-      attachment.instance.file_content_type = options[:content_type] || attachment.instance.file_content_type
-      attachment.instance.file_file_name    = File.basename(attachment.instance.file_file_name, '.*') + '.' + target_extension
+      attachment.instance_write(:content_type, options[:content_type] || attachment.instance_read(:content_type))
+      attachment.instance_write(:file_name, File.basename(attachment.instance_read(:file_name), '.*') + target_extension)
 
       @file
     end