From aff432db1ed612013244e4ef2891caa69702f674 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Tue, 20 Feb 2018 17:25:01 +0100 Subject: [PATCH 01/26] Set Docker permissions during the build process (#6514) * Set Docker permissions during the build process * Remove docker_entrypoint.sh and use COPY with chown --- Dockerfile | 16 +++++++++------- docker_entrypoint.sh | 14 -------------- 2 files changed, 9 insertions(+), 21 deletions(-) delete mode 100644 docker_entrypoint.sh diff --git a/Dockerfile b/Dockerfile index 765df58c25..628941dda3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,8 +3,10 @@ FROM ruby:2.5.0-alpine3.7 LABEL maintainer="https://github.com/tootsuite/mastodon" \ description="A GNU Social-compatible microblogging server" -ENV UID=991 GID=991 \ - RAILS_SERVE_STATIC_FILES=true \ +ARG UID=991 +ARG GID=991 + +ENV RAILS_SERVE_STATIC_FILES=true \ RAILS_ENV=production NODE_ENV=production ARG YARN_VERSION=1.3.2 @@ -68,12 +70,12 @@ RUN bundle config build.nokogiri --with-iconv-lib=/usr/local/lib --with-iconv-in && yarn --pure-lockfile \ && yarn cache clean -COPY . /mastodon - -COPY docker_entrypoint.sh /usr/local/bin/run +RUN addgroup -g ${GID} mastodon && adduser -h /mastodon -s /bin/sh -D -G mastodon -u ${UID} mastodon -RUN chmod +x /usr/local/bin/run +COPY --chown=${UID}:${GID} . /mastodon VOLUME /mastodon/public/system /mastodon/public/assets /mastodon/public/packs -ENTRYPOINT ["/usr/local/bin/run"] +USER mastodon + +ENTRYPOINT ["/sbin/tini", "--"] diff --git a/docker_entrypoint.sh b/docker_entrypoint.sh deleted file mode 100644 index e92959c8e4..0000000000 --- a/docker_entrypoint.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/sh - -### 1. Adds local user (UID and GID are provided from environment variables). -### 2. Updates permissions, except for ./public/system (should be chown on previous installations). -### 3. Executes the command as that user. - -echo "Creating mastodon user (UID : ${UID} and GID : ${GID})..." -addgroup -g ${GID} mastodon && adduser -h /mastodon -s /bin/sh -D -G mastodon -u ${UID} mastodon - -echo "Updating permissions..." -find /mastodon -path /mastodon/public/system -prune -o -not -user mastodon -not -group mastodon -print0 | xargs -0 chown -f mastodon:mastodon - -echo "Executing process..." -exec su-exec mastodon:mastodon /sbin/tini -- "$@" From 2815ef6d7f11f92b4885e1c2297fa427c0258749 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Tue, 20 Feb 2018 17:25:16 +0100 Subject: [PATCH 02/26] Fix #6509: Use pull queue for chewy jobs (#6513) --- config/initializers/chewy.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/config/initializers/chewy.rb b/config/initializers/chewy.rb index 702f7516c7..d5347f2bfd 100644 --- a/config/initializers/chewy.rb +++ b/config/initializers/chewy.rb @@ -9,6 +9,7 @@ Chewy.settings = { prefix: prefix, enabled: enabled, journal: false, + sidekiq: { queue: 'pull' }, } Chewy.root_strategy = enabled ? :sidekiq : :bypass From 0d1aeb0ff5afc37ac0f3c051226a234819f47d3a Mon Sep 17 00:00:00 2001 From: Moritz Heiber Date: Tue, 20 Feb 2018 19:11:36 +0100 Subject: [PATCH 03/26] Fix build error for missing variable interpolation in chown instruction (#6519) --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 628941dda3..dab4e87a64 100644 --- a/Dockerfile +++ b/Dockerfile @@ -72,7 +72,7 @@ RUN bundle config build.nokogiri --with-iconv-lib=/usr/local/lib --with-iconv-in RUN addgroup -g ${GID} mastodon && adduser -h /mastodon -s /bin/sh -D -G mastodon -u ${UID} mastodon -COPY --chown=${UID}:${GID} . /mastodon +COPY --chown=mastodon:mastodon . /mastodon VOLUME /mastodon/public/system /mastodon/public/assets /mastodon/public/packs From c34ff9fb825fa095bc330b7e38637e31cfc88e45 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Wed, 21 Feb 2018 03:40:12 +0100 Subject: [PATCH 04/26] Fix avatar and header issues by using custom geometry detector (#6515) * Fix avatar and header issues by using custom geometry detector Revert a part of #6508. The file passed to dynamic styles method was not actually a file, but an instance of Paperclip::Attachment, which broke all styles by always returning {} from the method. One problem with GIF avatars was that Paperclip::GeometryDetector reported wrong dimensions for them, e.g. 120x120 GIF avatar would for some reason be detected as 120x53. By writing our own geometry parser, we can use FastImage, which also happens to be faster than ImageMagick, to detect image dimensions, which are also correct. Unfortunately, this PR does not implement skipping a `convert` entirely if the dimensions are already correct, as I found no easy way to write that behaviour into Paperclip without rewriting the Paperclip::Thumbnail class. * Only invoke convert if dimension or format needs to be changed --- Gemfile | 1 + Gemfile.lock | 2 ++ app/lib/fast_geometry_parser.rb | 11 +++++++++++ app/models/concerns/account_avatar.rb | 12 +++--------- app/models/concerns/account_header.rb | 12 +++--------- app/models/media_attachment.rb | 27 +++++++++++++++++++-------- app/models/preview_card.rb | 13 +++++++------ app/models/site_upload.rb | 4 ++-- config/application.rb | 1 + lib/paperclip/lazy_thumbnail.rb | 24 ++++++++++++++++++++++++ 10 files changed, 73 insertions(+), 34 deletions(-) create mode 100644 app/lib/fast_geometry_parser.rb create mode 100644 lib/paperclip/lazy_thumbnail.rb diff --git a/Gemfile b/Gemfile index ef744064b2..ad1598af31 100644 --- a/Gemfile +++ b/Gemfile @@ -40,6 +40,7 @@ gem 'omniauth', '~> 1.2' gem 'doorkeeper', '~> 4.2' gem 'fast_blank', '~> 1.0' +gem 'fastimage' gem 'goldfinger', '~> 2.1' gem 'hiredis', '~> 0.6' gem 'redis-namespace', '~> 1.5' diff --git a/Gemfile.lock b/Gemfile.lock index 8e4edb8e12..920262eded 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -185,6 +185,7 @@ GEM faraday (0.14.0) multipart-post (>= 1.2, < 3) fast_blank (1.0.0) + fastimage (2.1.1) ffi (1.9.18) fog-core (1.45.0) builder @@ -641,6 +642,7 @@ DEPENDENCIES fabrication (~> 2.18) faker (~> 1.7) fast_blank (~> 1.0) + fastimage fog-core (~> 1.45) fog-local (~> 0.4) fog-openstack (~> 0.1) diff --git a/app/lib/fast_geometry_parser.rb b/app/lib/fast_geometry_parser.rb new file mode 100644 index 0000000000..5209c2bc59 --- /dev/null +++ b/app/lib/fast_geometry_parser.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +class FastGeometryParser + def self.from_file(file) + width, height = FastImage.size(file.path) + + raise Paperclip::Errors::NotIdentifiedByImageMagickError if width.nil? + + Paperclip::Geometry.new(width, height) + end +end diff --git a/app/models/concerns/account_avatar.rb b/app/models/concerns/account_avatar.rb index 53d0d876f5..619644c9a4 100644 --- a/app/models/concerns/account_avatar.rb +++ b/app/models/concerns/account_avatar.rb @@ -7,15 +7,9 @@ module AccountAvatar class_methods do def avatar_styles(file) - styles = {} - geometry = Paperclip::Geometry.from_file(file) - - styles[:original] = '120x120#' if geometry.width != geometry.height || geometry.width > 120 || geometry.height > 120 - styles[:static] = { format: 'png', convert_options: '-coalesce' } if file.content_type == 'image/gif' - + styles = { original: { geometry: '120x120#', file_geometry_parser: FastGeometryParser } } + styles[:static] = { format: 'png', convert_options: '-coalesce', file_geometry_parser: FastGeometryParser } if file.content_type == 'image/gif' styles - rescue Paperclip::Errors::NotIdentifiedByImageMagickError - {} end private :avatar_styles @@ -23,7 +17,7 @@ module AccountAvatar included do # Avatar upload - has_attached_file :avatar, styles: ->(f) { avatar_styles(f) }, convert_options: { all: '-strip' } + has_attached_file :avatar, styles: ->(f) { avatar_styles(f) }, convert_options: { all: '-strip' }, processors: [:lazy_thumbnail] validates_attachment_content_type :avatar, content_type: IMAGE_MIME_TYPES validates_attachment_size :avatar, less_than: 2.megabytes end diff --git a/app/models/concerns/account_header.rb b/app/models/concerns/account_header.rb index 991473d8c0..5ed8a9c832 100644 --- a/app/models/concerns/account_header.rb +++ b/app/models/concerns/account_header.rb @@ -7,15 +7,9 @@ module AccountHeader class_methods do def header_styles(file) - styles = {} - geometry = Paperclip::Geometry.from_file(file) - - styles[:original] = '700x335#' unless geometry.width == 700 && geometry.height == 335 - styles[:static] = { format: 'png', convert_options: '-coalesce' } if file.content_type == 'image/gif' - + styles = { original: { geometry: '700x335#', file_geometry_parser: FastGeometryParser } } + styles[:static] = { format: 'png', convert_options: '-coalesce', file_geometry_parser: FastGeometryParser } if file.content_type == 'image/gif' styles - rescue Paperclip::Errors::NotIdentifiedByImageMagickError - {} end private :header_styles @@ -23,7 +17,7 @@ module AccountHeader included do # Header upload - has_attached_file :header, styles: ->(f) { header_styles(f) }, convert_options: { all: '-strip' } + has_attached_file :header, styles: ->(f) { header_styles(f) }, convert_options: { all: '-strip' }, processors: [:lazy_thumbnail] validates_attachment_content_type :header, content_type: IMAGE_MIME_TYPES validates_attachment_size :header, less_than: 2.megabytes end diff --git a/app/models/media_attachment.rb b/app/models/media_attachment.rb index b6e5916cbe..38f88e9f7e 100644 --- a/app/models/media_attachment.rb +++ b/app/models/media_attachment.rb @@ -32,7 +32,18 @@ class MediaAttachment < ApplicationRecord IMAGE_MIME_TYPES = ['image/jpeg', 'image/png', 'image/gif'].freeze VIDEO_MIME_TYPES = ['video/webm', 'video/mp4'].freeze - IMAGE_STYLES = { original: '1280x1280>', small: '400x400>' }.freeze + IMAGE_STYLES = { + original: { + geometry: '1280x1280>', + file_geometry_parser: FastGeometryParser, + }, + + small: { + geometry: '400x400>', + file_geometry_parser: FastGeometryParser, + }, + }.freeze + VIDEO_STYLES = { small: { convert_options: { @@ -167,16 +178,16 @@ class MediaAttachment < ApplicationRecord end def image_geometry(file) - geo = Paperclip::Geometry.from_file file + width, height = FastImage.size(file.path) + + return {} if width.nil? { - width: geo.width.to_i, - height: geo.height.to_i, - size: "#{geo.width.to_i}x#{geo.height.to_i}", - aspect: geo.width.to_f / geo.height.to_f, + width: width, + height: height, + size: "#{width}x#{height}", + aspect: width.to_f / height.to_f, } - rescue Paperclip::Errors::NotIdentifiedByImageMagickError - {} end def video_metadata(file) diff --git a/app/models/preview_card.rb b/app/models/preview_card.rb index 716b822436..86eecdfe5b 100644 --- a/app/models/preview_card.rb +++ b/app/models/preview_card.rb @@ -33,7 +33,7 @@ class PreviewCard < ApplicationRecord has_and_belongs_to_many :statuses - has_attached_file :image, styles: { original: '400x400>' }, convert_options: { all: '-quality 80 -strip' } + has_attached_file :image, styles: { original: { geometry: '400x400>', file_geometry_parser: FastGeometryParser } }, convert_options: { all: '-quality 80 -strip' } include Attachmentable include Remotable @@ -58,10 +58,11 @@ class PreviewCard < ApplicationRecord return if file.nil? - geo = Paperclip::Geometry.from_file(file) - self.width = geo.width.to_i - self.height = geo.height.to_i - rescue Paperclip::Errors::NotIdentifiedByImageMagickError - nil + width, height = FastImage.size(file.path) + + return nil if width.nil? + + self.width = width + self.height = height end end diff --git a/app/models/site_upload.rb b/app/models/site_upload.rb index 8ffdc83131..641128adfc 100644 --- a/app/models/site_upload.rb +++ b/app/models/site_upload.rb @@ -34,8 +34,8 @@ class SiteUpload < ApplicationRecord return if tempfile.nil? - geometry = Paperclip::Geometry.from_file(tempfile) - self.meta = { width: geometry.width.to_i, height: geometry.height.to_i } + width, height = FastImage.size(tempfile.path) + self.meta = { width: width, height: height } end def clear_cache diff --git a/config/application.rb b/config/application.rb index 33981791e8..cd180782c3 100644 --- a/config/application.rb +++ b/config/application.rb @@ -7,6 +7,7 @@ require 'rails/all' Bundler.require(*Rails.groups) require_relative '../app/lib/exceptions' +require_relative '../lib/paperclip/lazy_thumbnail' require_relative '../lib/paperclip/gif_transcoder' require_relative '../lib/paperclip/video_transcoder' require_relative '../lib/mastodon/snowflake' diff --git a/lib/paperclip/lazy_thumbnail.rb b/lib/paperclip/lazy_thumbnail.rb new file mode 100644 index 0000000000..594f0ce39c --- /dev/null +++ b/lib/paperclip/lazy_thumbnail.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module Paperclip + class LazyThumbnail < Paperclip::Thumbnail + def make + return @file unless needs_convert? + Paperclip::Thumbnail.make(file, options, attachment) + end + + private + + def needs_convert? + needs_different_geometry? || needs_different_format? + end + + def needs_different_geometry? + !@target_geometry.nil? && @current_geometry.width != @target_geometry.width && @current_geometry.height != @target_geometry.height + end + + def needs_different_format? + @format.present? && @current_format != @format + end + end +end From 2b3d62a8a50034b04085ce125d1ebb653943b897 Mon Sep 17 00:00:00 2001 From: takayamaki Date: Thu, 22 Feb 2018 00:30:46 +0900 Subject: [PATCH 05/26] fix purge_removed_accounts task should suspend account before delete it (#6521) --- lib/tasks/mastodon.rake | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/tasks/mastodon.rake b/lib/tasks/mastodon.rake index e144621e57..d2e4f38a9a 100644 --- a/lib/tasks/mastodon.rake +++ b/lib/tasks/mastodon.rake @@ -752,6 +752,7 @@ namespace :mastodon do if [404, 410].include?(res.code) if options[:force] + SuspendAccountService.new.call(account) account.destroy else progress_bar.pause @@ -764,6 +765,7 @@ namespace :mastodon do if confirm.casecmp('n').zero? next else + SuspendAccountService.new.call(account) account.destroy end end From 9ff2739fb5abdd492daaca7d906827bdf2f5b04b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E3=82=8A=E3=82=93=E3=81=99=E3=81=8D?= <428rinsuki+git@gmail.com> Date: Thu, 22 Feb 2018 03:33:23 +0900 Subject: [PATCH 06/26] fix #6523 (#6524) --- app/javascript/mastodon/reducers/compose.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/javascript/mastodon/reducers/compose.js b/app/javascript/mastodon/reducers/compose.js index c709fb88c9..186134b1ab 100644 --- a/app/javascript/mastodon/reducers/compose.js +++ b/app/javascript/mastodon/reducers/compose.js @@ -34,7 +34,7 @@ import uuid from '../uuid'; import { me } from '../initial_state'; const initialState = ImmutableMap({ - mounted: false, + mounted: 0, sensitive: false, spoiler: false, spoiler_text: '', @@ -159,10 +159,10 @@ export default function compose(state = initialState, action) { case STORE_HYDRATE: return hydrate(state, action.state.get('compose')); case COMPOSE_MOUNT: - return state.set('mounted', true); + return state.set('mounted', state.get('mounted') + 1); case COMPOSE_UNMOUNT: return state - .set('mounted', false) + .set('mounted', Math.max(state.get('mounted') - 1, 0)) .set('is_composing', false); case COMPOSE_SENSITIVITY_CHANGE: return state.withMutations(map => { From f6884555d7337359b0ac1d69a996869b98f278d7 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Wed, 21 Feb 2018 23:21:32 +0100 Subject: [PATCH 07/26] Account archive download (#6460) * Fix #201: Account archive download * Export actor and private key in the archive * Optimize BackupService - Add conversation to cached associations of status, because somehow it was forgotten and is source of N+1 queries - Explicitly call GC between batches of records being fetched (Model class allocations are the worst offender) - Stream media files into the tar in 1MB chunks (Do not allocate media file (up to 8MB) as string into memory) - Use #bytesize instead of #size to calculate file size for JSON (Fix FileOverflow error) - Segment media into subfolders by status ID because apparently GIF-to-MP4 media are all named "media.mp4" for some reason * Keep uniquely generated filename in Paperclip::GifTranscoder * Ensure dumped files do not overwrite each other by maintaing directory partitions * Give tar archives a good name * Add scheduler to remove week-old backups * Fix code style issue --- Gemfile | 1 + Gemfile.lock | 2 + .../settings/exports_controller.rb | 14 +- app/javascript/images/icon_file_download.svg | 4 + .../images/mailer/icon_file_download.png | Bin 0 -> 271 bytes app/mailers/user_mailer.rb | 12 ++ app/models/backup.rb | 22 +++ app/models/status.rb | 2 +- app/models/user.rb | 1 + app/policies/application_policy.rb | 4 + app/policies/backup_policy.rb | 9 ++ .../activitypub/collection_serializer.rb | 4 +- app/services/backup_service.rb | 128 ++++++++++++++++++ app/views/settings/exports/show.html.haml | 23 ++++ app/views/user_mailer/backup_ready.html.haml | 59 ++++++++ app/views/user_mailer/backup_ready.text.erb | 7 + app/workers/backup_worker.rb | 17 +++ .../scheduler/backup_cleanup_scheduler.rb | 16 +++ config/locales/en.yml | 11 ++ config/routes.rb | 2 +- config/sidekiq.yml | 3 + db/migrate/20180211015820_create_backups.rb | 11 ++ db/schema.rb | 14 +- lib/paperclip/gif_transcoder.rb | 2 +- spec/fabricators/backup_fabricator.rb | 3 + spec/mailers/previews/user_mailer_preview.rb | 5 + spec/models/backup_spec.rb | 5 + 27 files changed, 374 insertions(+), 7 deletions(-) create mode 100644 app/javascript/images/icon_file_download.svg create mode 100644 app/javascript/images/mailer/icon_file_download.png create mode 100644 app/models/backup.rb create mode 100644 app/policies/backup_policy.rb create mode 100644 app/services/backup_service.rb create mode 100644 app/views/user_mailer/backup_ready.html.haml create mode 100644 app/views/user_mailer/backup_ready.text.erb create mode 100644 app/workers/backup_worker.rb create mode 100644 app/workers/scheduler/backup_cleanup_scheduler.rb create mode 100644 db/migrate/20180211015820_create_backups.rb create mode 100644 spec/fabricators/backup_fabricator.rb create mode 100644 spec/models/backup_spec.rb diff --git a/Gemfile b/Gemfile index ad1598af31..da5fc2f38b 100644 --- a/Gemfile +++ b/Gemfile @@ -116,6 +116,7 @@ group :development do gem 'bullet', '~> 5.5' gem 'letter_opener', '~> 1.4' gem 'letter_opener_web', '~> 1.3' + gem 'memory_profiler' gem 'rubocop', require: false gem 'brakeman', '~> 4.0', require: false gem 'bundler-audit', '~> 0.6', require: false diff --git a/Gemfile.lock b/Gemfile.lock index 920262eded..65a0dfabfd 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -301,6 +301,7 @@ GEM mini_mime (>= 0.1.1) mario-redis-lock (1.2.0) redis (~> 3, >= 3.0.5) + memory_profiler (0.9.10) method_source (0.9.0) microformats (4.0.7) json @@ -664,6 +665,7 @@ DEPENDENCIES link_header (~> 0.0) lograge (~> 0.7) mario-redis-lock (~> 1.2) + memory_profiler microformats (~> 4.0) mime-types (~> 3.1) nokogiri (~> 1.8) diff --git a/app/controllers/settings/exports_controller.rb b/app/controllers/settings/exports_controller.rb index ae62f00c1e..869e11d3bf 100644 --- a/app/controllers/settings/exports_controller.rb +++ b/app/controllers/settings/exports_controller.rb @@ -1,11 +1,23 @@ # frozen_string_literal: true class Settings::ExportsController < ApplicationController + include Authorization + layout 'admin' before_action :authenticate_user! def show - @export = Export.new(current_account) + @export = Export.new(current_account) + @backups = current_user.backups + end + + def create + authorize :backup, :create? + + backup = current_user.backups.create! + BackupWorker.perform_async(backup.id) + + redirect_to settings_export_path end end diff --git a/app/javascript/images/icon_file_download.svg b/app/javascript/images/icon_file_download.svg new file mode 100644 index 0000000000..53e97e4f8a --- /dev/null +++ b/app/javascript/images/icon_file_download.svg @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/app/javascript/images/mailer/icon_file_download.png b/app/javascript/images/mailer/icon_file_download.png new file mode 100644 index 0000000000000000000000000000000000000000..8a6a8673bcba80ab57027efa0ed7bd8453bcec50 GIT binary patch literal 271 zcmeAS@N?(olHy`uVBq!ia0vp^3LwnE1|*BCs=fdzmSQK*5Dp-y;YjHK@;M7UB8wRq zgi1h|@m`vI1yE43#5JNMI6tkVJh3R1p}f3YFEcN@I61K(RWH9NefB#WDWIYrPZ!6K zh}O4L5ArrB2(a)!K7ZuhnoC-LZg8~r#3!>@+L&CNy?%$Wd{f8g@}_U>4Q~=SR-N^e zPUvN?Xkb@iZeWbCC@`(C;a_tgB6NCJK!Cn^#}57df{E(uJ2p($R=RbPTj|zmW5M$$ z{gviTf3I}x`jh+YGq~Q>Xa=A9{44uL`aV~|nY}g_GmaO&xyjHee#I`iAbJVVT@0SC KelF{r5}E+IIAO8? literal 0 HcmV?d00001 diff --git a/app/mailers/user_mailer.rb b/app/mailers/user_mailer.rb index 2fc9caba3f..9848c34a25 100644 --- a/app/mailers/user_mailer.rb +++ b/app/mailers/user_mailer.rb @@ -66,4 +66,16 @@ class UserMailer < Devise::Mailer mail to: @resource.email, subject: I18n.t('user_mailer.welcome.subject') end end + + def backup_ready(user, backup) + @resource = user + @instance = Rails.configuration.x.local_domain + @backup = backup + + return if @resource.disabled? + + I18n.with_locale(@resource.locale || I18n.default_locale) do + mail to: @resource.email, subject: I18n.t('user_mailer.backup_ready.subject') + end + end end diff --git a/app/models/backup.rb b/app/models/backup.rb new file mode 100644 index 0000000000..5a7e6a14d5 --- /dev/null +++ b/app/models/backup.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true +# == Schema Information +# +# Table name: backups +# +# id :integer not null, primary key +# user_id :integer +# dump_file_name :string +# dump_content_type :string +# dump_file_size :integer +# dump_updated_at :datetime +# processed :boolean default(FALSE), not null +# created_at :datetime not null +# updated_at :datetime not null +# + +class Backup < ApplicationRecord + belongs_to :user, inverse_of: :backups + + has_attached_file :dump + do_not_validate_attachment_file_type :dump +end diff --git a/app/models/status.rb b/app/models/status.rb index 8186f47849..f806a59fcb 100644 --- a/app/models/status.rb +++ b/app/models/status.rb @@ -76,7 +76,7 @@ class Status < ApplicationRecord scope :not_excluded_by_account, ->(account) { where.not(account_id: account.excluded_from_timeline_account_ids) } scope :not_domain_blocked_by_account, ->(account) { account.excluded_from_timeline_domains.blank? ? left_outer_joins(:account) : left_outer_joins(:account).where('accounts.domain IS NULL OR accounts.domain NOT IN (?)', account.excluded_from_timeline_domains) } - cache_associated :account, :application, :media_attachments, :tags, :stream_entry, mentions: :account, reblog: [:account, :application, :stream_entry, :tags, :media_attachments, mentions: :account], thread: :account + cache_associated :account, :application, :media_attachments, :conversation, :tags, :stream_entry, mentions: :account, reblog: [:account, :application, :stream_entry, :tags, :media_attachments, :conversation, mentions: :account], thread: :account delegate :domain, to: :account, prefix: true diff --git a/app/models/user.rb b/app/models/user.rb index fd153912e6..fcd574f8bd 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -60,6 +60,7 @@ class User < ApplicationRecord accepts_nested_attributes_for :account has_many :applications, class_name: 'Doorkeeper::Application', as: :owner + has_many :backups, inverse_of: :user validates :locale, inclusion: I18n.available_locales.map(&:to_s), if: :locale? validates_with BlacklistedEmailValidator, if: :email_changed? diff --git a/app/policies/application_policy.rb b/app/policies/application_policy.rb index 3e617001fa..d1de5e81a2 100644 --- a/app/policies/application_policy.rb +++ b/app/policies/application_policy.rb @@ -15,4 +15,8 @@ class ApplicationPolicy def current_user current_account&.user end + + def user_signed_in? + !current_user.nil? + end end diff --git a/app/policies/backup_policy.rb b/app/policies/backup_policy.rb new file mode 100644 index 0000000000..0ef89a8d0c --- /dev/null +++ b/app/policies/backup_policy.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +class BackupPolicy < ApplicationPolicy + MIN_AGE = 1.week + + def create? + user_signed_in? && current_user.backups.where('created_at >= ?', MIN_AGE.ago).count.zero? + end +end diff --git a/app/serializers/activitypub/collection_serializer.rb b/app/serializers/activitypub/collection_serializer.rb index 9832133fc0..d43af3f8e3 100644 --- a/app/serializers/activitypub/collection_serializer.rb +++ b/app/serializers/activitypub/collection_serializer.rb @@ -13,8 +13,8 @@ class ActivityPub::CollectionSerializer < ActiveModel::Serializer attribute :part_of, if: -> { object.part_of.present? } has_one :first, if: -> { object.first.present? } - has_many :items, key: :items, if: -> { (object.items.present? || page?) && !ordered? } - has_many :items, key: :ordered_items, if: -> { (object.items.present? || page?) && ordered? } + has_many :items, key: :items, if: -> { (!object.items.nil? || page?) && !ordered? } + has_many :items, key: :ordered_items, if: -> { (!object.items.nil? || page?) && ordered? } def type if page? diff --git a/app/services/backup_service.rb b/app/services/backup_service.rb new file mode 100644 index 0000000000..fadc24a82c --- /dev/null +++ b/app/services/backup_service.rb @@ -0,0 +1,128 @@ +# frozen_string_literal: true + +require 'rubygems/package' + +class BackupService < BaseService + attr_reader :account, :backup, :collection + + def call(backup) + @backup = backup + @account = backup.user.account + + build_json! + build_archive! + end + + private + + def build_json! + @collection = serialize(collection_presenter, ActivityPub::CollectionSerializer) + + account.statuses.with_includes.find_in_batches do |statuses| + statuses.each do |status| + item = serialize(status, ActivityPub::ActivitySerializer) + item.delete(:'@context') + + unless item[:type] == 'Announce' || item[:object][:attachment].blank? + item[:object][:attachment].each do |attachment| + attachment[:url] = Addressable::URI.parse(attachment[:url]).path.gsub(/\A\/system\//, '') + end + end + + @collection[:orderedItems] << item + end + + GC.start + end + end + + def build_archive! + tmp_file = Tempfile.new(%w(archive .tar.gz)) + + File.open(tmp_file, 'wb') do |file| + Zlib::GzipWriter.wrap(file) do |gz| + Gem::Package::TarWriter.new(gz) do |tar| + dump_media_attachments!(tar) + dump_outbox!(tar) + dump_actor!(tar) + end + end + end + + archive_filename = ['archive', Time.now.utc.strftime('%Y%m%d%H%M%S'), SecureRandom.hex(2)].join('-') + '.tar.gz' + + @backup.dump = ActionDispatch::Http::UploadedFile.new(tempfile: tmp_file, filename: archive_filename) + @backup.processed = true + @backup.save! + ensure + tmp_file.close + tmp_file.unlink + end + + def dump_media_attachments!(tar) + MediaAttachment.attached.where(account: account).find_in_batches do |media_attachments| + media_attachments.each do |m| + download_to_tar(tar, m.file, m.file.path) + end + + GC.start + end + end + + def dump_outbox!(tar) + json = Oj.dump(collection) + + tar.add_file_simple('outbox.json', 0o444, json.bytesize) do |io| + io.write(json) + end + end + + def dump_actor!(tar) + actor = serialize(account, ActivityPub::ActorSerializer) + + actor[:icon][:url] = 'avatar' + File.extname(actor[:icon][:url]) if actor[:icon] + actor[:image][:url] = 'header' + File.extname(actor[:image][:url]) if actor[:image] + + download_to_tar(tar, account.avatar, 'avatar' + File.extname(account.avatar.path)) if account.avatar.exists? + download_to_tar(tar, account.header, 'header' + File.extname(account.header.path)) if account.header.exists? + + json = Oj.dump(actor) + + tar.add_file_simple('actor.json', 0o444, json.bytesize) do |io| + io.write(json) + end + + tar.add_file_simple('key.pem', 0o444, account.private_key.bytesize) do |io| + io.write(account.private_key) + end + end + + def collection_presenter + ActivityPub::CollectionPresenter.new( + id: account_outbox_url(account), + type: :ordered, + size: account.statuses_count, + items: [] + ) + end + + def serialize(object, serializer) + ActiveModelSerializers::SerializableResource.new( + object, + serializer: serializer, + adapter: ActivityPub::Adapter + ).as_json + end + + CHUNK_SIZE = 1.megabyte + + def download_to_tar(tar, attachment, filename) + adapter = Paperclip.io_adapters.for(attachment) + + tar.add_file_simple(filename, 0o444, adapter.size) do |io| + while (buffer = adapter.read(CHUNK_SIZE)) + io.write(buffer) + end + end + end +end diff --git a/app/views/settings/exports/show.html.haml b/app/views/settings/exports/show.html.haml index e0df1c4802..89d768d3fc 100644 --- a/app/views/settings/exports/show.html.haml +++ b/app/views/settings/exports/show.html.haml @@ -20,3 +20,26 @@ %th= t('exports.mutes') %td= @export.total_mutes %td= table_link_to 'download', t('exports.csv'), settings_exports_mutes_path(format: :csv) + +%p.muted-hint= t('exports.archive_takeout.hint_html') + +- if policy(:backup).create? + %p= link_to t('exports.archive_takeout.request'), settings_export_path, class: 'button', method: :post + +- unless @backups.empty? + .table-wrapper + %table.table + %thead + %tr + %th= t('exports.archive_takeout.date') + %th= t('exports.archive_takeout.size') + %th + %tbody + - @backups.each do |backup| + %tr + %td= l backup.created_at + - if backup.processed? + %td= number_to_human_size backup.dump_file_size + %td= table_link_to 'download', t('exports.archive_takeout.download'), backup.dump.url + - else + %td{ colspan: 2 }= t('exports.archive_takeout.in_progress') diff --git a/app/views/user_mailer/backup_ready.html.haml b/app/views/user_mailer/backup_ready.html.haml new file mode 100644 index 0000000000..d5a4b8b48b --- /dev/null +++ b/app/views/user_mailer/backup_ready.html.haml @@ -0,0 +1,59 @@ +%table.email-table{ cellspacing: 0, cellpadding: 0 } + %tbody + %tr + %td.email-body + .email-container + %table.content-section{ cellspacing: 0, cellpadding: 0 } + %tbody + %tr + %td.content-cell.hero + .email-row + .col-6 + %table.column{ cellspacing: 0, cellpadding: 0 } + %tbody + %tr + %td.column-cell.text-center.padded + %table.hero-icon{ align: 'center', cellspacing: 0, cellpadding: 0 } + %tbody + %tr + %td + = image_tag full_pack_url('icon_file_download.png'), alt: '' + + %h1= t 'user_mailer.backup_ready.title' + +%table.email-table{ cellspacing: 0, cellpadding: 0 } + %tbody + %tr + %td.email-body + .email-container + %table.content-section{ cellspacing: 0, cellpadding: 0 } + %tbody + %tr + %td.content-cell.content-start + .email-row + .col-6 + %table.column{ cellspacing: 0, cellpadding: 0 } + %tbody + %tr + %td.column-cell.text-center + %p= t 'user_mailer.backup_ready.explanation' + +%table.email-table{ cellspacing: 0, cellpadding: 0 } + %tbody + %tr + %td.email-body + .email-container + %table.content-section{ cellspacing: 0, cellpadding: 0 } + %tbody + %tr + %td.content-cell + %table.column{ cellspacing: 0, cellpadding: 0 } + %tbody + %tr + %td.column-cell.button-cell + %table.button{ align: 'center', cellspacing: 0, cellpadding: 0 } + %tbody + %tr + %td.button-primary + = link_to full_asset_url(@backup.dump.url) do + %span= t 'exports.archive_takeout.download' diff --git a/app/views/user_mailer/backup_ready.text.erb b/app/views/user_mailer/backup_ready.text.erb new file mode 100644 index 0000000000..eb89e7d743 --- /dev/null +++ b/app/views/user_mailer/backup_ready.text.erb @@ -0,0 +1,7 @@ +<%= t 'user_mailer.backup_ready.title' %> + +=== + +<%= t 'user_mailer.backup_ready.explanation' %> + +=> <%= full_asset_url(@backup.dump.url) %> diff --git a/app/workers/backup_worker.rb b/app/workers/backup_worker.rb new file mode 100644 index 0000000000..ec6db4e9e3 --- /dev/null +++ b/app/workers/backup_worker.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +class BackupWorker + include Sidekiq::Worker + + sidekiq_options queue: 'pull' + + def perform(backup_id) + backup = Backup.find(backup_id) + user = backup.user + + BackupService.new.call(backup) + + user.backups.where.not(id: backup.id).destroy_all + UserMailer.backup_ready(user, backup).deliver_later + end +end diff --git a/app/workers/scheduler/backup_cleanup_scheduler.rb b/app/workers/scheduler/backup_cleanup_scheduler.rb new file mode 100644 index 0000000000..7a9d4f894f --- /dev/null +++ b/app/workers/scheduler/backup_cleanup_scheduler.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true +require 'sidekiq-scheduler' + +class Scheduler::BackupCleanupScheduler + include Sidekiq::Worker + + def perform + old_backups.find_each(&:destroy!) + end + + private + + def old_backups + Backup.where('created_at < ?', 7.days.ago) + end +end diff --git a/config/locales/en.yml b/config/locales/en.yml index 5cd3b08cf0..b9dd5bd51f 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -421,6 +421,13 @@ en: title: This page is not correct noscript_html: To use the Mastodon web application, please enable JavaScript. Alternatively, try one of the native apps for Mastodon for your platform. exports: + archive_takeout: + date: Date + download: Download your archive + hint_html: You can request an archive of your toots and uploaded media. The exported data will be in ActivityPub format, readable by any compliant software. + in_progress: Compiling your archive... + request: Request your archive + size: Size blocks: You block csv: CSV follows: You follow @@ -733,6 +740,10 @@ en: setup: Set up wrong_code: The entered code was invalid! Are server time and device time correct? user_mailer: + backup_ready: + explanation: You requested a full backup of your Mastodon account. It's now ready for download! + subject: Your archive is ready for download + title: Archive takeout welcome: edit_profile_action: Setup profile edit_profile_step: You can customize your profile by uploading an avatar, header, changing your display name and more. If you’d like to review new followers before they’re allowed to follow you, you can lock your account. diff --git a/config/routes.rb b/config/routes.rb index 9f541200a2..7ed2d61f1b 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -76,7 +76,7 @@ Rails.application.routes.draw do resource :notifications, only: [:show, :update] resource :import, only: [:show, :create] - resource :export, only: [:show] + resource :export, only: [:show, :create] namespace :exports, constraints: { format: :csv } do resources :follows, only: :index, controller: :following_accounts resources :blocks, only: :index, controller: :blocked_accounts diff --git a/config/sidekiq.yml b/config/sidekiq.yml index bfe29b8f81..244e9ea48b 100644 --- a/config/sidekiq.yml +++ b/config/sidekiq.yml @@ -30,3 +30,6 @@ email_scheduler: cron: '0 10 * * 2' class: Scheduler::EmailScheduler + backup_cleanup_scheduler: + cron: '<%= Random.rand(0..59) %> <%= Random.rand(3..5) %> * * *' + class: Scheduler::BackupCleanupScheduler diff --git a/db/migrate/20180211015820_create_backups.rb b/db/migrate/20180211015820_create_backups.rb new file mode 100644 index 0000000000..9725a3e9f3 --- /dev/null +++ b/db/migrate/20180211015820_create_backups.rb @@ -0,0 +1,11 @@ +class CreateBackups < ActiveRecord::Migration[5.1] + def change + create_table :backups do |t| + t.references :user, foreign_key: { on_delete: :nullify } + t.attachment :dump + t.boolean :processed, null: false, default: false + + t.timestamps + end + end +end diff --git a/db/schema.rb b/db/schema.rb index 281110124e..213fbc8d9a 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20180206000000) do +ActiveRecord::Schema.define(version: 20180211015820) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -92,6 +92,18 @@ ActiveRecord::Schema.define(version: 20180206000000) do t.index ["target_type", "target_id"], name: "index_admin_action_logs_on_target_type_and_target_id" end + create_table "backups", force: :cascade do |t| + t.bigint "user_id" + t.string "dump_file_name" + t.string "dump_content_type" + t.integer "dump_file_size" + t.datetime "dump_updated_at" + t.boolean "processed", default: false, null: false + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index ["user_id"], name: "index_backups_on_user_id" + end + create_table "blocks", force: :cascade do |t| t.datetime "created_at", null: false t.datetime "updated_at", null: false diff --git a/lib/paperclip/gif_transcoder.rb b/lib/paperclip/gif_transcoder.rb index 629e37581e..62787983c4 100644 --- a/lib/paperclip/gif_transcoder.rb +++ b/lib/paperclip/gif_transcoder.rb @@ -16,7 +16,7 @@ module Paperclip final_file = Paperclip::Transcoder.make(file, options, attachment) - attachment.instance.file_file_name = 'media.mp4' + attachment.instance.file_file_name = File.basename(attachment.instance.file_file_name, '.*') + '.mp4' attachment.instance.file_content_type = 'video/mp4' attachment.instance.type = MediaAttachment.types[:gifv] diff --git a/spec/fabricators/backup_fabricator.rb b/spec/fabricators/backup_fabricator.rb new file mode 100644 index 0000000000..99a5bdcda1 --- /dev/null +++ b/spec/fabricators/backup_fabricator.rb @@ -0,0 +1,3 @@ +Fabricator(:backup) do + user +end diff --git a/spec/mailers/previews/user_mailer_preview.rb b/spec/mailers/previews/user_mailer_preview.rb index 8d2a9368df..d9cdb9264a 100644 --- a/spec/mailers/previews/user_mailer_preview.rb +++ b/spec/mailers/previews/user_mailer_preview.rb @@ -34,4 +34,9 @@ class UserMailerPreview < ActionMailer::Preview def welcome UserMailer.welcome(User.first) end + + # Preview this email at http://localhost:3000/rails/mailers/user_mailer/backup_ready + def backup_ready + UserMailer.backup_ready(User.first, Backup.first) + end end diff --git a/spec/models/backup_spec.rb b/spec/models/backup_spec.rb new file mode 100644 index 0000000000..fabcdc845a --- /dev/null +++ b/spec/models/backup_spec.rb @@ -0,0 +1,5 @@ +require 'rails_helper' + +RSpec.describe Backup, type: :model do + +end From cdb12bd39bd56f9e3a757af5f9caee1200a551ae Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Wed, 21 Feb 2018 23:22:12 +0100 Subject: [PATCH 08/26] Fix bug in relationships API introduced by #6482 (#6527) It was merge when it needed to be deep_merge. And added some tests --- .../v1/accounts/relationships_controller.rb | 2 +- .../account_relationships_presenter.rb | 2 +- .../accounts/relationships_controller_spec.rb | 22 +++++++++++++++++++ 3 files changed, 24 insertions(+), 2 deletions(-) diff --git a/app/controllers/api/v1/accounts/relationships_controller.rb b/app/controllers/api/v1/accounts/relationships_controller.rb index 6cc3da4985..70236d1a8b 100644 --- a/app/controllers/api/v1/accounts/relationships_controller.rb +++ b/app/controllers/api/v1/accounts/relationships_controller.rb @@ -21,6 +21,6 @@ class Api::V1::Accounts::RelationshipsController < Api::BaseController end def account_ids - @_account_ids ||= Array(params[:id]).map(&:to_i) + Array(params[:id]).map(&:to_i) end end diff --git a/app/presenters/account_relationships_presenter.rb b/app/presenters/account_relationships_presenter.rb index d27fb7b01d..b1e99b31b7 100644 --- a/app/presenters/account_relationships_presenter.rb +++ b/app/presenters/account_relationships_presenter.rb @@ -45,7 +45,7 @@ class AccountRelationshipsPresenter maps_for_account = Rails.cache.read("relationship:#{@current_account_id}:#{account_id}") if maps_for_account.is_a?(Hash) - @cached.merge!(maps_for_account) + @cached.deep_merge!(maps_for_account) else @uncached_account_ids << account_id end diff --git a/spec/controllers/api/v1/accounts/relationships_controller_spec.rb b/spec/controllers/api/v1/accounts/relationships_controller_spec.rb index 508415fc8c..e0de790c83 100644 --- a/spec/controllers/api/v1/accounts/relationships_controller_spec.rb +++ b/spec/controllers/api/v1/accounts/relationships_controller_spec.rb @@ -66,6 +66,28 @@ describe Api::V1::Accounts::RelationshipsController do expect(json.second[:requested]).to be false expect(json.second[:domain_blocking]).to be false end + + it 'returns JSON with correct data on cached requests too' do + get :index, params: { id: [simon.id] } + + json = body_as_json + + expect(json).to be_a Enumerable + expect(json.first[:following]).to be true + expect(json.first[:showing_reblogs]).to be true + end + + it 'returns JSON with correct data after change too' do + user.account.unfollow!(simon) + + get :index, params: { id: [simon.id] } + + json = body_as_json + + expect(json).to be_a Enumerable + expect(json.first[:following]).to be false + expect(json.first[:showing_reblogs]).to be false + end end end end From c9ed272a4a08589bb01dc4c13f2b5bc84fbbb834 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Thu, 22 Feb 2018 00:28:19 +0100 Subject: [PATCH 09/26] Fix #6525: Make sure file is opened in LazyThumbnail processor (#6529) --- app/models/concerns/account_avatar.rb | 2 +- app/models/concerns/account_header.rb | 2 +- lib/paperclip/lazy_thumbnail.rb | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/app/models/concerns/account_avatar.rb b/app/models/concerns/account_avatar.rb index 619644c9a4..7712a29fd0 100644 --- a/app/models/concerns/account_avatar.rb +++ b/app/models/concerns/account_avatar.rb @@ -8,7 +8,7 @@ module AccountAvatar class_methods do def avatar_styles(file) styles = { original: { geometry: '120x120#', file_geometry_parser: FastGeometryParser } } - styles[:static] = { format: 'png', convert_options: '-coalesce', file_geometry_parser: FastGeometryParser } if file.content_type == 'image/gif' + styles[:static] = { geometry: '120x120#', format: 'png', convert_options: '-coalesce', file_geometry_parser: FastGeometryParser } if file.content_type == 'image/gif' styles end diff --git a/app/models/concerns/account_header.rb b/app/models/concerns/account_header.rb index 5ed8a9c832..04c576b289 100644 --- a/app/models/concerns/account_header.rb +++ b/app/models/concerns/account_header.rb @@ -8,7 +8,7 @@ module AccountHeader class_methods do def header_styles(file) styles = { original: { geometry: '700x335#', file_geometry_parser: FastGeometryParser } } - styles[:static] = { format: 'png', convert_options: '-coalesce', file_geometry_parser: FastGeometryParser } if file.content_type == 'image/gif' + styles[:static] = { geometry: '700x335#', format: 'png', convert_options: '-coalesce', file_geometry_parser: FastGeometryParser } if file.content_type == 'image/gif' styles end diff --git a/lib/paperclip/lazy_thumbnail.rb b/lib/paperclip/lazy_thumbnail.rb index 594f0ce39c..42f9a557ac 100644 --- a/lib/paperclip/lazy_thumbnail.rb +++ b/lib/paperclip/lazy_thumbnail.rb @@ -3,7 +3,7 @@ module Paperclip class LazyThumbnail < Paperclip::Thumbnail def make - return @file unless needs_convert? + return File.open(@file.path) unless needs_convert? Paperclip::Thumbnail.make(file, options, attachment) end From 865c7e7178030223cf4cb5bd612799c425f70a79 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Thu, 22 Feb 2018 00:35:46 +0100 Subject: [PATCH 10/26] Focal points (#6520) * Add focus param to media API, center thumbnails on focus point * Add UI for setting a focal point * Improve focal point icon on upload item * Use focal point in upload preview * Add focalPoint property to ActivityPub * Don't show focal point button for non-image attachments --- app/controllers/api/v1/media_controller.rb | 2 +- app/javascript/images/reticle.png | Bin 0 -> 3053 bytes app/javascript/mastodon/actions/compose.js | 4 +- .../mastodon/components/media_gallery.js | 71 ++++++++-- .../features/compose/components/upload.js | 20 ++- .../compose/containers/upload_container.js | 7 +- .../ui/components/focal_point_modal.js | 122 ++++++++++++++++++ .../features/ui/components/modal_root.js | 2 + .../mastodon/features/video/index.js | 6 +- app/javascript/mastodon/reducers/compose.js | 2 +- .../styles/mastodon/components.scss | 69 +++++++++- app/lib/activitypub/activity/create.rb | 2 +- app/lib/activitypub/adapter.rb | 1 + app/models/media_attachment.rb | 20 ++- .../activitypub/image_serializer.rb | 9 ++ 15 files changed, 307 insertions(+), 30 deletions(-) create mode 100644 app/javascript/images/reticle.png create mode 100644 app/javascript/mastodon/features/ui/components/focal_point_modal.js diff --git a/app/controllers/api/v1/media_controller.rb b/app/controllers/api/v1/media_controller.rb index 9f330f0dfe..d4e6337e76 100644 --- a/app/controllers/api/v1/media_controller.rb +++ b/app/controllers/api/v1/media_controller.rb @@ -27,7 +27,7 @@ class Api::V1::MediaController < Api::BaseController private def media_params - params.permit(:file, :description) + params.permit(:file, :description, :focus) end def file_type_error diff --git a/app/javascript/images/reticle.png b/app/javascript/images/reticle.png new file mode 100644 index 0000000000000000000000000000000000000000..998994f5c00a37ee9b422af049ba1320e98f1ce9 GIT binary patch literal 3053 zcmbVOXIN8d77l_Mii*;UYY3odNF^a85F;d@tWp9IK^WvFxr87ogg~OKf(ob*P%Nlm zV`OPEG^JyU6j=r-gB1lru^}AY!wNkqQlgI8kLHfV~%#z*r!cC!oN`>+Zl|JPrlEi{OFr z5YfOe-nRW>(09Kllf8d0o5X=rH^ZD{WR(FvC;?zHeuN;3ETh1`*(Iy?i()h!_Dw~y zmjeHtl)r~Jj3yL=FanBzWMgnx7?Fg+5(q>h&K8ElVDV@S7LBz-Vo78?o{Yi3zF%;a zH!&xa>_d0^?n`x|z{4aG5gCn^N~I{N9ZD$XqOl|r35~&_aX6$(0~xhXAOU1ZL6qeW z3v@7wE#`?NJfQ%#Xc1ruqa_r$is|nm@I@XTe;5`-eNU7sWoQ{7LSs=FG@rj1*Ej7b zi4XW+8h@%C#oQ+X(LP|5Fj~x3)g#pM2UwN6zjm~!sA5C*67y6=0V3!^b~GOpNZjcZ zxatdv!{d-itWW|0V22{@2{;lG&vL*ZNqBn>l7k0WBsPl(fB^1?oj=3dIlDQ~u^2p# z?(Blay5b#hBs&+Pn~NQh?uy4axc$Jo3!)@|fDQie%~SdQj$-AX-HVz1S}Q(>^~$s-9fj{xJ-y z!yn@W3RE*BRt?MLw$2j}2vpmh?#z@ufBzFNg1OW9)`7pX)Q+TE>pS<<=PN37g2VJ~_AaT`O2u<J{h&qh(gYHe7RWKB$ zN~jX1tWg|Pd{L+=P)Y_+HQ}WMlqHIninR)1VqL=V(LBT&a*}*?FT6h3Aq`%oS;gqM zlDKwZIyk2D%$JSkw)Nh!vI(t&s4olJ%4I-GDaC!6b))Xh!;^{&3de*vU)y7q3>l{3 z3W4d69(sS~w&IaOnwT|Uzd=BIXr#K?Pj2{BUwoN@ylPe-=nyE|F`qm^h>?C!qp?CN zLFQh#OnjncmXJcCZCWNN zt+$yY6m9h_OS3qT(5^YBnb+*JIl;A?jUIWFcfLv2ws*5paKqALvKXH60?a?C1xJxu>P?sH>kC4~VjqOO$P3c#^Vd z{fEY+$=0_GAhfn%ccon5+xsSWC84{q)}n1Ay9s*@(+%LFc^{_L4z9|_j0SREAg58Y zO8MA`{h(m@t4)yJ?A6VuQhx49%-Z?2D6D@=pUncQ5H(`1+@#bCcMFcl_Pm}SXoVcw z_b@;Adhb$x)q8`@#+4Qx3StfMQ2{x2nWyXORq=W)D}d;LFfqClBwWLoXA97=Z)@vK zr=ZJo356x-Emdo~v7&Q{k2VZkXo@t=n>e~Xwa!tFii^{wWked(`V&^jxpFh3%8n9? zr_1g-Q(eH71(y$D&;RuGQB!KJtxRX%u_HB^t5=fqU$dx4jj|h#&&VtDGqSUj)fcSS z(=frshT@Wm0|ZNn`PJfSZJvd}`;mz&_Jv8=>UH;^0gbmqPMkBurQIns>4ek{;bduTuv%dn*{p%Yt&9#H>_FZHCl7L9=hX3M^h~wKT z%CB@$ZY3^QzV~hDt7I+UoX6v0<4zx>4mo6bVS7J@*><5Becgbfg>ctqfT-bkOMR#1 zb>s83-48h1&W3Xgj$PV(u}wE3tGm$6Z}QG)p@jC~JakQ=jreEsISU6_iEnBVOYW{| zqaxXs(e}YQ&l>Lh_RLRm196lbi_>U>?kYCSk4cBr-VVn(rDj)&M(@ud zgBAu26w8_Sz11jvn1J35lbJ1QRj!WA*1Hd~TJG&F)(c|D{AYFKcWXT3fuQ;zF5b*r zH{{nX&#CXn##%p|s2^%|*_3T9*|oYT+O`Eu+C_`cBlTov*M3?eW~Sw)k(3)=!&jE3 zeca_F}!BEO0U?aMU?Ide6ZLr0qTY7YNn?MkK7 zI+0R(V)EmAMu$<1=fqJqp-=H@ckxT6cJu__%{XZ$6ZfS0^G{Ho2_s2eh8Up|kC^}_ zOD=M*e9VYczhnBZu<*JIv%wsNY?6Gq*)UIYm6lP){Iyfs{_Q;Apum6C_FVPlf#b$R z>EMCfo0qHa_Uf1W4pM>=rV=uftqPluGq%zF`m-B_JKokmu)E~@e2+Y9fG|^!VN{%C z;x+dSB^%_WIi_?#%~47xHa>9SQ~tq|smsS0^N7Sfrqc0|fYYZUtd+(MQv%>)bLO#W z203F>(&uq6)43Q!L+?KAE4Y?hA-fx={vGndZi1ZgRn|>?S{B5(s2##y!!UZSS!G7- z4)f6gTXGt$GPsVd(zB67=r4``d}d6Wiq@Q#XU(ij@82f8{o<*|fP7G+e!?jW30a+W zY?G7uQCOszxX}D!MTqIXyh!4vGJ{DIqf8*wy^(-(vKJkR+#_pBf_cvB&ddFK{Fk~X zTfBW0vL^oZ-3>nV*DeYwqY7(|u8;1jM-`u1JBDx`I;4Lma|A9|pEcJD!3EpQ`&SbL zdZz^2)*lHc-8b=wEVD6l=|u1j(|l{6&fYDe1!<4Is*Wdg7F(Hk%ahI+n4g6O%zMSF ZL7pthq3q3=v{?KBcX#!q7rTTU{ttCeA~OI0 literal 0 HcmV?d00001 diff --git a/app/javascript/mastodon/actions/compose.js b/app/javascript/mastodon/actions/compose.js index 8a35049b32..1732ff189e 100644 --- a/app/javascript/mastodon/actions/compose.js +++ b/app/javascript/mastodon/actions/compose.js @@ -178,11 +178,11 @@ export function uploadCompose(files) { }; }; -export function changeUploadCompose(id, description) { +export function changeUploadCompose(id, params) { return (dispatch, getState) => { dispatch(changeUploadComposeRequest()); - api(getState).put(`/api/v1/media/${id}`, { description }).then(response => { + api(getState).put(`/api/v1/media/${id}`, params).then(response => { dispatch(changeUploadComposeSuccess(response.data)); }).catch(error => { dispatch(changeUploadComposeFail(id, error)); diff --git a/app/javascript/mastodon/components/media_gallery.js b/app/javascript/mastodon/components/media_gallery.js index a3ffc45eae..9e1bb77c2c 100644 --- a/app/javascript/mastodon/components/media_gallery.js +++ b/app/javascript/mastodon/components/media_gallery.js @@ -12,6 +12,26 @@ const messages = defineMessages({ toggle_visible: { id: 'media_gallery.toggle_visible', defaultMessage: 'Toggle visibility' }, }); +const shiftToPoint = (containerToImageRatio, containerSize, imageSize, focusSize, toMinus) => { + const containerCenter = Math.floor(containerSize / 2); + const focusFactor = (focusSize + 1) / 2; + const scaledImage = Math.floor(imageSize / containerToImageRatio); + + let focus = Math.floor(focusFactor * scaledImage); + + if (toMinus) focus = scaledImage - focus; + + let focusOffset = focus - containerCenter; + + const remainder = scaledImage - focus; + const containerRemainder = containerSize - containerCenter; + + if (remainder < containerRemainder) focusOffset -= containerRemainder - remainder; + if (focusOffset < 0) focusOffset = 0; + + return (focusOffset * -100 / containerSize) + '%'; +}; + class Item extends React.PureComponent { static contextTypes = { @@ -24,6 +44,8 @@ class Item extends React.PureComponent { index: PropTypes.number.isRequired, size: PropTypes.number.isRequired, onClick: PropTypes.func.isRequired, + containerWidth: PropTypes.number, + containerHeight: PropTypes.number, }; static defaultProps = { @@ -62,7 +84,7 @@ class Item extends React.PureComponent { } render () { - const { attachment, index, size, standalone } = this.props; + const { attachment, index, size, standalone, containerWidth, containerHeight } = this.props; let width = 50; let height = 100; @@ -116,16 +138,40 @@ class Item extends React.PureComponent { let thumbnail = ''; if (attachment.get('type') === 'image') { - const previewUrl = attachment.get('preview_url'); + const previewUrl = attachment.get('preview_url'); const previewWidth = attachment.getIn(['meta', 'small', 'width']); - const originalUrl = attachment.get('url'); - const originalWidth = attachment.getIn(['meta', 'original', 'width']); + const originalUrl = attachment.get('url'); + const originalWidth = attachment.getIn(['meta', 'original', 'width']); + const originalHeight = attachment.getIn(['meta', 'original', 'height']); const hasSize = typeof originalWidth === 'number' && typeof previewWidth === 'number'; const srcSet = hasSize ? `${originalUrl} ${originalWidth}w, ${previewUrl} ${previewWidth}w` : null; - const sizes = hasSize ? `(min-width: 1025px) ${320 * (width / 100)}px, ${width}vw` : null; + const sizes = hasSize ? `(min-width: 1025px) ${320 * (width / 100)}px, ${width}vw` : null; + + const focusX = attachment.getIn(['meta', 'focus', 'x']); + const focusY = attachment.getIn(['meta', 'focus', 'y']); + const imageStyle = {}; + + if (originalWidth && originalHeight && containerWidth && containerHeight && focusX && focusY) { + const widthRatio = originalWidth / (containerWidth * (width / 100)); + const heightRatio = originalHeight / (containerHeight * (height / 100)); + + let hShift = 0; + let vShift = 0; + + if (widthRatio > heightRatio) { + hShift = shiftToPoint(heightRatio, (containerWidth * (width / 100)), originalWidth, focusX); + } else if(widthRatio < heightRatio) { + vShift = shiftToPoint(widthRatio, (containerHeight * (height / 100)), originalHeight, focusY, true); + } + + imageStyle.top = vShift; + imageStyle.left = hShift; + } else { + imageStyle.height = '100%'; + } thumbnail = ( - {attachment.get('description')} + {attachment.get('description')} ); } else if (attachment.get('type') === 'gifv') { @@ -205,7 +258,7 @@ export default class MediaGallery extends React.PureComponent { } handleRef = (node) => { - if (node && this.isStandaloneEligible()) { + if (node /*&& this.isStandaloneEligible()*/) { // offsetWidth triggers a layout, so only calculate when we need to this.setState({ width: node.offsetWidth, @@ -256,12 +309,12 @@ export default class MediaGallery extends React.PureComponent { if (this.isStandaloneEligible()) { children = ; } else { - children = media.take(4).map((attachment, i) => ); + children = media.take(4).map((attachment, i) => ); } } return ( -
+
diff --git a/app/javascript/mastodon/features/compose/components/upload.js b/app/javascript/mastodon/features/compose/components/upload.js index 3a3d177100..61b2d19e0d 100644 --- a/app/javascript/mastodon/features/compose/components/upload.js +++ b/app/javascript/mastodon/features/compose/components/upload.js @@ -1,15 +1,13 @@ import React from 'react'; import ImmutablePropTypes from 'react-immutable-proptypes'; import PropTypes from 'prop-types'; -import IconButton from '../../../components/icon_button'; import Motion from '../../ui/util/optional_motion'; import spring from 'react-motion/lib/spring'; import ImmutablePureComponent from 'react-immutable-pure-component'; -import { defineMessages, injectIntl } from 'react-intl'; +import { defineMessages, injectIntl, FormattedMessage } from 'react-intl'; import classNames from 'classnames'; const messages = defineMessages({ - undo: { id: 'upload_form.undo', defaultMessage: 'Undo' }, description: { id: 'upload_form.description', defaultMessage: 'Describe for the visually impaired' }, }); @@ -21,6 +19,7 @@ export default class Upload extends ImmutablePureComponent { intl: PropTypes.object.isRequired, onUndo: PropTypes.func.isRequired, onDescriptionChange: PropTypes.func.isRequired, + onOpenFocalPoint: PropTypes.func.isRequired, }; state = { @@ -33,6 +32,10 @@ export default class Upload extends ImmutablePureComponent { this.props.onUndo(this.props.media.get('id')); } + handleFocalPointClick = () => { + this.props.onOpenFocalPoint(this.props.media.get('id')); + } + handleInputChange = e => { this.setState({ dirtyDescription: e.target.value }); } @@ -63,13 +66,20 @@ export default class Upload extends ImmutablePureComponent { const { intl, media } = this.props; const active = this.state.hovered || this.state.focused; const description = this.state.dirtyDescription || (this.state.dirtyDescription !== '' && media.get('description')) || ''; + const focusX = media.getIn(['meta', 'focus', 'x']); + const focusY = media.getIn(['meta', 'focus', 'y']); + const x = ((focusX / 2) + .5) * 100; + const y = ((focusY / -2) + .5) * 100; return (
{({ scale }) => ( -
- +
+
+ + {media.get('type') === 'image' && } +