From 76b0f84cd5e429f0d2b721fc7e8c5e0737a32d37 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Tue, 15 Dec 2020 12:55:29 +0100
Subject: [PATCH] Add stoplight for object storage failures, return HTTP 503
 (#13043)

---
 app/controllers/api/base_controller.rb    |  2 +-
 app/controllers/application_controller.rb |  2 +-
 app/lib/activitypub/activity/create.rb    |  4 ++++
 config/initializers/paperclip.rb          | 11 +++++++++++
 lib/paperclip/attachment_extensions.rb    | 17 +++++++++++++++++
 5 files changed, 34 insertions(+), 2 deletions(-)

diff --git a/app/controllers/api/base_controller.rb b/app/controllers/api/base_controller.rb
index fe199e689a..85f4cc7681 100644
--- a/app/controllers/api/base_controller.rb
+++ b/app/controllers/api/base_controller.rb
@@ -40,7 +40,7 @@ class Api::BaseController < ApplicationController
     render json: { error: 'This action is not allowed' }, status: 403
   end
 
-  rescue_from Mastodon::RaceConditionError do
+  rescue_from Mastodon::RaceConditionError, Seahorse::Client::NetworkingError, Stoplight::Error::RedLight do
     render json: { error: 'There was a temporary problem serving your request, please try again' }, status: 503
   end
 
diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb
index 2201e463e6..44616d6e5e 100644
--- a/app/controllers/application_controller.rb
+++ b/app/controllers/application_controller.rb
@@ -28,7 +28,7 @@ class ApplicationController < ActionController::Base
   rescue_from ActiveRecord::RecordNotFound, with: :not_found
   rescue_from Mastodon::NotPermittedError, with: :forbidden
   rescue_from HTTP::Error, OpenSSL::SSL::SSLError, with: :internal_server_error
-  rescue_from Mastodon::RaceConditionError, with: :service_unavailable
+  rescue_from Mastodon::RaceConditionError, Seahorse::Client::NetworkingError, Stoplight::Error::RedLight, with: :service_unavailable
   rescue_from Mastodon::RateLimitExceededError, with: :too_many_requests
 
   before_action :store_current_location, except: :raise_not_found, unless: :devise_controller?
diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb
index c77f237f9a..6127446763 100644
--- a/app/lib/activitypub/activity/create.rb
+++ b/app/lib/activitypub/activity/create.rb
@@ -228,6 +228,8 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
     emoji ||= CustomEmoji.new(domain: @account.domain, shortcode: shortcode, uri: uri)
     emoji.image_remote_url = image_url
     emoji.save
+  rescue Seahorse::Client::NetworkingError
+    nil
   end
 
   def process_attachments
@@ -250,6 +252,8 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
         media_attachment.save
       rescue Mastodon::UnexpectedResponseError, HTTP::TimeoutError, HTTP::ConnectionError, OpenSSL::SSL::SSLError
         RedownloadMediaWorker.perform_in(rand(30..600).seconds, media_attachment.id)
+      rescue Seahorse::Client::NetworkingError
+        nil
       end
     end
 
diff --git a/config/initializers/paperclip.rb b/config/initializers/paperclip.rb
index 25adcd8d63..9ad7fd814c 100644
--- a/config/initializers/paperclip.rb
+++ b/config/initializers/paperclip.rb
@@ -113,3 +113,14 @@ else
 end
 
 Paperclip.options[:content_type_mappings] = { csv: Import::FILE_TYPES }
+
+# In some places in the code, we rescue this exception, but we don't always
+# load the S3 library, so it may be an undefined constant:
+
+unless defined?(Seahorse)
+  module Seahorse
+    module Client
+      class NetworkingError < StandardError; end
+    end
+  end
+end
diff --git a/lib/paperclip/attachment_extensions.rb b/lib/paperclip/attachment_extensions.rb
index 752e79e65e..94f7769b65 100644
--- a/lib/paperclip/attachment_extensions.rb
+++ b/lib/paperclip/attachment_extensions.rb
@@ -39,6 +39,23 @@ module Paperclip
     def default_url(style_name = default_style)
       @url_generator.for_as_default(style_name)
     end
+
+    STOPLIGHT_THRESHOLD = 10
+    STOPLIGHT_COOLDOWN  = 30
+
+    # We overwrite this method to put a circuit breaker around
+    # calls to object storage, to stop hitting APIs that are slow
+    # to respond or don't respond at all and as such minimize the
+    # impact of object storage outages on application throughput
+    def save
+      Stoplight('object-storage') { super }.with_threshold(STOPLIGHT_THRESHOLD).with_cool_off_time(STOPLIGHT_COOLDOWN).with_error_handler do |error, handle|
+        if error.is_a?(Seahorse::Client::NetworkingError)
+          handle.call(error)
+        else
+          raise error
+        end
+      end.run
+    end
   end
 end