From 2d3219549b18b845917e4772465f74377d157a44 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Wed, 15 Apr 2020 20:33:24 +0200
Subject: [PATCH] Change delivery failure tracking to work with hostnames
 instead of URLs (#13437)

---
 .../activitypub/inboxes_controller.rb         |  2 +-
 app/controllers/admin/instances_controller.rb |  2 +-
 app/lib/delivery_failure_tracker.rb           | 34 +++++++++++--------
 app/models/account.rb                         |  2 +-
 app/models/announcement.rb                    |  2 +-
 app/models/relay.rb                           |  4 +--
 app/models/unavailable_domain.rb              | 22 ++++++++++++
 app/views/admin/accounts/show.html.haml       |  4 +--
 app/workers/activitypub/delivery_worker.rb    |  2 +-
 ...200407201300_create_unavailable_domains.rb |  9 +++++
 ...00407202420_migrate_unavailable_inboxes.rb | 16 +++++++++
 db/schema.rb                                  |  9 ++++-
 .../unavailable_domain_fabricator.rb          |  3 ++
 spec/lib/delivery_failure_tracker_spec.rb     | 30 ++++++----------
 spec/models/unavailable_domain_spec.rb        |  4 +++
 15 files changed, 102 insertions(+), 43 deletions(-)
 create mode 100644 app/models/unavailable_domain.rb
 create mode 100644 db/migrate/20200407201300_create_unavailable_domains.rb
 create mode 100644 db/migrate/20200407202420_migrate_unavailable_inboxes.rb
 create mode 100644 spec/fabricators/unavailable_domain_fabricator.rb
 create mode 100644 spec/models/unavailable_domain_spec.rb

diff --git a/app/controllers/activitypub/inboxes_controller.rb b/app/controllers/activitypub/inboxes_controller.rb
index 291eec19a8..0a561e7f0f 100644
--- a/app/controllers/activitypub/inboxes_controller.rb
+++ b/app/controllers/activitypub/inboxes_controller.rb
@@ -49,7 +49,7 @@ class ActivityPub::InboxesController < ActivityPub::BaseController
       ResolveAccountWorker.perform_async(signed_request_account.acct)
     end
 
-    DeliveryFailureTracker.track_inverse_success!(signed_request_account)
+    DeliveryFailureTracker.reset!(signed_request_account.inbox_url)
   end
 
   def process_payload
diff --git a/app/controllers/admin/instances_controller.rb b/app/controllers/admin/instances_controller.rb
index 2fc0412075..1790becbf2 100644
--- a/app/controllers/admin/instances_controller.rb
+++ b/app/controllers/admin/instances_controller.rb
@@ -19,7 +19,7 @@ module Admin
       @followers_count = Follow.where(target_account: Account.where(domain: params[:id])).count
       @reports_count   = Report.where(target_account: Account.where(domain: params[:id])).count
       @blocks_count    = Block.where(target_account: Account.where(domain: params[:id])).count
-      @available       = DeliveryFailureTracker.available?(Account.select(:shared_inbox_url).where(domain: params[:id]).first&.shared_inbox_url)
+      @available       = DeliveryFailureTracker.available?(params[:id])
       @media_storage   = MediaAttachment.where(account: Account.where(domain: params[:id])).sum(:file_file_size)
       @private_comment = @domain_block&.private_comment
       @public_comment  = @domain_block&.public_comment
diff --git a/app/lib/delivery_failure_tracker.rb b/app/lib/delivery_failure_tracker.rb
index 8d3be35def..25fa694d22 100644
--- a/app/lib/delivery_failure_tracker.rb
+++ b/app/lib/delivery_failure_tracker.rb
@@ -3,47 +3,53 @@
 class DeliveryFailureTracker
   FAILURE_DAYS_THRESHOLD = 7
 
-  def initialize(inbox_url)
-    @inbox_url = inbox_url
+  def initialize(url_or_host)
+    @host = url_or_host.start_with?('https://') || url_or_host.start_with?('http://') ? Addressable::URI.parse(url_or_host).normalized_host : url_or_host
   end
 
   def track_failure!
     Redis.current.sadd(exhausted_deliveries_key, today)
-    Redis.current.sadd('unavailable_inboxes', @inbox_url) if reached_failure_threshold?
+    UnavailableDomain.create(domain: @host) if reached_failure_threshold?
   end
 
   def track_success!
     Redis.current.del(exhausted_deliveries_key)
-    Redis.current.srem('unavailable_inboxes', @inbox_url)
+    UnavailableDomain.find_by(domain: @host)&.destroy
   end
 
   def days
     Redis.current.scard(exhausted_deliveries_key) || 0
   end
 
+  def available?
+    !UnavailableDomain.where(domain: @host).exists?
+  end
+
+  alias reset! track_success!
+
   class << self
-    def filter(arr)
-      arr.reject(&method(:unavailable?))
-    end
+    def without_unavailable(urls)
+      unavailable_domains_map = Rails.cache.fetch('unavailable_domains') { UnavailableDomain.pluck(:domain).each_with_object({}) { |domain, hash| hash[domain] = true } }
 
-    def unavailable?(url)
-      Redis.current.sismember('unavailable_inboxes', url)
+      urls.reject do |url|
+        host = Addressable::URI.parse(url).normalized_host
+        unavailable_domains_map[host]
+      end
     end
 
     def available?(url)
-      !unavailable?(url)
+      new(url).available?
     end
 
-    def track_inverse_success!(from_account)
-      new(from_account.inbox_url).track_success! if from_account.inbox_url.present?
-      new(from_account.shared_inbox_url).track_success! if from_account.shared_inbox_url.present?
+    def reset!(url)
+      new(url).reset!
     end
   end
 
   private
 
   def exhausted_deliveries_key
-    "exhausted_deliveries:#{@inbox_url}"
+    "exhausted_deliveries:#{@host}"
   end
 
   def today
diff --git a/app/models/account.rb b/app/models/account.rb
index 6aceb809cd..dc14e85386 100644
--- a/app/models/account.rb
+++ b/app/models/account.rb
@@ -413,7 +413,7 @@ class Account < ApplicationRecord
 
     def inboxes
       urls = reorder(nil).where(protocol: :activitypub).pluck(Arel.sql("distinct coalesce(nullif(accounts.shared_inbox_url, ''), accounts.inbox_url)"))
-      DeliveryFailureTracker.filter(urls)
+      DeliveryFailureTracker.without_unavailable(urls)
     end
 
     def search_for(terms, limit = 10, offset = 0)
diff --git a/app/models/announcement.rb b/app/models/announcement.rb
index a4e427b494..c493604c22 100644
--- a/app/models/announcement.rb
+++ b/app/models/announcement.rb
@@ -14,7 +14,7 @@
 #  created_at   :datetime         not null
 #  updated_at   :datetime         not null
 #  published_at :datetime
-#  status_ids   :bigint           is an Array
+#  status_ids   :bigint(8)        is an Array
 #
 
 class Announcement < ApplicationRecord
diff --git a/app/models/relay.rb b/app/models/relay.rb
index 8c8a97db32..870f319795 100644
--- a/app/models/relay.rb
+++ b/app/models/relay.rb
@@ -27,7 +27,7 @@ class Relay < ApplicationRecord
     payload     = Oj.dump(follow_activity(activity_id))
 
     update!(state: :pending, follow_activity_id: activity_id)
-    DeliveryFailureTracker.new(inbox_url).track_success!
+    DeliveryFailureTracker.track_success!(inbox_url)
     ActivityPub::DeliveryWorker.perform_async(payload, some_local_account.id, inbox_url)
   end
 
@@ -36,7 +36,7 @@ class Relay < ApplicationRecord
     payload     = Oj.dump(unfollow_activity(activity_id))
 
     update!(state: :idle, follow_activity_id: nil)
-    DeliveryFailureTracker.new(inbox_url).track_success!
+    DeliveryFailureTracker.track_success!(inbox_url)
     ActivityPub::DeliveryWorker.perform_async(payload, some_local_account.id, inbox_url)
   end
 
diff --git a/app/models/unavailable_domain.rb b/app/models/unavailable_domain.rb
new file mode 100644
index 0000000000..e2918b5860
--- /dev/null
+++ b/app/models/unavailable_domain.rb
@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+# == Schema Information
+#
+# Table name: unavailable_domains
+#
+#  id         :bigint(8)        not null, primary key
+#  domain     :string           default(""), not null
+#  created_at :datetime         not null
+#  updated_at :datetime         not null
+#
+
+class UnavailableDomain < ApplicationRecord
+  include DomainNormalizable
+
+  after_commit :reset_cache!
+
+  private
+
+  def reset_cache!
+    Rails.cache.delete('unavailable_domains')
+  end
+end
diff --git a/app/views/admin/accounts/show.html.haml b/app/views/admin/accounts/show.html.haml
index 965fd6fb63..408f94eed8 100644
--- a/app/views/admin/accounts/show.html.haml
+++ b/app/views/admin/accounts/show.html.haml
@@ -171,12 +171,12 @@
             %th= t('admin.accounts.inbox_url')
             %td
               = @account.inbox_url
-              = fa_icon DeliveryFailureTracker.unavailable?(@account.inbox_url) ? 'times' : 'check'
+              = fa_icon DeliveryFailureTracker.available?(@account.inbox_url) ? 'check' : 'times'
           %tr
             %th= t('admin.accounts.shared_inbox_url')
             %td
               = @account.shared_inbox_url
-              = fa_icon DeliveryFailureTracker.unavailable?(@account.shared_inbox_url) ? 'times' : 'check'
+              = fa_icon DeliveryFailureTracker.available?(@account.shared_inbox_url) ? 'check': 'times'
 
   %div{ style: 'overflow: hidden' }
     %div{ style: 'float: right' }
diff --git a/app/workers/activitypub/delivery_worker.rb b/app/workers/activitypub/delivery_worker.rb
index 196af4af16..0f925658f5 100644
--- a/app/workers/activitypub/delivery_worker.rb
+++ b/app/workers/activitypub/delivery_worker.rb
@@ -12,7 +12,7 @@ class ActivityPub::DeliveryWorker
   HEADERS = { 'Content-Type' => 'application/activity+json' }.freeze
 
   def perform(json, source_account_id, inbox_url, options = {})
-    return if DeliveryFailureTracker.unavailable?(inbox_url)
+    return unless DeliveryFailureTracker.available?(inbox_url)
 
     @options        = options.with_indifferent_access
     @json           = json
diff --git a/db/migrate/20200407201300_create_unavailable_domains.rb b/db/migrate/20200407201300_create_unavailable_domains.rb
new file mode 100644
index 0000000000..56b477da5d
--- /dev/null
+++ b/db/migrate/20200407201300_create_unavailable_domains.rb
@@ -0,0 +1,9 @@
+class CreateUnavailableDomains < ActiveRecord::Migration[5.2]
+  def change
+    create_table :unavailable_domains do |t|
+      t.string :domain, default: '', null: false, index: { unique: true }
+
+      t.timestamps
+    end
+  end
+end
diff --git a/db/migrate/20200407202420_migrate_unavailable_inboxes.rb b/db/migrate/20200407202420_migrate_unavailable_inboxes.rb
new file mode 100644
index 0000000000..0dce26c6f1
--- /dev/null
+++ b/db/migrate/20200407202420_migrate_unavailable_inboxes.rb
@@ -0,0 +1,16 @@
+class MigrateUnavailableInboxes < ActiveRecord::Migration[5.2]
+  disable_ddl_transaction!
+
+  def up
+    urls = Redis.current.smembers('unavailable_inboxes')
+
+    urls.each do |url|
+      host = Addressable::URI.parse(url).normalized_host
+      UnavailableDomain.create(domain: host)
+    end
+
+    Redis.current.del(*(['unavailable_inboxes'] + Redis.current.keys('exhausted_deliveries:*')))
+  end
+
+  def down; end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 021ddac897..54e81bd3ff 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2020_03_12_185443) do
+ActiveRecord::Schema.define(version: 2020_04_07_202420) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -769,6 +769,13 @@ ActiveRecord::Schema.define(version: 2020_03_12_185443) do
     t.index ["uri"], name: "index_tombstones_on_uri"
   end
 
+  create_table "unavailable_domains", force: :cascade do |t|
+    t.string "domain", default: "", null: false
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+    t.index ["domain"], name: "index_unavailable_domains_on_domain", unique: true
+  end
+
   create_table "user_invite_requests", force: :cascade do |t|
     t.bigint "user_id"
     t.text "text"
diff --git a/spec/fabricators/unavailable_domain_fabricator.rb b/spec/fabricators/unavailable_domain_fabricator.rb
new file mode 100644
index 0000000000..f661b87c4a
--- /dev/null
+++ b/spec/fabricators/unavailable_domain_fabricator.rb
@@ -0,0 +1,3 @@
+Fabricator(:unavailable_domain) do
+  domain { Faker::Internet.domain }
+end
diff --git a/spec/lib/delivery_failure_tracker_spec.rb b/spec/lib/delivery_failure_tracker_spec.rb
index 39c8c7aafc..52a1a92f0a 100644
--- a/spec/lib/delivery_failure_tracker_spec.rb
+++ b/spec/lib/delivery_failure_tracker_spec.rb
@@ -22,11 +22,11 @@ describe DeliveryFailureTracker do
 
   describe '#track_failure!' do
     it 'marks URL as unavailable after 7 days of being called' do
-      6.times { |i| Redis.current.sadd('exhausted_deliveries:http://example.com/inbox', i) }
+      6.times { |i| Redis.current.sadd('exhausted_deliveries:example.com', i) }
       subject.track_failure!
 
       expect(subject.days).to eq 7
-      expect(described_class.unavailable?('http://example.com/inbox')).to be true
+      expect(described_class.available?('http://example.com/inbox')).to be false
     end
 
     it 'repeated calls on the same day do not count' do
@@ -37,35 +37,27 @@ describe DeliveryFailureTracker do
     end
   end
 
-  describe '.filter' do
+  describe '.without_unavailable' do
     before do
-      Redis.current.sadd('unavailable_inboxes', 'http://example.com/unavailable/inbox')
+      Fabricate(:unavailable_domain, domain: 'foo.bar')
     end
 
     it 'removes URLs that are unavailable' do
-      result = described_class.filter(['http://example.com/good/inbox', 'http://example.com/unavailable/inbox'])
+      results = described_class.without_unavailable(['http://example.com/good/inbox', 'http://foo.bar/unavailable/inbox'])
 
-      expect(result).to include('http://example.com/good/inbox')
-      expect(result).to_not include('http://example.com/unavailable/inbox')
+      expect(results).to include('http://example.com/good/inbox')
+      expect(results).to_not include('http://foo.bar/unavailable/inbox')
     end
   end
 
-  describe '.track_inverse_success!' do
-    let(:from_account) { Fabricate(:account, inbox_url: 'http://example.com/inbox', shared_inbox_url: 'http://example.com/shared/inbox') }
-
+  describe '.reset!' do
     before do
-      Redis.current.sadd('unavailable_inboxes', 'http://example.com/inbox')
-      Redis.current.sadd('unavailable_inboxes', 'http://example.com/shared/inbox')
-
-      described_class.track_inverse_success!(from_account)
+      Fabricate(:unavailable_domain, domain: 'foo.bar')
+      described_class.reset!('https://foo.bar/inbox')
     end
 
     it 'marks inbox URL as available again' do
-      expect(described_class.available?('http://example.com/inbox')).to be true
-    end
-
-    it 'marks shared inbox URL as available again' do
-      expect(described_class.available?('http://example.com/shared/inbox')).to be true
+      expect(described_class.available?('http://foo.bar/inbox')).to be true
     end
   end
 end
diff --git a/spec/models/unavailable_domain_spec.rb b/spec/models/unavailable_domain_spec.rb
new file mode 100644
index 0000000000..3f2621034c
--- /dev/null
+++ b/spec/models/unavailable_domain_spec.rb
@@ -0,0 +1,4 @@
+require 'rails_helper'
+
+RSpec.describe UnavailableDomain, type: :model do
+end