From b5ad0eb4eaab2fc17110ae18d49f6492ec7aee48 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Mon, 28 Nov 2016 13:36:47 +0100
Subject: [PATCH] Adding embedded PuSH server

---
 app/controllers/api/push_controller.rb        | 37 +++++++++++++++++++
 app/models/account.rb                         |  4 ++
 app/models/subscription.rb                    | 29 +++++++++++++++
 app/services/favourite_service.rb             |  2 +
 app/services/follow_service.rb                |  3 ++
 app/services/post_status_service.rb           |  3 ++
 .../pubsubhubbub/subscribe_service.rb         | 13 +++++++
 .../pubsubhubbub/unsubscribe_service.rb       | 15 ++++++++
 app/services/reblog_service.rb                |  2 +
 app/services/remove_status_service.rb         |  3 ++
 app/views/accounts/show.atom.ruby             |  3 ++
 .../pubsubhubbub/confirmation_worker.rb       | 29 +++++++++++++++
 app/workers/pubsubhubbub/delivery_worker.rb   | 28 ++++++++++++++
 .../pubsubhubbub/distribution_worker.rb       | 15 ++++++++
 app/workers/thread_resolve_worker.rb          |  8 ++--
 config/routes.rb                              | 11 ++++--
 .../20161128103007_create_subscriptions.rb    | 15 ++++++++
 db/schema.rb                                  | 26 ++++++++++++-
 spec/controllers/api/push_controller_spec.rb  | 13 +++++++
 spec/fabricators/subscription_fabricator.rb   |  6 +++
 spec/models/subscription_spec.rb              |  5 +++
 21 files changed, 262 insertions(+), 8 deletions(-)
 create mode 100644 app/controllers/api/push_controller.rb
 create mode 100644 app/models/subscription.rb
 create mode 100644 app/services/pubsubhubbub/subscribe_service.rb
 create mode 100644 app/services/pubsubhubbub/unsubscribe_service.rb
 create mode 100644 app/workers/pubsubhubbub/confirmation_worker.rb
 create mode 100644 app/workers/pubsubhubbub/delivery_worker.rb
 create mode 100644 app/workers/pubsubhubbub/distribution_worker.rb
 create mode 100644 db/migrate/20161128103007_create_subscriptions.rb
 create mode 100644 spec/controllers/api/push_controller_spec.rb
 create mode 100644 spec/fabricators/subscription_fabricator.rb
 create mode 100644 spec/models/subscription_spec.rb

diff --git a/app/controllers/api/push_controller.rb b/app/controllers/api/push_controller.rb
new file mode 100644
index 0000000000..78d4e36e69
--- /dev/null
+++ b/app/controllers/api/push_controller.rb
@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+class Api::PushController < ApiController
+  def update
+    mode          = params['hub.mode']
+    topic         = params['hub.topic']
+    callback      = params['hub.callback']
+    lease_seconds = params['hub.lease_seconds']
+    secret        = params['hub.secret']
+
+    case mode
+    when 'subscribe'
+      response, status = Pubsubhubbub::SubscribeService.new.call(topic_to_account(topic), callback, secret, lease_seconds)
+    when 'unsubscribe'
+      response, status = Pubsubhubbub::UnsubscribeService.new.call(topic_to_account(topic), callback)
+    else
+      response = "Unknown mode: #{mode}"
+      status   = 422
+    end
+
+    render plain: response, status: status
+  end
+
+  private
+
+  def topic_to_account(topic_url)
+    return if topic_url.blank?
+
+    uri    = Addressable::URI.parse(topic_url)
+    params = Rails.application.routes.recognize_path(uri.path)
+    domain = uri.host + (uri.port ? ":#{uri.port}" : '')
+
+    return unless TagManager.instance.local_domain?(domain) && params[:controller] == 'accounts' && params[:action] == 'show' && params[:format] == 'atom'
+
+    Account.find_local(params[:username])
+  end
+end
diff --git a/app/models/account.rb b/app/models/account.rb
index 65fad2f475..f1a4d4b4f1 100644
--- a/app/models/account.rb
+++ b/app/models/account.rb
@@ -44,8 +44,12 @@ class Account < ApplicationRecord
   has_many :block_relationships, class_name: 'Block', foreign_key: 'account_id', dependent: :destroy
   has_many :blocking, -> { order('blocks.id desc') }, through: :block_relationships, source: :target_account
 
+  # Media
   has_many :media_attachments, dependent: :destroy
 
+  # PuSH subscriptions
+  has_many :subscriptions, dependent: :destroy
+
   pg_search_scope :search_for, against: { username: 'A', domain: 'B' }, using: { tsearch: { prefix: true } }
 
   scope :remote, -> { where.not(domain: nil) }
diff --git a/app/models/subscription.rb b/app/models/subscription.rb
new file mode 100644
index 0000000000..e968c6675d
--- /dev/null
+++ b/app/models/subscription.rb
@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+class Subscription < ApplicationRecord
+  MIN_EXPIRATION = 3600 * 24
+  MAX_EXPIRATION = 3600 * 24 * 30
+
+  belongs_to :account
+
+  validates :callback_url, presence: true
+  validates :callback_url, uniqueness: { scope: :account_id }
+
+  scope :active, -> { where(confirmed: true).where('expires_at > ?', Time.now.utc) }
+
+  def lease_seconds=(str)
+    self.expires_at = Time.now.utc + [[MIN_EXPIRATION, str.to_i].max, MAX_EXPIRATION].min.seconds
+  end
+
+  def lease_seconds
+    (expires_at - Time.now.utc).to_i
+  end
+
+  before_validation :set_min_expiration
+
+  private
+
+  def set_min_expiration
+    self.lease_seconds = 0 unless expires_at
+  end
+end
diff --git a/app/services/favourite_service.rb b/app/services/favourite_service.rb
index 781b03b409..2f280e03fe 100644
--- a/app/services/favourite_service.rb
+++ b/app/services/favourite_service.rb
@@ -7,7 +7,9 @@ class FavouriteService < BaseService
   # @return [Favourite]
   def call(account, status)
     favourite = Favourite.create!(account: account, status: status)
+
     HubPingWorker.perform_async(account.id)
+    Pubsubhubbub::DistributionWorker.perform_async(favourite.stream_entry.id)
 
     if status.local?
       NotifyService.new.call(status.account, favourite)
diff --git a/app/services/follow_service.rb b/app/services/follow_service.rb
index a57e1b28af..09fa295e36 100644
--- a/app/services/follow_service.rb
+++ b/app/services/follow_service.rb
@@ -19,7 +19,10 @@ class FollowService < BaseService
     end
 
     merge_into_timeline(target_account, source_account)
+
     HubPingWorker.perform_async(source_account.id)
+    Pubsubhubbub::DistributionWorker.perform_async(follow.stream_entry.id)
+
     follow
   end
 
diff --git a/app/services/post_status_service.rb b/app/services/post_status_service.rb
index 76366e984d..979a157e96 100644
--- a/app/services/post_status_service.rb
+++ b/app/services/post_status_service.rb
@@ -14,8 +14,11 @@ class PostStatusService < BaseService
     attach_media(status, options[:media_ids])
     process_mentions_service.call(status)
     process_hashtags_service.call(status)
+
     DistributionWorker.perform_async(status.id)
     HubPingWorker.perform_async(account.id)
+    Pubsubhubbub::DistributionWorker.perform_async(status.stream_entry.id)
+
     status
   end
 
diff --git a/app/services/pubsubhubbub/subscribe_service.rb b/app/services/pubsubhubbub/subscribe_service.rb
new file mode 100644
index 0000000000..343376d778
--- /dev/null
+++ b/app/services/pubsubhubbub/subscribe_service.rb
@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+class Pubsubhubbub::SubscribeService < BaseService
+  def call(account, callback, secret, lease_seconds)
+    return ['Invalid topic URL', 422] if account.nil?
+    return ['Invalid callback URL', 422] unless !callback.blank? && callback =~ /\A#{URI.regexp(%w(http https))}\z/
+
+    subscription = Subscription.where(account: account, callback_url: callback).first_or_create!(account: account, callback_url: callback)
+    Pubsubhubbub::ConfirmationWorker.perform_async(subscription.id, 'subscribe', secret, lease_seconds)
+
+    ['', 202]
+  end
+end
diff --git a/app/services/pubsubhubbub/unsubscribe_service.rb b/app/services/pubsubhubbub/unsubscribe_service.rb
new file mode 100644
index 0000000000..a2fdc548a9
--- /dev/null
+++ b/app/services/pubsubhubbub/unsubscribe_service.rb
@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+class Pubsubhubbub::SubscribeService < BaseService
+  def call(account, callback)
+    return ['Invalid topic URL', 422] if account.nil?
+
+    subscription = Subscription.where(account: account, callback_url: callback)
+
+    unless subscription.nil?
+      Pubsubhubbub::ConfirmationWorker.perform_async(subscription.id, 'unsubscribe')
+    end
+
+    ['', 202]
+  end
+end
diff --git a/app/services/reblog_service.rb b/app/services/reblog_service.rb
index 6543d4ae78..39fdb4ea7d 100644
--- a/app/services/reblog_service.rb
+++ b/app/services/reblog_service.rb
@@ -7,8 +7,10 @@ class ReblogService < BaseService
   # @return [Status]
   def call(account, reblogged_status)
     reblog = account.statuses.create!(reblog: reblogged_status, text: '')
+
     DistributionWorker.perform_async(reblog.id)
     HubPingWorker.perform_async(account.id)
+    Pubsubhubbub::DistributionWorker.perform_async(reblog.stream_entry.id)
 
     if reblogged_status.local?
       NotifyService.new.call(reblogged_status.account, reblog)
diff --git a/app/services/remove_status_service.rb b/app/services/remove_status_service.rb
index 689abc97bb..058fd3f18a 100644
--- a/app/services/remove_status_service.rb
+++ b/app/services/remove_status_service.rb
@@ -10,6 +10,9 @@ class RemoveStatusService < BaseService
     remove_from_public(status)
 
     status.destroy!
+
+    HubPingWorker.perform_async(status.account.id)
+    Pubsubhubbub::DistributionWorker.perform_async(status.stream_entry.id)
   end
 
   private
diff --git a/app/views/accounts/show.atom.ruby b/app/views/accounts/show.atom.ruby
index d7b2201d40..558c777f01 100644
--- a/app/views/accounts/show.atom.ruby
+++ b/app/views/accounts/show.atom.ruby
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 Nokogiri::XML::Builder.new do |xml|
   feed(xml) do
     simple_id  xml, account_url(@account, format: 'atom')
@@ -12,6 +14,7 @@ Nokogiri::XML::Builder.new do |xml|
 
     link_alternate xml, TagManager.instance.url_for(@account)
     link_self      xml, account_url(@account, format: 'atom')
+    link_hub       xml, api_push_url
     link_hub       xml, Rails.configuration.x.hub_url
     link_salmon    xml, api_salmon_url(@account.id)
 
diff --git a/app/workers/pubsubhubbub/confirmation_worker.rb b/app/workers/pubsubhubbub/confirmation_worker.rb
new file mode 100644
index 0000000000..6a036adbe0
--- /dev/null
+++ b/app/workers/pubsubhubbub/confirmation_worker.rb
@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+class Pubsubhubbub::ConfirmationWorker
+  include Sidekiq::Worker
+  include RoutingHelper
+
+  def perform(subscription_id, mode, secret = nil, lease_seconds = nil)
+    subscription = Subscription.find(subscription_id)
+    challenge    = SecureRandom.hex
+
+    subscription.secret        = secret
+    subscription.lease_seconds = lease_seconds
+
+    response = HTTP.headers(user_agent: 'Mastodon/PubSubHubbub')
+                   .timeout(:per_operation, write: 20, connect: 20, read: 50)
+                   .get(subscription.callback_url, params: {
+                          'hub.topic' => account_url(subscription.account, format: :atom),
+                          'hub.mode'          => mode,
+                          'hub.challenge'     => challenge,
+                          'hub.lease_seconds' => subscription.lease_seconds,
+                        })
+
+    if mode == 'subscribe' && response.body.to_s == challenge
+      subscription.save!
+    elsif (mode == 'unsubscribe' && response.body.to_s == challenge) || !subscription.confirmed?
+      subscription.destroy!
+    end
+  end
+end
diff --git a/app/workers/pubsubhubbub/delivery_worker.rb b/app/workers/pubsubhubbub/delivery_worker.rb
new file mode 100644
index 0000000000..4d55798e8b
--- /dev/null
+++ b/app/workers/pubsubhubbub/delivery_worker.rb
@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+class Pubsubhubbub::DeliveryWorker
+  include Sidekiq::Worker
+  include RoutingHelper
+
+  def perform(subscription_id, payload)
+    subscription = Subscription.find(subscription_id)
+    headers      = {}
+
+    headers['User-Agent']      = 'Mastodon/PubSubHubbub'
+    headers['Link']            = LinkHeader.new([[api_push_url, [%w(rel hub)]], [account_url(subscription.account, format: :atom), [%w(rel self)]]]).to_s
+    headers['X-Hub-Signature'] = signature(subscription.secret, payload) unless subscription.secret.blank?
+
+    response = HTTP.timeout(:per_operation, write: 50, connect: 20, read: 50)
+                   .headers(headers)
+                   .post(subscription.callback_url, body: payload)
+
+    raise "Delivery failed for #{subscription.callback_url}: HTTP #{response.code}" unless response.code > 199 && response.code < 300
+  end
+
+  private
+
+  def signature(secret, payload)
+    hmac = OpenSSL::HMAC.hexdigest(OpenSSL::Digest.new('sha1'), secret, payload)
+    "sha1=#{hmac}"
+  end
+end
diff --git a/app/workers/pubsubhubbub/distribution_worker.rb b/app/workers/pubsubhubbub/distribution_worker.rb
new file mode 100644
index 0000000000..d8cec2ef28
--- /dev/null
+++ b/app/workers/pubsubhubbub/distribution_worker.rb
@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+class Pubsubhubbub::DistributionWorker
+  include Sidekiq::Worker
+
+  def perform(stream_entry_id)
+    stream_entry = StreamEntry.find(stream_entry_id)
+    account      = stream_entry.account
+    payload      = AccountsController.render(:show, assigns: { account: account, entries: [stream_entry] }, formats: [:atom])
+
+    Subscription.where(account: account).active.select('id').find_each do |subscription|
+      Pubsubhubbub::DeliveryWorker.perform_async(subscription.id, payload)
+    end
+  end
+end
diff --git a/app/workers/thread_resolve_worker.rb b/app/workers/thread_resolve_worker.rb
index 7001619896..84eae73bef 100644
--- a/app/workers/thread_resolve_worker.rb
+++ b/app/workers/thread_resolve_worker.rb
@@ -7,9 +7,9 @@ class ThreadResolveWorker
     child_status  = Status.find(child_status_id)
     parent_status = FetchRemoteStatusService.new.call(parent_url)
 
-    unless parent_status.nil?
-      child_status.thread = parent_status
-      child_status.save!
-    end
+    return if parent_status.nil?
+
+    child_status.thread = parent_status
+    child_status.save!
   end
 end
diff --git a/config/routes.rb b/config/routes.rb
index e0c14b47a1..5c65682985 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -1,7 +1,9 @@
+# frozen_string_literal: true
+
 require 'sidekiq/web'
 
 Rails.application.routes.draw do
-  mount ActionCable.server => '/cable'
+  mount ActionCable.server, at: 'cable'
 
   authenticate :user, lambda { |u| u.admin? } do
     mount Sidekiq::Web, at: 'sidekiq'
@@ -19,7 +21,7 @@ Rails.application.routes.draw do
     sessions:           'auth/sessions',
     registrations:      'auth/registrations',
     passwords:          'auth/passwords',
-    confirmations:      'auth/confirmations'
+    confirmations:      'auth/confirmations',
   }
 
   resources :accounts, path: 'users', only: [:show], param: :username do
@@ -43,10 +45,13 @@ Rails.application.routes.draw do
   resources :tags,  only: [:show]
 
   namespace :api do
-    # PubSubHubbub
+    # PubSubHubbub outgoing subscriptions
     resources :subscriptions, only: [:show]
     post '/subscriptions/:id', to: 'subscriptions#update'
 
+    # PubSubHubbub incoming subscriptions
+    post '/push', to: 'push#update', as: :push
+
     # Salmon
     post '/salmon/:id', to: 'salmon#update', as: :salmon
 
diff --git a/db/migrate/20161128103007_create_subscriptions.rb b/db/migrate/20161128103007_create_subscriptions.rb
new file mode 100644
index 0000000000..46443680ad
--- /dev/null
+++ b/db/migrate/20161128103007_create_subscriptions.rb
@@ -0,0 +1,15 @@
+class CreateSubscriptions < ActiveRecord::Migration[5.0]
+  def change
+    create_table :subscriptions do |t|
+      t.string :callback_url, null: false, default: ''
+      t.string :secret
+      t.datetime :expires_at, null: true, default: nil
+      t.boolean :confirmed, null: false, default: false
+      t.integer :account_id, null: false
+
+      t.timestamps
+    end
+
+    add_index :subscriptions, [:callback_url, :account_id], unique: true
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 356badf8e6..2c0e6de5bc 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20161123093447) do
+ActiveRecord::Schema.define(version: 20161128103007) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -143,6 +143,19 @@ ActiveRecord::Schema.define(version: 20161123093447) do
     t.index ["uid"], name: "index_oauth_applications_on_uid", unique: true, using: :btree
   end
 
+  create_table "pubsubhubbub_subscriptions", force: :cascade do |t|
+    t.string   "topic",      default: "",    null: false
+    t.string   "callback",   default: "",    null: false
+    t.string   "mode",       default: "",    null: false
+    t.string   "challenge",  default: "",    null: false
+    t.string   "secret"
+    t.boolean  "confirmed",  default: false, null: false
+    t.datetime "expires_at",                 null: false
+    t.datetime "created_at",                 null: false
+    t.datetime "updated_at",                 null: false
+    t.index ["topic", "callback"], name: "index_pubsubhubbub_subscriptions_on_topic_and_callback", unique: true, using: :btree
+  end
+
   create_table "settings", force: :cascade do |t|
     t.string   "var",         null: false
     t.text     "value"
@@ -185,6 +198,17 @@ ActiveRecord::Schema.define(version: 20161123093447) do
     t.index ["activity_id", "activity_type"], name: "index_stream_entries_on_activity_id_and_activity_type", using: :btree
   end
 
+  create_table "subscriptions", force: :cascade do |t|
+    t.string   "callback_url", default: "",    null: false
+    t.string   "secret"
+    t.datetime "expires_at"
+    t.boolean  "confirmed",    default: false, null: false
+    t.integer  "account_id",                   null: false
+    t.datetime "created_at",                   null: false
+    t.datetime "updated_at",                   null: false
+    t.index ["callback_url", "account_id"], name: "index_subscriptions_on_callback_url_and_account_id", unique: true, using: :btree
+  end
+
   create_table "tags", force: :cascade do |t|
     t.string   "name",       default: "", null: false
     t.datetime "created_at",              null: false
diff --git a/spec/controllers/api/push_controller_spec.rb b/spec/controllers/api/push_controller_spec.rb
new file mode 100644
index 0000000000..e699006f7b
--- /dev/null
+++ b/spec/controllers/api/push_controller_spec.rb
@@ -0,0 +1,13 @@
+require 'rails_helper'
+
+RSpec.describe Api::PushController, type: :controller do
+  describe 'POST #update' do
+    context 'with hub.mode=subscribe' do
+      pending
+    end
+
+    context 'with hub.mode=unsubscribe' do
+      pending
+    end
+  end
+end
diff --git a/spec/fabricators/subscription_fabricator.rb b/spec/fabricators/subscription_fabricator.rb
new file mode 100644
index 0000000000..0c8290494e
--- /dev/null
+++ b/spec/fabricators/subscription_fabricator.rb
@@ -0,0 +1,6 @@
+Fabricator(:subscription) do
+  callback_url "http://example.com/callback"
+  secret       "foobar"
+  expires_at   "2016-11-28 11:30:07"
+  confirmed    false
+end
diff --git a/spec/models/subscription_spec.rb b/spec/models/subscription_spec.rb
new file mode 100644
index 0000000000..d40bf0b447
--- /dev/null
+++ b/spec/models/subscription_spec.rb
@@ -0,0 +1,5 @@
+require 'rails_helper'
+
+RSpec.describe Subscription, type: :model do
+  pending "add some examples to (or delete) #{__FILE__}"
+end