* Add structure for lists

* Add list timeline streaming API

* Add list APIs, bind list-account relation to follow relation

* Add API for adding/removing accounts from lists

* Add pagination to lists API

* Add pagination to list accounts API

* Adjust scopes for new APIs

- Creating and modifying lists merely requires "write" scope
- Fetching information about lists merely requires "read" scope

* Add test for wrong user context on list timeline

* Clean up tests
main
Eugen Rochko 7 years ago committed by GitHub
parent 4a2fc2d444
commit 24cafd73a2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -0,0 +1,81 @@
# frozen_string_literal: true
class Api::V1::Lists::AccountsController < Api::BaseController
before_action -> { doorkeeper_authorize! :read }, only: [:show]
before_action -> { doorkeeper_authorize! :write }, except: [:show]
before_action :require_user!
before_action :set_list
after_action :insert_pagination_headers, only: :show
def show
@accounts = @list.accounts.paginate_by_max_id(limit_param(DEFAULT_ACCOUNTS_LIMIT), params[:max_id], params[:since_id])
render json: @accounts, each_serializer: REST::AccountSerializer
end
def create
ApplicationRecord.transaction do
list_accounts.each do |account|
@list.accounts << account
end
end
render_empty
end
def destroy
ListAccount.where(list: @list, account_id: account_ids).destroy_all
render_empty
end
private
def set_list
@list = List.where(account: current_account).find(params[:list_id])
end
def list_accounts
Account.find(account_ids)
end
def account_ids
Array(resource_params[:account_ids])
end
def resource_params
params.permit(account_ids: [])
end
def insert_pagination_headers
set_pagination_headers(next_path, prev_path)
end
def next_path
if records_continue?
api_v1_list_accounts_url pagination_params(max_id: pagination_max_id)
end
end
def prev_path
unless @accounts.empty?
api_v1_list_accounts_url pagination_params(since_id: pagination_since_id)
end
end
def pagination_max_id
@accounts.last.id
end
def pagination_since_id
@accounts.first.id
end
def records_continue?
@accounts.size == limit_param(DEFAULT_ACCOUNTS_LIMIT)
end
def pagination_params(core_params)
params.permit(:limit).merge(core_params)
end
end

@ -0,0 +1,79 @@
# frozen_string_literal: true
class Api::V1::ListsController < Api::BaseController
LISTS_LIMIT = 50
before_action -> { doorkeeper_authorize! :read }, only: [:index, :show]
before_action -> { doorkeeper_authorize! :write }, except: [:index, :show]
before_action :require_user!
before_action :set_list, except: [:index, :create]
after_action :insert_pagination_headers, only: :index
def index
@lists = List.where(account: current_account).paginate_by_max_id(limit_param(LISTS_LIMIT), params[:max_id], params[:since_id])
render json: @lists, each_serializer: REST::ListSerializer
end
def show
render json: @list, serializer: REST::ListSerializer
end
def create
@list = List.create!(list_params.merge(account: current_account))
render json: @list, serializer: REST::ListSerializer
end
def update
@list.update!(list_params)
render json: @list, serializer: REST::ListSerializer
end
def destroy
@list.destroy!
render_empty
end
private
def set_list
@list = List.where(account: current_account).find(params[:id])
end
def list_params
params.permit(:title)
end
def insert_pagination_headers
set_pagination_headers(next_path, prev_path)
end
def next_path
if records_continue?
api_v1_lists_url pagination_params(max_id: pagination_max_id)
end
end
def prev_path
unless @lists.empty?
api_v1_lists_url pagination_params(since_id: pagination_since_id)
end
end
def pagination_max_id
@lists.last.id
end
def pagination_since_id
@lists.first.id
end
def records_continue?
@lists.size == limit_param(LISTS_LIMIT)
end
def pagination_params(core_params)
params.permit(:limit).merge(core_params)
end
end

@ -31,7 +31,7 @@ class Api::V1::Timelines::HomeController < Api::BaseController
end end
def account_home_feed def account_home_feed
Feed.new(:home, current_account) HomeFeed.new(current_account)
end end
def insert_pagination_headers def insert_pagination_headers

@ -0,0 +1,66 @@
# frozen_string_literal: true
class Api::V1::Timelines::ListController < Api::BaseController
before_action -> { doorkeeper_authorize! :read }
before_action :require_user!
before_action :set_list
before_action :set_statuses
after_action :insert_pagination_headers, unless: -> { @statuses.empty? }
def show
render json: @statuses,
each_serializer: REST::StatusSerializer,
relationships: StatusRelationshipsPresenter.new(@statuses, current_user.account_id)
end
private
def set_list
@list = List.where(account: current_account).find(params[:id])
end
def set_statuses
@statuses = cached_list_statuses
end
def cached_list_statuses
cache_collection list_statuses, Status
end
def list_statuses
list_feed.get(
limit_param(DEFAULT_STATUSES_LIMIT),
params[:max_id],
params[:since_id]
)
end
def list_feed
ListFeed.new(@list)
end
def insert_pagination_headers
set_pagination_headers(next_path, prev_path)
end
def pagination_params(core_params)
params.permit(:limit).merge(core_params)
end
def next_path
api_v1_timelines_list_url params[:id], pagination_params(max_id: pagination_max_id)
end
def prev_path
api_v1_timelines_list_url params[:id], pagination_params(since_id: pagination_since_id)
end
def pagination_max_id
@statuses.last.id
end
def pagination_since_id
@statuses.first.id
end
end

@ -26,34 +26,42 @@ class FeedManager
end end
end end
def push(timeline_type, account, status) def push_to_home(account, status)
return false unless add_to_feed(timeline_type, account, status) return false unless add_to_feed(:home, account.id, status)
trim(:home, account.id)
trim(timeline_type, account.id) PushUpdateWorker.perform_async(account.id, status.id, "timeline:#{account.id}") if push_update_required?("timeline:#{account.id}")
PushUpdateWorker.perform_async(account.id, status.id) if push_update_required?(timeline_type, account.id)
true true
end end
def unpush(timeline_type, account, status) def unpush_from_home(account, status)
return false unless remove_from_feed(timeline_type, account, status) return false unless remove_from_feed(:home, account.id, status)
Redis.current.publish("timeline:#{account.id}", Oj.dump(event: :delete, payload: status.id.to_s))
true
end
payload = Oj.dump(event: :delete, payload: status.id.to_s) def push_to_list(list, status)
Redis.current.publish("timeline:#{account.id}", payload) return false unless add_to_feed(:list, list.id, status)
trim(:list, list.id)
PushUpdateWorker.perform_async(list.account_id, status.id, "timeline:list:#{list.id}") if push_update_required?("timeline:list:#{list.id}")
true
end
def unpush_from_list(list, status)
return false unless remove_from_feed(:list, list.id, status)
Redis.current.publish("timeline:list:#{list.id}", Oj.dump(event: :delete, payload: status.id.to_s))
true true
end end
def trim(type, account_id) def trim(type, account_id)
timeline_key = key(type, account_id) timeline_key = key(type, account_id)
reblog_key = key(type, account_id, 'reblogs') reblog_key = key(type, account_id, 'reblogs')
# Remove any items past the MAX_ITEMS'th entry in our feed # Remove any items past the MAX_ITEMS'th entry in our feed
redis.zremrangebyrank(timeline_key, '0', (-(FeedManager::MAX_ITEMS + 1)).to_s) redis.zremrangebyrank(timeline_key, '0', (-(FeedManager::MAX_ITEMS + 1)).to_s)
# Get the score of the REBLOG_FALLOFF'th item in our feed, and stop # Get the score of the REBLOG_FALLOFF'th item in our feed, and stop
# tracking anything after it for deduplication purposes. # tracking anything after it for deduplication purposes.
falloff_rank = FeedManager::REBLOG_FALLOFF - 1 falloff_rank = FeedManager::REBLOG_FALLOFF - 1
falloff_range = redis.zrevrange(timeline_key, falloff_rank, falloff_rank, with_scores: true) falloff_range = redis.zrevrange(timeline_key, falloff_rank, falloff_rank, with_scores: true)
falloff_score = falloff_range&.first&.last&.to_i || 0 falloff_score = falloff_range&.first&.last&.to_i || 0
@ -69,10 +77,6 @@ class FeedManager
end end
end end
def push_update_required?(timeline_type, account_id)
timeline_type != :home || redis.get("subscribed:timeline:#{account_id}").present?
end
def merge_into_timeline(from_account, into_account) def merge_into_timeline(from_account, into_account)
timeline_key = key(:home, into_account.id) timeline_key = key(:home, into_account.id)
query = from_account.statuses.limit(FeedManager::MAX_ITEMS / 4) query = from_account.statuses.limit(FeedManager::MAX_ITEMS / 4)
@ -84,28 +88,28 @@ class FeedManager
query.each do |status| query.each do |status|
next if status.direct_visibility? || filter?(:home, status, into_account) next if status.direct_visibility? || filter?(:home, status, into_account)
add_to_feed(:home, into_account, status) add_to_feed(:home, into_account.id, status)
end end
trim(:home, into_account.id) trim(:home, into_account.id)
end end
def unmerge_from_timeline(from_account, into_account) def unmerge_from_timeline(from_account, into_account)
timeline_key = key(:home, into_account.id) timeline_key = key(:home, into_account.id)
oldest_home_score = redis.zrange(timeline_key, 0, 0, with_scores: true)&.first&.last&.to_i || 0 oldest_home_score = redis.zrange(timeline_key, 0, 0, with_scores: true)&.first&.last&.to_i || 0
from_account.statuses.select('id, reblog_of_id').where('id > ?', oldest_home_score).reorder(nil).find_each do |status| from_account.statuses.select('id, reblog_of_id').where('id > ?', oldest_home_score).reorder(nil).find_each do |status|
remove_from_feed(:home, into_account, status) remove_from_feed(:home, into_account.id, status)
end end
end end
def clear_from_timeline(account, target_account) def clear_from_timeline(account, target_account)
timeline_key = key(:home, account.id) timeline_key = key(:home, account.id)
timeline_status_ids = redis.zrange(timeline_key, 0, -1) timeline_status_ids = redis.zrange(timeline_key, 0, -1)
target_statuses = Status.where(id: timeline_status_ids, account: target_account) target_statuses = Status.where(id: timeline_status_ids, account: target_account)
target_statuses.each do |status| target_statuses.each do |status|
unpush(:home, account, status) unpush_from_home(account, status)
end end
end end
@ -122,7 +126,7 @@ class FeedManager
statuses.each do |status| statuses.each do |status|
next if filter_from_home?(status, account) next if filter_from_home?(status, account)
added += 1 if add_to_feed(:home, account, status) added += 1 if add_to_feed(:home, account.id, status)
end end
break unless added.zero? break unless added.zero?
@ -137,6 +141,10 @@ class FeedManager
Redis.current Redis.current
end end
def push_update_required?(timeline_id)
redis.exists("subscribed:#{timeline_id}")
end
def filter_from_home?(status, receiver_id) def filter_from_home?(status, receiver_id)
return false if receiver_id == status.account_id return false if receiver_id == status.account_id
return true if status.reply? && (status.in_reply_to_id.nil? || status.in_reply_to_account_id.nil?) return true if status.reply? && (status.in_reply_to_id.nil? || status.in_reply_to_account_id.nil?)
@ -182,9 +190,9 @@ class FeedManager
# added, and false if it was not added to the feed. Note that this is # added, and false if it was not added to the feed. Note that this is
# an internal helper: callers must call trim or push updates if # an internal helper: callers must call trim or push updates if
# either action is appropriate. # either action is appropriate.
def add_to_feed(timeline_type, account, status) def add_to_feed(timeline_type, account_id, status)
timeline_key = key(timeline_type, account.id) timeline_key = key(timeline_type, account_id)
reblog_key = key(timeline_type, account.id, 'reblogs') reblog_key = key(timeline_type, account_id, 'reblogs')
if status.reblog? if status.reblog?
# If the original status or a reblog of it is within # If the original status or a reblog of it is within
@ -195,6 +203,7 @@ class FeedManager
return false if !rank.nil? && rank < FeedManager::REBLOG_FALLOFF return false if !rank.nil? && rank < FeedManager::REBLOG_FALLOFF
reblog_rank = redis.zrevrank(reblog_key, status.reblog_of_id) reblog_rank = redis.zrevrank(reblog_key, status.reblog_of_id)
if reblog_rank.nil? if reblog_rank.nil?
# This is not something we've already seen reblogged, so we # This is not something we've already seen reblogged, so we
# can just add it to the feed (and note that we're # can just add it to the feed (and note that we're
@ -205,7 +214,7 @@ class FeedManager
# Another reblog of the same status was already in the # Another reblog of the same status was already in the
# REBLOG_FALLOFF most recent statuses, so we note that this # REBLOG_FALLOFF most recent statuses, so we note that this
# is an "extra" reblog, by storing it in reblog_set_key. # is an "extra" reblog, by storing it in reblog_set_key.
reblog_set_key = key(timeline_type, account.id, "reblogs:#{status.reblog_of_id}") reblog_set_key = key(timeline_type, account_id, "reblogs:#{status.reblog_of_id}")
redis.sadd(reblog_set_key, status.id) redis.sadd(reblog_set_key, status.id)
return false return false
end end
@ -220,8 +229,8 @@ class FeedManager
# with reblogs, and returning true if a status was removed. As with # with reblogs, and returning true if a status was removed. As with
# `add_to_feed`, this does not trigger push updates, so callers must # `add_to_feed`, this does not trigger push updates, so callers must
# do so if appropriate. # do so if appropriate.
def remove_from_feed(timeline_type, account, status) def remove_from_feed(timeline_type, account_id, status)
timeline_key = key(timeline_type, account.id) timeline_key = key(timeline_type, account_id)
if status.reblog? if status.reblog?
# 1. If the reblogging status is not in the feed, stop. # 1. If the reblogging status is not in the feed, stop.
@ -229,7 +238,7 @@ class FeedManager
return false if status_rank.nil? return false if status_rank.nil?
# 2. Remove reblog from set of this status's reblogs. # 2. Remove reblog from set of this status's reblogs.
reblog_set_key = key(timeline_type, account.id, "reblogs:#{status.reblog_of_id}") reblog_set_key = key(timeline_type, account_id, "reblogs:#{status.reblog_of_id}")
redis.srem(reblog_set_key, status.id) redis.srem(reblog_set_key, status.id)
# 3. Re-insert another reblog or original into the feed if one # 3. Re-insert another reblog or original into the feed if one
@ -244,7 +253,7 @@ class FeedManager
# (outside conditional) # (outside conditional)
else else
# If the original is getting deleted, no use for reblog references # If the original is getting deleted, no use for reblog references
redis.del(key(timeline_type, account.id, "reblogs:#{status.id}")) redis.del(key(timeline_type, account_id, "reblogs:#{status.id}"))
end end
redis.zrem(timeline_key, status.id) redis.zrem(timeline_key, status.id)

@ -3,7 +3,7 @@
# #
# Table name: accounts # Table name: accounts
# #
# id :bigint not null, primary key # id :integer not null, primary key
# username :string default(""), not null # username :string default(""), not null
# domain :string # domain :string
# secret :string default(""), not null # secret :string default(""), not null
@ -53,6 +53,7 @@ class Account < ApplicationRecord
include AccountInteractions include AccountInteractions
include Attachmentable include Attachmentable
include Remotable include Remotable
include Paginable
enum protocol: [:ostatus, :activitypub] enum protocol: [:ostatus, :activitypub]
@ -95,6 +96,10 @@ class Account < ApplicationRecord
has_many :account_moderation_notes, dependent: :destroy has_many :account_moderation_notes, dependent: :destroy
has_many :targeted_moderation_notes, class_name: 'AccountModerationNote', foreign_key: :target_account_id, dependent: :destroy has_many :targeted_moderation_notes, class_name: 'AccountModerationNote', foreign_key: :target_account_id, dependent: :destroy
# Lists
has_many :list_accounts, inverse_of: :account, dependent: :destroy
has_many :lists, through: :list_accounts
scope :remote, -> { where.not(domain: nil) } scope :remote, -> { where.not(domain: nil) }
scope :local, -> { where(domain: nil) } scope :local, -> { where(domain: nil) }
scope :without_followers, -> { where(followers_count: 0) } scope :without_followers, -> { where(followers_count: 0) }

@ -3,11 +3,11 @@
# #
# Table name: account_domain_blocks # Table name: account_domain_blocks
# #
# id :integer not null, primary key
# domain :string # domain :string
# created_at :datetime not null # created_at :datetime not null
# updated_at :datetime not null # updated_at :datetime not null
# account_id :bigint # account_id :integer
# id :bigint not null, primary key
# #
class AccountDomainBlock < ApplicationRecord class AccountDomainBlock < ApplicationRecord

@ -3,10 +3,10 @@
# #
# Table name: account_moderation_notes # Table name: account_moderation_notes
# #
# id :bigint not null, primary key # id :integer not null, primary key
# content :text not null # content :text not null
# account_id :bigint not null # account_id :integer not null
# target_account_id :bigint not null # target_account_id :integer not null
# created_at :datetime not null # created_at :datetime not null
# updated_at :datetime not null # updated_at :datetime not null
# #

@ -3,11 +3,11 @@
# #
# Table name: blocks # Table name: blocks
# #
# id :integer not null, primary key
# created_at :datetime not null # created_at :datetime not null
# updated_at :datetime not null # updated_at :datetime not null
# account_id :bigint not null # account_id :integer not null
# id :bigint not null, primary key # target_account_id :integer not null
# target_account_id :bigint not null
# #
class Block < ApplicationRecord class Block < ApplicationRecord

@ -3,7 +3,7 @@
# #
# Table name: conversations # Table name: conversations
# #
# id :bigint not null, primary key # id :integer not null, primary key
# uri :string # uri :string
# created_at :datetime not null # created_at :datetime not null
# updated_at :datetime not null # updated_at :datetime not null

@ -3,9 +3,9 @@
# #
# Table name: conversation_mutes # Table name: conversation_mutes
# #
# conversation_id :bigint not null # id :integer not null, primary key
# account_id :bigint not null # conversation_id :integer not null
# id :bigint not null, primary key # account_id :integer not null
# #
class ConversationMute < ApplicationRecord class ConversationMute < ApplicationRecord

@ -3,7 +3,7 @@
# #
# Table name: custom_emojis # Table name: custom_emojis
# #
# id :bigint not null, primary key # id :integer not null, primary key
# shortcode :string default(""), not null # shortcode :string default(""), not null
# domain :string # domain :string
# image_file_name :string # image_file_name :string

@ -3,12 +3,12 @@
# #
# Table name: domain_blocks # Table name: domain_blocks
# #
# id :integer not null, primary key
# domain :string default(""), not null # domain :string default(""), not null
# created_at :datetime not null # created_at :datetime not null
# updated_at :datetime not null # updated_at :datetime not null
# severity :integer default("silence") # severity :integer default("silence")
# reject_media :boolean default(FALSE), not null # reject_media :boolean default(FALSE), not null
# id :bigint not null, primary key
# #
class DomainBlock < ApplicationRecord class DomainBlock < ApplicationRecord

@ -3,7 +3,7 @@
# #
# Table name: email_domain_blocks # Table name: email_domain_blocks
# #
# id :bigint not null, primary key # id :integer not null, primary key
# domain :string default(""), not null # domain :string default(""), not null
# created_at :datetime not null # created_at :datetime not null
# updated_at :datetime not null # updated_at :datetime not null

@ -3,11 +3,11 @@
# #
# Table name: favourites # Table name: favourites
# #
# id :integer not null, primary key
# created_at :datetime not null # created_at :datetime not null
# updated_at :datetime not null # updated_at :datetime not null
# account_id :bigint not null # account_id :integer not null
# id :bigint not null, primary key # status_id :integer not null
# status_id :bigint not null
# #
class Favourite < ApplicationRecord class Favourite < ApplicationRecord

@ -1,36 +1,27 @@
# frozen_string_literal: true # frozen_string_literal: true
class Feed class Feed
def initialize(type, account) def initialize(type, id)
@type = type @type = type
@account = account @id = id
end end
def get(limit, max_id = nil, since_id = nil) def get(limit, max_id = nil, since_id = nil)
if redis.exists("account:#{@account.id}:regeneration") from_redis(limit, max_id, since_id)
from_database(limit, max_id, since_id)
else
from_redis(limit, max_id, since_id)
end
end end
private protected
def from_redis(limit, max_id, since_id) def from_redis(limit, max_id, since_id)
max_id = '+inf' if max_id.blank? max_id = '+inf' if max_id.blank?
since_id = '-inf' if since_id.blank? since_id = '-inf' if since_id.blank?
unhydrated = redis.zrevrangebyscore(key, "(#{max_id}", "(#{since_id}", limit: [0, limit], with_scores: true).map(&:first).map(&:to_i) unhydrated = redis.zrevrangebyscore(key, "(#{max_id}", "(#{since_id}", limit: [0, limit], with_scores: true).map(&:first).map(&:to_i)
Status.where(id: unhydrated).cache_ids
end
def from_database(limit, max_id, since_id) Status.where(id: unhydrated).cache_ids
Status.as_home_timeline(@account)
.paginate_by_max_id(limit, max_id, since_id)
.reject { |status| FeedManager.instance.filter?(:home, status, @account.id) }
end end
def key def key
FeedManager.instance.key(@type, @account.id) FeedManager.instance.key(@type, @id)
end end
def redis def redis

@ -3,11 +3,11 @@
# #
# Table name: follows # Table name: follows
# #
# id :integer not null, primary key
# created_at :datetime not null # created_at :datetime not null
# updated_at :datetime not null # updated_at :datetime not null
# account_id :bigint not null # account_id :integer not null
# id :bigint not null, primary key # target_account_id :integer not null
# target_account_id :bigint not null
# #
class Follow < ApplicationRecord class Follow < ApplicationRecord

@ -3,11 +3,11 @@
# #
# Table name: follow_requests # Table name: follow_requests
# #
# id :integer not null, primary key
# created_at :datetime not null # created_at :datetime not null
# updated_at :datetime not null # updated_at :datetime not null
# account_id :bigint not null # account_id :integer not null
# id :bigint not null, primary key # target_account_id :integer not null
# target_account_id :bigint not null
# #
class FollowRequest < ApplicationRecord class FollowRequest < ApplicationRecord

@ -0,0 +1,25 @@
# frozen_string_literal: true
class HomeFeed < Feed
def initialize(account)
@type = :home
@id = account.id
@account = account
end
def get(limit, max_id = nil, since_id = nil)
if redis.exists("account:#{@account.id}:regeneration")
from_database(limit, max_id, since_id)
else
super
end
end
private
def from_database(limit, max_id, since_id)
Status.as_home_timeline(@account)
.paginate_by_max_id(limit, max_id, since_id)
.reject { |status| FeedManager.instance.filter?(:home, status, @account.id) }
end
end

@ -3,6 +3,7 @@
# #
# Table name: imports # Table name: imports
# #
# id :integer not null, primary key
# type :integer not null # type :integer not null
# approved :boolean default(FALSE), not null # approved :boolean default(FALSE), not null
# created_at :datetime not null # created_at :datetime not null
@ -11,8 +12,7 @@
# data_content_type :string # data_content_type :string
# data_file_size :integer # data_file_size :integer
# data_updated_at :datetime # data_updated_at :datetime
# account_id :bigint not null # account_id :integer not null
# id :bigint not null, primary key
# #
class Import < ApplicationRecord class Import < ApplicationRecord

@ -0,0 +1,22 @@
# frozen_string_literal: true
# == Schema Information
#
# Table name: lists
#
# id :integer not null, primary key
# account_id :integer
# title :string default(""), not null
# created_at :datetime not null
# updated_at :datetime not null
#
class List < ApplicationRecord
include Paginable
belongs_to :account
has_many :list_accounts, inverse_of: :list, dependent: :destroy
has_many :accounts, through: :list_accounts
validates :title, presence: true
end

@ -0,0 +1,24 @@
# frozen_string_literal: true
# == Schema Information
#
# Table name: list_accounts
#
# id :integer not null, primary key
# list_id :integer not null
# account_id :integer not null
# follow_id :integer not null
#
class ListAccount < ApplicationRecord
belongs_to :list, required: true
belongs_to :account, required: true
belongs_to :follow, required: true
before_validation :set_follow
private
def set_follow
self.follow = Follow.find_by(account_id: list.account_id, target_account_id: account.id)
end
end

@ -0,0 +1,8 @@
# frozen_string_literal: true
class ListFeed < Feed
def initialize(list)
@type = :list
@id = list.id
end
end

@ -3,19 +3,19 @@
# #
# Table name: media_attachments # Table name: media_attachments
# #
# id :bigint not null, primary key # id :integer not null, primary key
# status_id :bigint # status_id :integer
# file_file_name :string # file_file_name :string
# file_content_type :string # file_content_type :string
# file_file_size :integer # file_file_size :integer
# file_updated_at :datetime # file_updated_at :datetime
# remote_url :string default(""), not null # remote_url :string default(""), not null
# account_id :bigint
# created_at :datetime not null # created_at :datetime not null
# updated_at :datetime not null # updated_at :datetime not null
# shortcode :string # shortcode :string
# type :integer default("image"), not null # type :integer default("image"), not null
# file_meta :json # file_meta :json
# account_id :integer
# description :text # description :text
# #

@ -3,11 +3,11 @@
# #
# Table name: mentions # Table name: mentions
# #
# status_id :bigint # id :integer not null, primary key
# status_id :integer
# created_at :datetime not null # created_at :datetime not null
# updated_at :datetime not null # updated_at :datetime not null
# account_id :bigint # account_id :integer
# id :bigint not null, primary key
# #
class Mention < ApplicationRecord class Mention < ApplicationRecord

@ -3,13 +3,13 @@
# #
# Table name: notifications # Table name: notifications
# #
# id :bigint not null, primary key # id :integer not null, primary key
# account_id :bigint # activity_id :integer
# activity_id :bigint
# activity_type :string # activity_type :string
# created_at :datetime not null # created_at :datetime not null
# updated_at :datetime not null # updated_at :datetime not null
# from_account_id :bigint # account_id :integer
# from_account_id :integer
# #
class Notification < ApplicationRecord class Notification < ApplicationRecord

@ -3,7 +3,7 @@
# #
# Table name: preview_cards # Table name: preview_cards
# #
# id :bigint not null, primary key # id :integer not null, primary key
# url :string default(""), not null # url :string default(""), not null
# title :string default(""), not null # title :string default(""), not null
# description :string default(""), not null # description :string default(""), not null

@ -3,15 +3,15 @@
# #
# Table name: reports # Table name: reports
# #
# id :integer not null, primary key
# status_ids :integer default([]), not null, is an Array # status_ids :integer default([]), not null, is an Array
# comment :text default(""), not null # comment :text default(""), not null
# action_taken :boolean default(FALSE), not null # action_taken :boolean default(FALSE), not null
# created_at :datetime not null # created_at :datetime not null
# updated_at :datetime not null # updated_at :datetime not null
# account_id :bigint not null # account_id :integer not null
# action_taken_by_account_id :bigint # action_taken_by_account_id :integer
# id :bigint not null, primary key # target_account_id :integer not null
# target_account_id :bigint not null
# #
class Report < ApplicationRecord class Report < ApplicationRecord

@ -3,15 +3,15 @@
# #
# Table name: session_activations # Table name: session_activations
# #
# id :bigint not null, primary key # id :integer not null, primary key
# user_id :bigint not null
# session_id :string not null # session_id :string not null
# created_at :datetime not null # created_at :datetime not null
# updated_at :datetime not null # updated_at :datetime not null
# user_agent :string default(""), not null # user_agent :string default(""), not null
# ip :inet # ip :inet
# access_token_id :bigint # access_token_id :integer
# web_push_subscription_id :bigint # user_id :integer not null
# web_push_subscription_id :integer
# #
# id :bigint not null, primary key # id :bigint not null, primary key

@ -3,13 +3,13 @@
# #
# Table name: settings # Table name: settings
# #
# id :integer not null, primary key
# var :string not null # var :string not null
# value :text # value :text
# thing_type :string # thing_type :string
# created_at :datetime # created_at :datetime
# updated_at :datetime # updated_at :datetime
# id :bigint not null, primary key # thing_id :integer
# thing_id :bigint
# #
class Setting < RailsSettings::Base class Setting < RailsSettings::Base

@ -3,7 +3,7 @@
# #
# Table name: site_uploads # Table name: site_uploads
# #
# id :bigint not null, primary key # id :integer not null, primary key
# var :string default(""), not null # var :string default(""), not null
# file_file_name :string # file_file_name :string
# file_content_type :string # file_content_type :string

@ -3,26 +3,26 @@
# #
# Table name: statuses # Table name: statuses
# #
# id :bigint not null, primary key # id :integer not null, primary key
# uri :string # uri :string
# account_id :bigint not null
# text :text default(""), not null # text :text default(""), not null
# created_at :datetime not null # created_at :datetime not null
# updated_at :datetime not null # updated_at :datetime not null
# in_reply_to_id :bigint # in_reply_to_id :integer
# reblog_of_id :bigint # reblog_of_id :integer
# url :string # url :string
# sensitive :boolean default(FALSE), not null # sensitive :boolean default(FALSE), not null
# visibility :integer default("public"), not null # visibility :integer default("public"), not null
# in_reply_to_account_id :bigint
# application_id :bigint
# spoiler_text :text default(""), not null # spoiler_text :text default(""), not null
# reply :boolean default(FALSE), not null # reply :boolean default(FALSE), not null
# favourites_count :integer default(0), not null # favourites_count :integer default(0), not null
# reblogs_count :integer default(0), not null # reblogs_count :integer default(0), not null
# language :string # language :string
# conversation_id :bigint # conversation_id :integer
# local :boolean # local :boolean
# account_id :integer not null
# application_id :integer
# in_reply_to_account_id :integer
# #
class Status < ApplicationRecord class Status < ApplicationRecord

@ -3,9 +3,9 @@
# #
# Table name: status_pins # Table name: status_pins
# #
# id :bigint not null, primary key # id :integer not null, primary key
# account_id :bigint not null # account_id :integer not null
# status_id :bigint not null # status_id :integer not null
# created_at :datetime not null # created_at :datetime not null
# updated_at :datetime not null # updated_at :datetime not null
# #

@ -3,13 +3,13 @@
# #
# Table name: stream_entries # Table name: stream_entries
# #
# activity_id :bigint # id :integer not null, primary key
# activity_id :integer
# activity_type :string # activity_type :string
# created_at :datetime not null # created_at :datetime not null
# updated_at :datetime not null # updated_at :datetime not null
# hidden :boolean default(FALSE), not null # hidden :boolean default(FALSE), not null
# account_id :bigint # account_id :integer
# id :bigint not null, primary key
# #
class StreamEntry < ApplicationRecord class StreamEntry < ApplicationRecord

@ -3,6 +3,7 @@
# #
# Table name: subscriptions # Table name: subscriptions
# #
# id :integer not null, primary key
# callback_url :string default(""), not null # callback_url :string default(""), not null
# secret :string # secret :string
# expires_at :datetime # expires_at :datetime
@ -11,8 +12,7 @@
# updated_at :datetime not null # updated_at :datetime not null
# last_successful_delivery_at :datetime # last_successful_delivery_at :datetime
# domain :string # domain :string
# account_id :bigint not null # account_id :integer not null
# id :bigint not null, primary key
# #
class Subscription < ApplicationRecord class Subscription < ApplicationRecord

@ -3,7 +3,7 @@
# #
# Table name: tags # Table name: tags
# #
# id :bigint not null, primary key # id :integer not null, primary key
# name :string default(""), not null # name :string default(""), not null
# created_at :datetime not null # created_at :datetime not null
# updated_at :datetime not null # updated_at :datetime not null

@ -3,7 +3,7 @@
# #
# Table name: users # Table name: users
# #
# id :bigint not null, primary key # id :integer not null, primary key
# email :string default(""), not null # email :string default(""), not null
# created_at :datetime not null # created_at :datetime not null
# updated_at :datetime not null # updated_at :datetime not null
@ -30,7 +30,7 @@
# last_emailed_at :datetime # last_emailed_at :datetime
# otp_backup_codes :string is an Array # otp_backup_codes :string is an Array
# filtered_languages :string default([]), not null, is an Array # filtered_languages :string default([]), not null, is an Array
# account_id :bigint not null # account_id :integer not null
# disabled :boolean default(FALSE), not null # disabled :boolean default(FALSE), not null
# moderator :boolean default(FALSE), not null # moderator :boolean default(FALSE), not null
# #

@ -3,7 +3,7 @@
# #
# Table name: web_push_subscriptions # Table name: web_push_subscriptions
# #
# id :bigint not null, primary key # id :integer not null, primary key
# endpoint :string not null # endpoint :string not null
# key_p256dh :string not null # key_p256dh :string not null
# key_auth :string not null # key_auth :string not null

@ -3,11 +3,11 @@
# #
# Table name: web_settings # Table name: web_settings
# #
# id :integer not null, primary key
# data :json # data :json
# created_at :datetime not null # created_at :datetime not null
# updated_at :datetime not null # updated_at :datetime not null
# id :bigint not null, primary key # user_id :integer
# user_id :bigint
# #
class Web::Setting < ApplicationRecord class Web::Setting < ApplicationRecord

@ -0,0 +1,5 @@
# frozen_string_literal: true
class REST::ListSerializer < ActiveModel::Serializer
attributes :id, :title
end

@ -30,6 +30,7 @@ class BatchedRemoveStatusService < BaseService
account = account_statuses.first.account account = account_statuses.first.account
unpush_from_home_timelines(account, account_statuses) unpush_from_home_timelines(account, account_statuses)
unpush_from_list_timelines(account, account_statuses)
if account.local? if account.local?
batch_stream_entries(account, account_statuses) batch_stream_entries(account, account_statuses)
@ -79,7 +80,15 @@ class BatchedRemoveStatusService < BaseService
recipients.each do |follower| recipients.each do |follower|
statuses.each do |status| statuses.each do |status|
FeedManager.instance.unpush(:home, follower, status) FeedManager.instance.unpush_from_home(follower, status)
end
end
end
def unpush_from_list_timelines(account, statuses)
account.lists.select(:id, :account_id).each do |list|
statuses.each do |status|
FeedManager.instance.unpush_from_list(list, status)
end end
end end
end end

@ -14,6 +14,7 @@ class FanOutOnWriteService < BaseService
deliver_to_mentioned_followers(status) deliver_to_mentioned_followers(status)
else else
deliver_to_followers(status) deliver_to_followers(status)
deliver_to_lists(status)
end end
return if status.account.silenced? || !status.public_visibility? || status.reblog? return if status.account.silenced? || !status.public_visibility? || status.reblog?
@ -30,7 +31,7 @@ class FanOutOnWriteService < BaseService
def deliver_to_self(status) def deliver_to_self(status)
Rails.logger.debug "Delivering status #{status.id} to author" Rails.logger.debug "Delivering status #{status.id} to author"
FeedManager.instance.push(:home, status.account, status) FeedManager.instance.push_to_home(status.account, status)
end end
def deliver_to_followers(status) def deliver_to_followers(status)
@ -38,7 +39,17 @@ class FanOutOnWriteService < BaseService
status.account.followers.where(domain: nil).joins(:user).where('users.current_sign_in_at > ?', 14.days.ago).select(:id).reorder(nil).find_in_batches do |followers| status.account.followers.where(domain: nil).joins(:user).where('users.current_sign_in_at > ?', 14.days.ago).select(:id).reorder(nil).find_in_batches do |followers|
FeedInsertWorker.push_bulk(followers) do |follower| FeedInsertWorker.push_bulk(followers) do |follower|
[status.id, follower.id] [status.id, follower.id, :home]
end
end
end
def deliver_to_lists(status)
Rails.logger.debug "Delivering status #{status.id} to lists"
status.account.lists.joins(account: :user).where('users.current_sign_in_at > ?', 14.days.ago).select(:id).reorder(nil).find_in_batches do |lists|
FeedInsertWorker.push_bulk(lists) do |list|
[status.id, list.id, :list]
end end
end end
end end
@ -49,7 +60,7 @@ class FanOutOnWriteService < BaseService
status.mentions.includes(:account).each do |mention| status.mentions.includes(:account).each do |mention|
mentioned_account = mention.account mentioned_account = mention.account
next if !mentioned_account.local? || !mentioned_account.following?(status.account) || FeedManager.instance.filter?(:home, status, mention.account_id) next if !mentioned_account.local? || !mentioned_account.following?(status.account) || FeedManager.instance.filter?(:home, status, mention.account_id)
FeedManager.instance.push(:home, mentioned_account, status) FeedManager.instance.push_to_home(mentioned_account, status)
end end
end end

@ -14,6 +14,7 @@ class RemoveStatusService < BaseService
remove_from_self if status.account.local? remove_from_self if status.account.local?
remove_from_followers remove_from_followers
remove_from_lists
remove_from_affected remove_from_affected
remove_reblogs remove_reblogs
remove_from_hashtags remove_from_hashtags
@ -30,12 +31,18 @@ class RemoveStatusService < BaseService
private private
def remove_from_self def remove_from_self
unpush(:home, @account, @status) FeedManager.instance.unpush_from_home(@account, @status)
end end
def remove_from_followers def remove_from_followers
@account.followers.local.find_each do |follower| @account.followers.local.find_each do |follower|
unpush(:home, follower, @status) FeedManager.instance.unpush_from_home(follower, @status)
end
end
def remove_from_lists
@account.lists.select(:id, :account_id).find_each do |list|
FeedManager.instance.unpush_from_list(list, @status)
end end
end end
@ -101,10 +108,6 @@ class RemoveStatusService < BaseService
end end
end end
def unpush(type, receiver, status)
FeedManager.instance.unpush(type, receiver, status)
end
def remove_from_hashtags def remove_from_hashtags
return unless @status.public_visibility? return unless @status.public_visibility?

@ -3,34 +3,41 @@
class FeedInsertWorker class FeedInsertWorker
include Sidekiq::Worker include Sidekiq::Worker
attr_reader :status, :follower def perform(status_id, id, type = :home)
@type = type.to_sym
def perform(status_id, follower_id) @status = Status.find(status_id)
@status = Status.find_by(id: status_id)
@follower = Account.find_by(id: follower_id) case @type
when :home
@follower = Account.find(id)
when :list
@list = List.find(id)
@follower = @list.account
end
check_and_insert check_and_insert
rescue ActiveRecord::RecordNotFound
true
end end
private private
def check_and_insert def check_and_insert
if records_available? perform_push unless feed_filtered?
perform_push unless feed_filtered?
else
true
end
end
def records_available?
status.present? && follower.present?
end end
def feed_filtered? def feed_filtered?
FeedManager.instance.filter?(:home, status, follower.id) # Note: Lists are a variation of home, so the filtering rules
# of home apply to both
FeedManager.instance.filter?(:home, @status, @follower.id)
end end
def perform_push def perform_push
FeedManager.instance.push(:home, follower, status) case @type
when :home
FeedManager.instance.push_to_home(@follower, @status)
when :list
FeedManager.instance.push_to_list(@list, @status)
end
end end
end end

@ -3,12 +3,13 @@
class PushUpdateWorker class PushUpdateWorker
include Sidekiq::Worker include Sidekiq::Worker
def perform(account_id, status_id) def perform(account_id, status_id, timeline_id = nil)
account = Account.find(account_id) account = Account.find(account_id)
status = Status.find(status_id) status = Status.find(status_id)
message = InlineRenderer.render(status, account, :status) message = InlineRenderer.render(status, account, :status)
timeline_id = "timeline:#{account.id}" if timeline_id.nil?
Redis.current.publish("timeline:#{account.id}", Oj.dump(event: :update, payload: message, queued_at: (Time.now.to_f * 1000.0).to_i)) Redis.current.publish(timeline_id, Oj.dump(event: :update, payload: message, queued_at: (Time.now.to_f * 1000.0).to_i))
rescue ActiveRecord::RecordNotFound rescue ActiveRecord::RecordNotFound
true true
end end

@ -212,6 +212,7 @@ Rails.application.routes.draw do
resource :home, only: :show, controller: :home resource :home, only: :show, controller: :home
resource :public, only: :show, controller: :public resource :public, only: :show, controller: :public
resources :tag, only: :show resources :tag, only: :show
resources :list, only: :show
end end
resources :streaming, only: [:index] resources :streaming, only: [:index]
@ -270,6 +271,10 @@ Rails.application.routes.draw do
post :unmute post :unmute
end end
end end
resources :lists, only: [:index, :create, :show, :update, :destroy] do
resource :accounts, only: [:show, :create, :destroy], controller: 'lists/accounts'
end
end end
namespace :web do namespace :web do

@ -0,0 +1,10 @@
class CreateLists < ActiveRecord::Migration[5.1]
def change
create_table :lists do |t|
t.references :account, foreign_key: { on_delete: :cascade }
t.string :title, null: false, default: ''
t.timestamps
end
end
end

@ -0,0 +1,12 @@
class CreateListAccounts < ActiveRecord::Migration[5.1]
def change
create_table :list_accounts do |t|
t.belongs_to :list, foreign_key: { on_delete: :cascade }, null: false
t.belongs_to :account, foreign_key: { on_delete: :cascade }, null: false
t.belongs_to :follow, foreign_key: { on_delete: :cascade }, null: false
end
add_index :list_accounts, [:account_id, :list_id], unique: true
add_index :list_accounts, [:list_id, :account_id]
end
end

@ -10,7 +10,7 @@
# #
# It's strongly recommended that you check this file into your version control system. # It's strongly recommended that you check this file into your version control system.
ActiveRecord::Schema.define(version: 20171114080328) do ActiveRecord::Schema.define(version: 20171116161857) do
# These are extensions that must be enabled in order to support this database # These are extensions that must be enabled in order to support this database
enable_extension "plpgsql" enable_extension "plpgsql"
@ -170,6 +170,25 @@ ActiveRecord::Schema.define(version: 20171114080328) do
t.bigint "account_id", null: false t.bigint "account_id", null: false
end end
create_table "list_accounts", force: :cascade do |t|
t.bigint "list_id", null: false
t.bigint "account_id", null: false
t.bigint "follow_id", null: false
t.index ["account_id", "list_id"], name: "index_list_accounts_on_account_id_and_list_id", unique: true
t.index ["account_id"], name: "index_list_accounts_on_account_id"
t.index ["follow_id"], name: "index_list_accounts_on_follow_id"
t.index ["list_id", "account_id"], name: "index_list_accounts_on_list_id_and_account_id"
t.index ["list_id"], name: "index_list_accounts_on_list_id"
end
create_table "lists", force: :cascade do |t|
t.bigint "account_id"
t.string "title", default: "", null: false
t.datetime "created_at", null: false
t.datetime "updated_at", null: false
t.index ["account_id"], name: "index_lists_on_account_id"
end
create_table "media_attachments", force: :cascade do |t| create_table "media_attachments", force: :cascade do |t|
t.bigint "status_id" t.bigint "status_id"
t.string "file_file_name" t.string "file_file_name"
@ -478,6 +497,10 @@ ActiveRecord::Schema.define(version: 20171114080328) do
add_foreign_key "follows", "accounts", column: "target_account_id", name: "fk_745ca29eac", on_delete: :cascade add_foreign_key "follows", "accounts", column: "target_account_id", name: "fk_745ca29eac", on_delete: :cascade
add_foreign_key "follows", "accounts", name: "fk_32ed1b5560", on_delete: :cascade add_foreign_key "follows", "accounts", name: "fk_32ed1b5560", on_delete: :cascade
add_foreign_key "imports", "accounts", name: "fk_6db1b6e408", on_delete: :cascade add_foreign_key "imports", "accounts", name: "fk_6db1b6e408", on_delete: :cascade
add_foreign_key "list_accounts", "accounts", on_delete: :cascade
add_foreign_key "list_accounts", "follows", on_delete: :cascade
add_foreign_key "list_accounts", "lists", on_delete: :cascade
add_foreign_key "lists", "accounts", on_delete: :cascade
add_foreign_key "media_attachments", "accounts", name: "fk_96dd81e81b", on_delete: :nullify add_foreign_key "media_attachments", "accounts", name: "fk_96dd81e81b", on_delete: :nullify
add_foreign_key "media_attachments", "statuses", on_delete: :nullify add_foreign_key "media_attachments", "statuses", on_delete: :nullify
add_foreign_key "mentions", "accounts", name: "fk_970d43f9d1", on_delete: :cascade add_foreign_key "mentions", "accounts", name: "fk_970d43f9d1", on_delete: :cascade

@ -0,0 +1,54 @@
require 'rails_helper'
describe Api::V1::Lists::AccountsController do
render_views
let(:user) { Fabricate(:user, account: Fabricate(:account, username: 'alice')) }
let(:token) { Fabricate(:accessible_access_token, resource_owner_id: user.id, scopes: 'read write') }
let(:list) { Fabricate(:list, account: user.account) }
before do
follow = Fabricate(:follow, account: user.account)
list.accounts << follow.target_account
allow(controller).to receive(:doorkeeper_token) { token }
end
describe 'GET #index' do
it 'returns http success' do
get :show, params: { list_id: list.id }
expect(response).to have_http_status(:success)
end
end
describe 'POST #create' do
let(:bob) { Fabricate(:account, username: 'bob') }
before do
user.account.follow!(bob)
post :create, params: { list_id: list.id, account_ids: [bob.id] }
end
it 'returns http success' do
expect(response).to have_http_status(:success)
end
it 'adds account to the list' do
expect(list.accounts.include?(bob)).to be true
end
end
describe 'DELETE #destroy' do
before do
delete :destroy, params: { list_id: list.id, account_ids: [list.accounts.first.id] }
end
it 'returns http success' do
expect(response).to have_http_status(:success)
end
it 'removes account from the list' do
expect(list.accounts.count).to eq 0
end
end
end

@ -0,0 +1,68 @@
require 'rails_helper'
RSpec.describe Api::V1::ListsController, type: :controller do
render_views
let!(:user) { Fabricate(:user, account: Fabricate(:account, username: 'alice')) }
let!(:token) { Fabricate(:accessible_access_token, resource_owner_id: user.id, scopes: 'read write') }
let!(:list) { Fabricate(:list, account: user.account) }
before { allow(controller).to receive(:doorkeeper_token) { token } }
describe 'GET #index' do
it 'returns http success' do
get :index
expect(response).to have_http_status(:success)
end
end
describe 'GET #show' do
it 'returns http success' do
get :show, params: { id: list.id }
expect(response).to have_http_status(:success)
end
end
describe 'POST #create' do
before do
post :create, params: { title: 'Foo bar' }
end
it 'returns http success' do
expect(response).to have_http_status(:success)
end
it 'creates list' do
expect(List.where(account: user.account).count).to eq 2
expect(List.last.title).to eq 'Foo bar'
end
end
describe 'PUT #update' do
before do
put :update, params: { id: list.id, title: 'Updated title' }
end
it 'returns http success' do
expect(response).to have_http_status(:success)
end
it 'updates the list' do
expect(list.reload.title).to eq 'Updated title'
end
end
describe 'DELETE #destroy' do
before do
delete :destroy, params: { id: list.id }
end
it 'returns http success' do
expect(response).to have_http_status(:success)
end
it 'deletes the list' do
expect(List.find_by(id: list.id)).to be_nil
end
end
end

@ -0,0 +1,56 @@
# frozen_string_literal: true
require 'rails_helper'
describe Api::V1::Timelines::ListController do
render_views
let(:user) { Fabricate(:user, account: Fabricate(:account, username: 'alice')) }
let(:list) { Fabricate(:list, account: user.account) }
before do
allow(controller).to receive(:doorkeeper_token) { token }
end
context 'with a user context' do
let(:token) { Fabricate(:accessible_access_token, resource_owner_id: user.id, scopes: 'read') }
describe 'GET #show' do
before do
follow = Fabricate(:follow, account: user.account)
list.accounts << follow.target_account
PostStatusService.new.call(follow.target_account, 'New status for user home timeline.')
end
it 'returns http success' do
get :show, params: { id: list.id }
expect(response).to have_http_status(:success)
end
end
end
context 'with the wrong user context' do
let(:other_user) { Fabricate(:user, account: Fabricate(:account, username: 'bob')) }
let(:token) { Fabricate(:accessible_access_token, resource_owner_id: other_user.id, scopes: 'read') }
describe 'GET #show' do
it 'returns http not found' do
get :show, params: { id: list.id }
expect(response).to have_http_status(:not_found)
end
end
end
context 'without a user context' do
let(:token) { Fabricate(:accessible_access_token, resource_owner_id: nil, scopes: 'read') }
describe 'GET #show' do
it 'returns http unprocessable entity' do
get :show, params: { id: list.id }
expect(response).to have_http_status(:unprocessable_entity)
expect(response.headers['Link']).to be_nil
end
end
end
end

@ -5,7 +5,7 @@ require 'rails_helper'
describe Api::V1::Timelines::TagController do describe Api::V1::Timelines::TagController do
render_views render_views
let(:user) { Fabricate(:user, account: Fabricate(:account, username: 'alice')) } let(:user) { Fabricate(:user, account: Fabricate(:account, username: 'alice')) }
before do before do
allow(controller).to receive(:doorkeeper_token) { token } allow(controller).to receive(:doorkeeper_token) { token }

@ -0,0 +1,5 @@
Fabricator(:list_account) do
list nil
account nil
follow nil
end

@ -0,0 +1,4 @@
Fabricator(:list) do
account nil
title "MyString"
end

@ -148,21 +148,11 @@ RSpec.describe FeedManager do
account = Fabricate(:account) account = Fabricate(:account)
status = Fabricate(:status) status = Fabricate(:status)
members = FeedManager::MAX_ITEMS.times.map { |count| [count, count] } members = FeedManager::MAX_ITEMS.times.map { |count| [count, count] }
Redis.current.zadd("feed:type:#{account.id}", members) Redis.current.zadd("feed:home:#{account.id}", members)
FeedManager.instance.push('type', account, status) FeedManager.instance.push_to_home(account, status)
expect(Redis.current.zcard("feed:type:#{account.id}")).to eq FeedManager::MAX_ITEMS expect(Redis.current.zcard("feed:home:#{account.id}")).to eq FeedManager::MAX_ITEMS
end
it 'sends push updates for non-home timelines' do
account = Fabricate(:account)
status = Fabricate(:status)
allow(Redis.current).to receive_messages(publish: nil)
FeedManager.instance.push('type', account, status)
expect(Redis.current).to have_received(:publish).with("timeline:#{account.id}", any_args).at_least(:once)
end end
context 'reblogs' do context 'reblogs' do
@ -171,7 +161,7 @@ RSpec.describe FeedManager do
reblogged = Fabricate(:status) reblogged = Fabricate(:status)
reblog = Fabricate(:status, reblog: reblogged) reblog = Fabricate(:status, reblog: reblogged)
expect(FeedManager.instance.push('type', account, reblog)).to be true expect(FeedManager.instance.push_to_home(account, reblog)).to be true
end end
it 'does not save a new reblog of a recent status' do it 'does not save a new reblog of a recent status' do
@ -179,9 +169,9 @@ RSpec.describe FeedManager do
reblogged = Fabricate(:status) reblogged = Fabricate(:status)
reblog = Fabricate(:status, reblog: reblogged) reblog = Fabricate(:status, reblog: reblogged)
FeedManager.instance.push('type', account, reblogged) FeedManager.instance.push_to_home(account, reblogged)
expect(FeedManager.instance.push('type', account, reblog)).to be false expect(FeedManager.instance.push_to_home(account, reblog)).to be false
end end
it 'saves a new reblog of an old status' do it 'saves a new reblog of an old status' do
@ -189,14 +179,14 @@ RSpec.describe FeedManager do
reblogged = Fabricate(:status) reblogged = Fabricate(:status)
reblog = Fabricate(:status, reblog: reblogged) reblog = Fabricate(:status, reblog: reblogged)
FeedManager.instance.push('type', account, reblogged) FeedManager.instance.push_to_home(account, reblogged)
# Fill the feed with intervening statuses # Fill the feed with intervening statuses
FeedManager::REBLOG_FALLOFF.times do FeedManager::REBLOG_FALLOFF.times do
FeedManager.instance.push('type', account, Fabricate(:status)) FeedManager.instance.push_to_home(account, Fabricate(:status))
end end
expect(FeedManager.instance.push('type', account, reblog)).to be true expect(FeedManager.instance.push_to_home(account, reblog)).to be true
end end
it 'does not save a new reblog of a recently-reblogged status' do it 'does not save a new reblog of a recently-reblogged status' do
@ -205,10 +195,10 @@ RSpec.describe FeedManager do
reblogs = 2.times.map { Fabricate(:status, reblog: reblogged) } reblogs = 2.times.map { Fabricate(:status, reblog: reblogged) }
# The first reblog will be accepted # The first reblog will be accepted
FeedManager.instance.push('type', account, reblogs.first) FeedManager.instance.push_to_home(account, reblogs.first)
# The second reblog should be ignored # The second reblog should be ignored
expect(FeedManager.instance.push('type', account, reblogs.last)).to be false expect(FeedManager.instance.push_to_home(account, reblogs.last)).to be false
end end
it 'does not save a new reblog of a multiply-reblogged-then-unreblogged status' do it 'does not save a new reblog of a multiply-reblogged-then-unreblogged status' do
@ -217,14 +207,14 @@ RSpec.describe FeedManager do
reblogs = 3.times.map { Fabricate(:status, reblog: reblogged) } reblogs = 3.times.map { Fabricate(:status, reblog: reblogged) }
# Accept the reblogs # Accept the reblogs
FeedManager.instance.push('type', account, reblogs[0]) FeedManager.instance.push_to_home(account, reblogs[0])
FeedManager.instance.push('type', account, reblogs[1]) FeedManager.instance.push_to_home(account, reblogs[1])
# Unreblog the first one # Unreblog the first one
FeedManager.instance.unpush('type', account, reblogs[0]) FeedManager.instance.unpush_from_home(account, reblogs[0])
# The last reblog should still be ignored # The last reblog should still be ignored
expect(FeedManager.instance.push('type', account, reblogs.last)).to be false expect(FeedManager.instance.push_to_home(account, reblogs.last)).to be false
end end
it 'saves a new reblog of a long-ago-reblogged status' do it 'saves a new reblog of a long-ago-reblogged status' do
@ -233,15 +223,15 @@ RSpec.describe FeedManager do
reblogs = 2.times.map { Fabricate(:status, reblog: reblogged) } reblogs = 2.times.map { Fabricate(:status, reblog: reblogged) }
# The first reblog will be accepted # The first reblog will be accepted
FeedManager.instance.push('type', account, reblogs.first) FeedManager.instance.push_to_home(account, reblogs.first)
# Fill the feed with intervening statuses # Fill the feed with intervening statuses
FeedManager::REBLOG_FALLOFF.times do FeedManager::REBLOG_FALLOFF.times do
FeedManager.instance.push('type', account, Fabricate(:status)) FeedManager.instance.push_to_home(account, Fabricate(:status))
end end
# The second reblog should also be accepted # The second reblog should also be accepted
expect(FeedManager.instance.push('type', account, reblogs.last)).to be true expect(FeedManager.instance.push_to_home(account, reblogs.last)).to be true
end end
end end
end end
@ -253,11 +243,11 @@ RSpec.describe FeedManager do
reblogged = Fabricate(:status) reblogged = Fabricate(:status)
status = Fabricate(:status, reblog: reblogged) status = Fabricate(:status, reblog: reblogged)
another_status = Fabricate(:status, reblog: reblogged) another_status = Fabricate(:status, reblog: reblogged)
reblogs_key = FeedManager.instance.key('type', receiver.id, 'reblogs') reblogs_key = FeedManager.instance.key('home', receiver.id, 'reblogs')
reblog_set_key = FeedManager.instance.key('type', receiver.id, "reblogs:#{reblogged.id}") reblog_set_key = FeedManager.instance.key('home', receiver.id, "reblogs:#{reblogged.id}")
FeedManager.instance.push('type', receiver, status) FeedManager.instance.push_to_home(receiver, status)
FeedManager.instance.push('type', receiver, another_status) FeedManager.instance.push_to_home(receiver, another_status)
# We should have a tracking set and an entry in reblogs. # We should have a tracking set and an entry in reblogs.
expect(Redis.current.exists(reblog_set_key)).to be true expect(Redis.current.exists(reblog_set_key)).to be true
@ -265,12 +255,12 @@ RSpec.describe FeedManager do
# Push everything off the end of the feed. # Push everything off the end of the feed.
FeedManager::MAX_ITEMS.times do FeedManager::MAX_ITEMS.times do
FeedManager.instance.push('type', receiver, Fabricate(:status)) FeedManager.instance.push_to_home(receiver, Fabricate(:status))
end end
# `trim` should be called automatically, but do it anyway, as # `trim` should be called automatically, but do it anyway, as
# we're testing `trim`, not side effects of `push`. # we're testing `trim`, not side effects of `push`.
FeedManager.instance.trim('type', receiver.id) FeedManager.instance.trim('home', receiver.id)
# We should not have any reblog tracking data. # We should not have any reblog tracking data.
expect(Redis.current.exists(reblog_set_key)).to be false expect(Redis.current.exists(reblog_set_key)).to be false
@ -285,32 +275,32 @@ RSpec.describe FeedManager do
reblogged = Fabricate(:status) reblogged = Fabricate(:status)
status = Fabricate(:status, reblog: reblogged) status = Fabricate(:status, reblog: reblogged)
FeedManager.instance.push('type', receiver, reblogged) FeedManager.instance.push_to_home(receiver, reblogged)
FeedManager::REBLOG_FALLOFF.times { FeedManager.instance.push('type', receiver, Fabricate(:status)) } FeedManager::REBLOG_FALLOFF.times { FeedManager.instance.push_to_home(receiver, Fabricate(:status)) }
FeedManager.instance.push('type', receiver, status) FeedManager.instance.push_to_home(receiver, status)
# The reblogging status should show up under normal conditions. # The reblogging status should show up under normal conditions.
expect(Redis.current.zrange("feed:type:#{receiver.id}", 0, -1)).to include(status.id.to_s) expect(Redis.current.zrange("feed:home:#{receiver.id}", 0, -1)).to include(status.id.to_s)
FeedManager.instance.unpush('type', receiver, status) FeedManager.instance.unpush_from_home(receiver, status)
# Restore original status # Restore original status
expect(Redis.current.zrange("feed:type:#{receiver.id}", 0, -1)).to_not include(status.id.to_s) expect(Redis.current.zrange("feed:home:#{receiver.id}", 0, -1)).to_not include(status.id.to_s)
expect(Redis.current.zrange("feed:type:#{receiver.id}", 0, -1)).to include(reblogged.id.to_s) expect(Redis.current.zrange("feed:home:#{receiver.id}", 0, -1)).to include(reblogged.id.to_s)
end end
it 'removes a reblogged status if it was only reblogged once' do it 'removes a reblogged status if it was only reblogged once' do
reblogged = Fabricate(:status) reblogged = Fabricate(:status)
status = Fabricate(:status, reblog: reblogged) status = Fabricate(:status, reblog: reblogged)
FeedManager.instance.push('type', receiver, status) FeedManager.instance.push_to_home(receiver, status)
# The reblogging status should show up under normal conditions. # The reblogging status should show up under normal conditions.
expect(Redis.current.zrange("feed:type:#{receiver.id}", 0, -1)).to eq [status.id.to_s] expect(Redis.current.zrange("feed:home:#{receiver.id}", 0, -1)).to eq [status.id.to_s]
FeedManager.instance.unpush('type', receiver, status) FeedManager.instance.unpush_from_home(receiver, status)
expect(Redis.current.zrange("feed:type:#{receiver.id}", 0, -1)).to be_empty expect(Redis.current.zrange("feed:home:#{receiver.id}", 0, -1)).to be_empty
end end
it 'leaves a multiply-reblogged status if another reblog was in feed' do it 'leaves a multiply-reblogged status if another reblog was in feed' do
@ -318,26 +308,26 @@ RSpec.describe FeedManager do
reblogs = 3.times.map { Fabricate(:status, reblog: reblogged) } reblogs = 3.times.map { Fabricate(:status, reblog: reblogged) }
reblogs.each do |reblog| reblogs.each do |reblog|
FeedManager.instance.push('type', receiver, reblog) FeedManager.instance.push_to_home(receiver, reblog)
end end
# The reblogging status should show up under normal conditions. # The reblogging status should show up under normal conditions.
expect(Redis.current.zrange("feed:type:#{receiver.id}", 0, -1)).to eq [reblogs.first.id.to_s] expect(Redis.current.zrange("feed:home:#{receiver.id}", 0, -1)).to eq [reblogs.first.id.to_s]
reblogs[0...-1].each do |reblog| reblogs[0...-1].each do |reblog|
FeedManager.instance.unpush('type', receiver, reblog) FeedManager.instance.unpush_from_home(receiver, reblog)
end end
expect(Redis.current.zrange("feed:type:#{receiver.id}", 0, -1)).to eq [reblogs.last.id.to_s] expect(Redis.current.zrange("feed:home:#{receiver.id}", 0, -1)).to eq [reblogs.last.id.to_s]
end end
it 'sends push updates' do it 'sends push updates' do
status = Fabricate(:status) status = Fabricate(:status)
FeedManager.instance.push('type', receiver, status) FeedManager.instance.push_to_home(receiver, status)
allow(Redis.current).to receive_messages(publish: nil) allow(Redis.current).to receive_messages(publish: nil)
FeedManager.instance.unpush('type', receiver, status) FeedManager.instance.unpush_from_home(receiver, status)
deletion = Oj.dump(event: :delete, payload: status.id.to_s) deletion = Oj.dump(event: :delete, payload: status.id.to_s)
expect(Redis.current).to have_received(:publish).with("timeline:#{receiver.id}", deletion) expect(Redis.current).to have_received(:publish).with("timeline:#{receiver.id}", deletion)

@ -1,5 +1,5 @@
require 'rails_helper' require 'rails_helper'
RSpec.describe AccountModerationNote, type: :model do RSpec.describe AccountModerationNote, type: :model do
pending "add some examples to (or delete) #{__FILE__}"
end end

@ -1,9 +1,9 @@
require 'rails_helper' require 'rails_helper'
RSpec.describe Feed, type: :model do RSpec.describe HomeFeed, type: :model do
let(:account) { Fabricate(:account) } let(:account) { Fabricate(:account) }
subject { described_class.new(:home, account) } subject { described_class.new(account) }
describe '#get' do describe '#get' do
before do before do

@ -0,0 +1,5 @@
require 'rails_helper'
RSpec.describe ListAccount, type: :model do
end

@ -0,0 +1,5 @@
require 'rails_helper'
RSpec.describe List, type: :model do
end

@ -18,8 +18,8 @@ RSpec.describe AfterBlockService do
end end
it "clears account's statuses" do it "clears account's statuses" do
FeedManager.instance.push(:home, account, status) FeedManager.instance.push_to_home(account, status)
FeedManager.instance.push(:home, account, other_account_status) FeedManager.instance.push_to_home(account, other_account_status)
is_expected.to change { is_expected.to change {
Redis.current.zrange(home_timeline_key, 0, -1) Redis.current.zrange(home_timeline_key, 0, -1)

@ -30,11 +30,11 @@ RSpec.describe BatchedRemoveStatusService do
end end
it 'removes statuses from author\'s home feed' do it 'removes statuses from author\'s home feed' do
expect(Feed.new(:home, alice).get(10)).to_not include([status1.id, status2.id]) expect(HomeFeed.new(alice).get(10)).to_not include([status1.id, status2.id])
end end
it 'removes statuses from local follower\'s home feed' do it 'removes statuses from local follower\'s home feed' do
expect(Feed.new(:home, jeff).get(10)).to_not include([status1.id, status2.id]) expect(HomeFeed.new(jeff).get(10)).to_not include([status1.id, status2.id])
end end
it 'notifies streaming API of followers' do it 'notifies streaming API of followers' do

@ -19,12 +19,12 @@ RSpec.describe FanOutOnWriteService do
end end
it 'delivers status to home timeline' do it 'delivers status to home timeline' do
expect(Feed.new(:home, author).get(10).map(&:id)).to include status.id expect(HomeFeed.new(author).get(10).map(&:id)).to include status.id
end end
it 'delivers status to local followers' do it 'delivers status to local followers' do
pending 'some sort of problem in test environment causes this to sometimes fail' pending 'some sort of problem in test environment causes this to sometimes fail'
expect(Feed.new(:home, follower).get(10).map(&:id)).to include status.id expect(HomeFeed.new(follower).get(10).map(&:id)).to include status.id
end end
it 'delivers status to hashtag' do it 'delivers status to hashtag' do

@ -18,8 +18,8 @@ RSpec.describe MuteService do
end end
it "clears account's statuses" do it "clears account's statuses" do
FeedManager.instance.push(:home, account, status) FeedManager.instance.push_to_home(account, status)
FeedManager.instance.push(:home, account, other_account_status) FeedManager.instance.push_to_home(account, other_account_status)
is_expected.to change { is_expected.to change {
Redis.current.zrange(home_timeline_key, 0, -1) Redis.current.zrange(home_timeline_key, 0, -1)

@ -25,11 +25,11 @@ RSpec.describe RemoveStatusService do
end end
it 'removes status from author\'s home feed' do it 'removes status from author\'s home feed' do
expect(Feed.new(:home, alice).get(10)).to_not include(@status.id) expect(HomeFeed.new(alice).get(10)).to_not include(@status.id)
end end
it 'removes status from local follower\'s home feed' do it 'removes status from local follower\'s home feed' do
expect(Feed.new(:home, jeff).get(10)).to_not include(@status.id) expect(HomeFeed.new(jeff).get(10)).to_not include(@status.id)
end end
it 'sends PuSH update to PuSH subscribers' do it 'sends PuSH update to PuSH subscribers' do

@ -11,41 +11,41 @@ describe FeedInsertWorker do
context 'when there are no records' do context 'when there are no records' do
it 'skips push with missing status' do it 'skips push with missing status' do
instance = double(push: nil) instance = double(push_to_home: nil)
allow(FeedManager).to receive(:instance).and_return(instance) allow(FeedManager).to receive(:instance).and_return(instance)
result = subject.perform(nil, follower.id) result = subject.perform(nil, follower.id)
expect(result).to eq true expect(result).to eq true
expect(instance).not_to have_received(:push) expect(instance).not_to have_received(:push_to_home)
end end
it 'skips push with missing account' do it 'skips push with missing account' do
instance = double(push: nil) instance = double(push_to_home: nil)
allow(FeedManager).to receive(:instance).and_return(instance) allow(FeedManager).to receive(:instance).and_return(instance)
result = subject.perform(status.id, nil) result = subject.perform(status.id, nil)
expect(result).to eq true expect(result).to eq true
expect(instance).not_to have_received(:push) expect(instance).not_to have_received(:push_to_home)
end end
end end
context 'when there are real records' do context 'when there are real records' do
it 'skips the push when there is a filter' do it 'skips the push when there is a filter' do
instance = double(push: nil, filter?: true) instance = double(push_to_home: nil, filter?: true)
allow(FeedManager).to receive(:instance).and_return(instance) allow(FeedManager).to receive(:instance).and_return(instance)
result = subject.perform(status.id, follower.id) result = subject.perform(status.id, follower.id)
expect(result).to be_nil expect(result).to be_nil
expect(instance).not_to have_received(:push) expect(instance).not_to have_received(:push_to_home)
end end
it 'pushes the status onto the home timeline without filter' do it 'pushes the status onto the home timeline without filter' do
instance = double(push: nil, filter?: false) instance = double(push_to_home: nil, filter?: false)
allow(FeedManager).to receive(:instance).and_return(instance) allow(FeedManager).to receive(:instance).and_return(instance)
result = subject.perform(status.id, follower.id) result = subject.perform(status.id, follower.id)
expect(result).to be_nil expect(result).to be_nil
expect(instance).to have_received(:push).with(:home, follower, status) expect(instance).to have_received(:push_to_home).with(follower, status)
end end
end end
end end

@ -254,6 +254,26 @@ const startWorker = (workerId) => {
const placeholders = (arr, shift = 0) => arr.map((_, i) => `$${i + 1 + shift}`).join(', '); const placeholders = (arr, shift = 0) => arr.map((_, i) => `$${i + 1 + shift}`).join(', ');
const authorizeListAccess = (id, req, next) => {
pgPool.connect((err, client, done) => {
if (err) {
next(false);
return;
}
client.query('SELECT id, account_id FROM lists WHERE id = $1 LIMIT 1', [id], (err, result) => {
done();
if (err || result.rows.length === 0 || result.rows[0].account_id !== req.accountId) {
next(false);
return;
}
next(true);
});
});
};
const streamFrom = (id, req, output, attachCloseHandler, needsFiltering = false, notificationOnly = false) => { const streamFrom = (id, req, output, attachCloseHandler, needsFiltering = false, notificationOnly = false) => {
const streamType = notificationOnly ? ' (notification)' : ''; const streamType = notificationOnly ? ' (notification)' : '';
log.verbose(req.requestId, `Starting stream from ${id} for ${req.accountId}${streamType}`); log.verbose(req.requestId, `Starting stream from ${id} for ${req.accountId}${streamType}`);
@ -410,7 +430,22 @@ const startWorker = (workerId) => {
streamFrom(`timeline:hashtag:${req.query.tag.toLowerCase()}:local`, req, streamToHttp(req, res), streamHttpEnd(req), true); streamFrom(`timeline:hashtag:${req.query.tag.toLowerCase()}:local`, req, streamToHttp(req, res), streamHttpEnd(req), true);
}); });
const wss = new WebSocket.Server({ server, verifyClient: wsVerifyClient }); app.get('/api/v1/streaming/list', (req, res) => {
const listId = req.query.list;
authorizeListAccess(listId, req, authorized => {
if (!authorized) {
res.writeHead(404, { 'Content-Type': 'application/json' });
res.end(JSON.stringify({ error: 'Not found' }));
return;
}
const channel = `timeline:list:${listId}`;
streamFrom(channel, req, streamToHttp(req, res), streamHttpEnd(req, subscriptionHeartbeat(channel)));
});
});
const wss = new WebSocket.Server({ server, verifyClient: wsVerifyClient });
wss.on('connection', ws => { wss.on('connection', ws => {
const req = ws.upgradeReq; const req = ws.upgradeReq;
@ -443,6 +478,19 @@ const startWorker = (workerId) => {
case 'hashtag:local': case 'hashtag:local':
streamFrom(`timeline:hashtag:${location.query.tag.toLowerCase()}:local`, req, streamToWs(req, ws), streamWsEnd(req, ws), true); streamFrom(`timeline:hashtag:${location.query.tag.toLowerCase()}:local`, req, streamToWs(req, ws), streamWsEnd(req, ws), true);
break; break;
case 'list':
const listId = location.query.list;
authorizeListAccess(listId, req, authorized => {
if (!authorized) {
ws.close();
return;
}
const channel = `timeline:list:${listId}`;
streamFrom(channel, req, streamToWs(req, ws), streamWsEnd(req, ws, subscriptionHeartbeat(channel)));
});
break;
default: default:
ws.close(); ws.close();
} }

Loading…
Cancel
Save